diff --git a/Base/src/main/java/io/deephaven/base/MathUtil.java b/Base/src/main/java/io/deephaven/base/MathUtil.java index 1f07a5923ef..555409a709b 100644 --- a/Base/src/main/java/io/deephaven/base/MathUtil.java +++ b/Base/src/main/java/io/deephaven/base/MathUtil.java @@ -8,6 +8,11 @@ */ public class MathUtil { + /** + * The maximum power of 2. + */ + public static final int MAX_POWER_OF_2 = 1 << 30; + /** * Compute ceil(log2(x)). See {@link Integer#numberOfLeadingZeros(int)}. * @@ -108,4 +113,36 @@ public static int base10digits(int n) { } return base10guess; } + + /** + * Rounds up to the next power of 2 for {@code x}; if {@code x} is already a power of 2, {@code x} will be returned. + * Values outside the range {@code 1 <= x <= MAX_POWER_OF_2} will return {@code 1}. + * + *

+ * Equivalent to {@code Math.max(Integer.highestOneBit(x - 1) << 1, 1)}. + * + * @param x the value + * @return the next power of 2 for {@code x} + * @see #MAX_POWER_OF_2 + */ + public static int roundUpPowerOf2(int x) { + return Math.max(Integer.highestOneBit(x - 1) << 1, 1); + } + + /** + * Rounds up to the next power of 2 for {@code size <= MAX_POWER_OF_2}, otherwise returns + * {@link ArrayUtil#MAX_ARRAY_SIZE}. + * + *

+ * Equivalent to {@code size <= MAX_POWER_OF_2 ? roundUpPowerOf2(size) : ArrayUtil.MAX_ARRAY_SIZE}. + * + * @param size the size + * @return the + * @see #MAX_POWER_OF_2 + * @see #roundUpPowerOf2(int) + * @see ArrayUtil#MAX_ARRAY_SIZE + */ + public static int roundUpArraySize(int size) { + return size <= MAX_POWER_OF_2 ? roundUpPowerOf2(size) : ArrayUtil.MAX_ARRAY_SIZE; + } } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/ByteRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/ByteRingBuffer.java index ee8e47041cd..aede7521309 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/ByteRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/ByteRingBuffer.java @@ -7,7 +7,7 @@ // @formatter:off package io.deephaven.base.ringbuffer; -import io.deephaven.base.ArrayUtil; +import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; import java.io.Serializable; @@ -20,8 +20,6 @@ * determination of storage indices through a mask operation. */ public class ByteRingBuffer implements Serializable { - /** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */ - static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE); static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; byte[] storage; @@ -45,21 +43,13 @@ public ByteRingBuffer(int capacity) { * @param growable whether to allow growth when the buffer is full. */ public ByteRingBuffer(int capacity, boolean growable) { - Assert.leq(capacity, "ByteRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(capacity, "ByteRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; // use next larger power of 2 for our storage - final int newCapacity; - if (capacity < 2) { - // sensibly handle the size=0 and size=1 cases - newCapacity = 1; - } else { - newCapacity = Integer.highestOneBit(capacity - 1) << 1; - } - // reset the data structure members - storage = new byte[newCapacity]; + storage = new byte[MathUtil.roundUpPowerOf2(capacity)]; mask = storage.length - 1; tail = head = 0; } @@ -73,9 +63,9 @@ protected void grow(int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible - Assert.leq(newCapacity, "ByteRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(newCapacity, "ByteRingBuffer capacity", MathUtil.MAX_POWER_OF_2); - final byte[] newStorage = new byte[Integer.highestOneBit((int) newCapacity - 1) << 1]; + final byte[] newStorage = new byte[MathUtil.roundUpPowerOf2((int) newCapacity)]; // move the current data to the new buffer copyRingBufferToArray(newStorage); diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/CharRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/CharRingBuffer.java index a2d934e9537..68c58a866e7 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/CharRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/CharRingBuffer.java @@ -3,7 +3,7 @@ // package io.deephaven.base.ringbuffer; -import io.deephaven.base.ArrayUtil; +import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; import java.io.Serializable; @@ -16,8 +16,6 @@ * determination of storage indices through a mask operation. */ public class CharRingBuffer implements Serializable { - /** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */ - static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE); static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; char[] storage; @@ -41,21 +39,13 @@ public CharRingBuffer(int capacity) { * @param growable whether to allow growth when the buffer is full. */ public CharRingBuffer(int capacity, boolean growable) { - Assert.leq(capacity, "CharRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(capacity, "CharRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; // use next larger power of 2 for our storage - final int newCapacity; - if (capacity < 2) { - // sensibly handle the size=0 and size=1 cases - newCapacity = 1; - } else { - newCapacity = Integer.highestOneBit(capacity - 1) << 1; - } - // reset the data structure members - storage = new char[newCapacity]; + storage = new char[MathUtil.roundUpPowerOf2(capacity)]; mask = storage.length - 1; tail = head = 0; } @@ -69,9 +59,9 @@ protected void grow(int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible - Assert.leq(newCapacity, "CharRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(newCapacity, "CharRingBuffer capacity", MathUtil.MAX_POWER_OF_2); - final char[] newStorage = new char[Integer.highestOneBit((int) newCapacity - 1) << 1]; + final char[] newStorage = new char[MathUtil.roundUpPowerOf2((int) newCapacity)]; // move the current data to the new buffer copyRingBufferToArray(newStorage); diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/DoubleRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/DoubleRingBuffer.java index 3f0f5cfbe35..8e942f8937a 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/DoubleRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/DoubleRingBuffer.java @@ -7,7 +7,7 @@ // @formatter:off package io.deephaven.base.ringbuffer; -import io.deephaven.base.ArrayUtil; +import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; import java.io.Serializable; @@ -20,8 +20,6 @@ * determination of storage indices through a mask operation. */ public class DoubleRingBuffer implements Serializable { - /** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */ - static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE); static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; double[] storage; @@ -45,21 +43,13 @@ public DoubleRingBuffer(int capacity) { * @param growable whether to allow growth when the buffer is full. */ public DoubleRingBuffer(int capacity, boolean growable) { - Assert.leq(capacity, "DoubleRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(capacity, "DoubleRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; // use next larger power of 2 for our storage - final int newCapacity; - if (capacity < 2) { - // sensibly handle the size=0 and size=1 cases - newCapacity = 1; - } else { - newCapacity = Integer.highestOneBit(capacity - 1) << 1; - } - // reset the data structure members - storage = new double[newCapacity]; + storage = new double[MathUtil.roundUpPowerOf2(capacity)]; mask = storage.length - 1; tail = head = 0; } @@ -73,9 +63,9 @@ protected void grow(int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible - Assert.leq(newCapacity, "DoubleRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(newCapacity, "DoubleRingBuffer capacity", MathUtil.MAX_POWER_OF_2); - final double[] newStorage = new double[Integer.highestOneBit((int) newCapacity - 1) << 1]; + final double[] newStorage = new double[MathUtil.roundUpPowerOf2((int) newCapacity)]; // move the current data to the new buffer copyRingBufferToArray(newStorage); diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/FloatRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/FloatRingBuffer.java index 70cbded55ac..16c8751c447 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/FloatRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/FloatRingBuffer.java @@ -7,7 +7,7 @@ // @formatter:off package io.deephaven.base.ringbuffer; -import io.deephaven.base.ArrayUtil; +import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; import java.io.Serializable; @@ -20,8 +20,6 @@ * determination of storage indices through a mask operation. */ public class FloatRingBuffer implements Serializable { - /** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */ - static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE); static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; float[] storage; @@ -45,21 +43,13 @@ public FloatRingBuffer(int capacity) { * @param growable whether to allow growth when the buffer is full. */ public FloatRingBuffer(int capacity, boolean growable) { - Assert.leq(capacity, "FloatRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(capacity, "FloatRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; // use next larger power of 2 for our storage - final int newCapacity; - if (capacity < 2) { - // sensibly handle the size=0 and size=1 cases - newCapacity = 1; - } else { - newCapacity = Integer.highestOneBit(capacity - 1) << 1; - } - // reset the data structure members - storage = new float[newCapacity]; + storage = new float[MathUtil.roundUpPowerOf2(capacity)]; mask = storage.length - 1; tail = head = 0; } @@ -73,9 +63,9 @@ protected void grow(int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible - Assert.leq(newCapacity, "FloatRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(newCapacity, "FloatRingBuffer capacity", MathUtil.MAX_POWER_OF_2); - final float[] newStorage = new float[Integer.highestOneBit((int) newCapacity - 1) << 1]; + final float[] newStorage = new float[MathUtil.roundUpPowerOf2((int) newCapacity)]; // move the current data to the new buffer copyRingBufferToArray(newStorage); diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/IntRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/IntRingBuffer.java index 590014fa926..80a47f0a389 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/IntRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/IntRingBuffer.java @@ -7,7 +7,7 @@ // @formatter:off package io.deephaven.base.ringbuffer; -import io.deephaven.base.ArrayUtil; +import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; import java.io.Serializable; @@ -20,8 +20,6 @@ * determination of storage indices through a mask operation. */ public class IntRingBuffer implements Serializable { - /** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */ - static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE); static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; int[] storage; @@ -45,21 +43,13 @@ public IntRingBuffer(int capacity) { * @param growable whether to allow growth when the buffer is full. */ public IntRingBuffer(int capacity, boolean growable) { - Assert.leq(capacity, "IntRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(capacity, "IntRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; // use next larger power of 2 for our storage - final int newCapacity; - if (capacity < 2) { - // sensibly handle the size=0 and size=1 cases - newCapacity = 1; - } else { - newCapacity = Integer.highestOneBit(capacity - 1) << 1; - } - // reset the data structure members - storage = new int[newCapacity]; + storage = new int[MathUtil.roundUpPowerOf2(capacity)]; mask = storage.length - 1; tail = head = 0; } @@ -73,9 +63,9 @@ protected void grow(int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible - Assert.leq(newCapacity, "IntRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(newCapacity, "IntRingBuffer capacity", MathUtil.MAX_POWER_OF_2); - final int[] newStorage = new int[Integer.highestOneBit((int) newCapacity - 1) << 1]; + final int[] newStorage = new int[MathUtil.roundUpPowerOf2((int) newCapacity)]; // move the current data to the new buffer copyRingBufferToArray(newStorage); diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/LongRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/LongRingBuffer.java index 49a3203860a..de5f8df9c90 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/LongRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/LongRingBuffer.java @@ -7,7 +7,7 @@ // @formatter:off package io.deephaven.base.ringbuffer; -import io.deephaven.base.ArrayUtil; +import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; import java.io.Serializable; @@ -20,8 +20,6 @@ * determination of storage indices through a mask operation. */ public class LongRingBuffer implements Serializable { - /** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */ - static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE); static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; long[] storage; @@ -45,21 +43,13 @@ public LongRingBuffer(int capacity) { * @param growable whether to allow growth when the buffer is full. */ public LongRingBuffer(int capacity, boolean growable) { - Assert.leq(capacity, "LongRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(capacity, "LongRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; // use next larger power of 2 for our storage - final int newCapacity; - if (capacity < 2) { - // sensibly handle the size=0 and size=1 cases - newCapacity = 1; - } else { - newCapacity = Integer.highestOneBit(capacity - 1) << 1; - } - // reset the data structure members - storage = new long[newCapacity]; + storage = new long[MathUtil.roundUpPowerOf2(capacity)]; mask = storage.length - 1; tail = head = 0; } @@ -73,9 +63,9 @@ protected void grow(int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible - Assert.leq(newCapacity, "LongRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(newCapacity, "LongRingBuffer capacity", MathUtil.MAX_POWER_OF_2); - final long[] newStorage = new long[Integer.highestOneBit((int) newCapacity - 1) << 1]; + final long[] newStorage = new long[MathUtil.roundUpPowerOf2((int) newCapacity)]; // move the current data to the new buffer copyRingBufferToArray(newStorage); diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/ObjectRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/ObjectRingBuffer.java index e65291e23a6..ad93ddd1f43 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/ObjectRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/ObjectRingBuffer.java @@ -9,7 +9,7 @@ import java.util.Arrays; -import io.deephaven.base.ArrayUtil; +import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; import java.io.Serializable; @@ -22,8 +22,6 @@ * determination of storage indices through a mask operation. */ public class ObjectRingBuffer implements Serializable { - /** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */ - static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE); static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; T[] storage; @@ -47,21 +45,13 @@ public ObjectRingBuffer(int capacity) { * @param growable whether to allow growth when the buffer is full. */ public ObjectRingBuffer(int capacity, boolean growable) { - Assert.leq(capacity, "ObjectRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(capacity, "ObjectRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; // use next larger power of 2 for our storage - final int newCapacity; - if (capacity < 2) { - // sensibly handle the size=0 and size=1 cases - newCapacity = 1; - } else { - newCapacity = Integer.highestOneBit(capacity - 1) << 1; - } - // reset the data structure members - storage = (T[]) new Object[newCapacity]; + storage = (T[]) new Object[MathUtil.roundUpPowerOf2(capacity)]; mask = storage.length - 1; tail = head = 0; } @@ -75,9 +65,9 @@ protected void grow(int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible - Assert.leq(newCapacity, "ObjectRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(newCapacity, "ObjectRingBuffer capacity", MathUtil.MAX_POWER_OF_2); - final T[] newStorage = (T[]) new Object[Integer.highestOneBit((int) newCapacity - 1) << 1]; + final T[] newStorage = (T[]) new Object[MathUtil.roundUpPowerOf2((int) newCapacity)]; // move the current data to the new buffer copyRingBufferToArray(newStorage); diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/ShortRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/ShortRingBuffer.java index d074fab3431..89619fb839c 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/ShortRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/ShortRingBuffer.java @@ -7,7 +7,7 @@ // @formatter:off package io.deephaven.base.ringbuffer; -import io.deephaven.base.ArrayUtil; +import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; import java.io.Serializable; @@ -20,8 +20,6 @@ * determination of storage indices through a mask operation. */ public class ShortRingBuffer implements Serializable { - /** Maximum capacity is the highest power of two that can be allocated (i.e. <= than ArrayUtil.MAX_ARRAY_SIZE). */ - static final int RING_BUFFER_MAX_CAPACITY = Integer.highestOneBit(ArrayUtil.MAX_ARRAY_SIZE); static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; short[] storage; @@ -45,21 +43,13 @@ public ShortRingBuffer(int capacity) { * @param growable whether to allow growth when the buffer is full. */ public ShortRingBuffer(int capacity, boolean growable) { - Assert.leq(capacity, "ShortRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(capacity, "ShortRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; // use next larger power of 2 for our storage - final int newCapacity; - if (capacity < 2) { - // sensibly handle the size=0 and size=1 cases - newCapacity = 1; - } else { - newCapacity = Integer.highestOneBit(capacity - 1) << 1; - } - // reset the data structure members - storage = new short[newCapacity]; + storage = new short[MathUtil.roundUpPowerOf2(capacity)]; mask = storage.length - 1; tail = head = 0; } @@ -73,9 +63,9 @@ protected void grow(int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible - Assert.leq(newCapacity, "ShortRingBuffer capacity", RING_BUFFER_MAX_CAPACITY); + Assert.leq(newCapacity, "ShortRingBuffer capacity", MathUtil.MAX_POWER_OF_2); - final short[] newStorage = new short[Integer.highestOneBit((int) newCapacity - 1) << 1]; + final short[] newStorage = new short[MathUtil.roundUpPowerOf2((int) newCapacity)]; // move the current data to the new buffer copyRingBufferToArray(newStorage); diff --git a/Base/src/test/java/io/deephaven/base/MathUtilTest.java b/Base/src/test/java/io/deephaven/base/MathUtilTest.java index 2f9833d96a1..b29cc79d3ca 100644 --- a/Base/src/test/java/io/deephaven/base/MathUtilTest.java +++ b/Base/src/test/java/io/deephaven/base/MathUtilTest.java @@ -24,4 +24,43 @@ public void check(int a, int b, int expect) { assertEquals(expect, MathUtil.gcd(-a, -b)); assertEquals(expect, MathUtil.gcd(-b, -a)); } + + public void testRoundUpPowerOf2() { + pow2(0, 1); + pow2(1, 1); + pow2(2, 2); + for (int i = 2; i < 31; ++i) { + final int pow2 = 1 << i; + pow2(pow2, pow2); + pow2(pow2 - 1, pow2); + if (i < 30) { + pow2(pow2 + 1, pow2 * 2); + } + } + } + + public void testRoundUpArraySize() { + arraySize(0, 1); + arraySize(1, 1); + arraySize(2, 2); + for (int i = 2; i < 31; ++i) { + final int pow2 = 1 << i; + arraySize(pow2, pow2); + arraySize(pow2 - 1, pow2); + if (i < 30) { + arraySize(pow2 + 1, pow2 * 2); + } else { + arraySize(pow2 + 1, ArrayUtil.MAX_ARRAY_SIZE); + } + } + arraySize(Integer.MAX_VALUE, ArrayUtil.MAX_ARRAY_SIZE); + } + + public static void pow2(int newSize, int expectedSize) { + assertEquals(MathUtil.roundUpPowerOf2(newSize), expectedSize); + } + + public static void arraySize(int newSize, int expectedSize) { + assertEquals(MathUtil.roundUpArraySize(newSize), expectedSize); + } } diff --git a/buildSrc/src/main/groovy/Classpaths.groovy b/buildSrc/src/main/groovy/Classpaths.groovy index 3624a77f408..797458664a1 100644 --- a/buildSrc/src/main/groovy/Classpaths.groovy +++ b/buildSrc/src/main/groovy/Classpaths.groovy @@ -109,7 +109,7 @@ class Classpaths { static final String JACKSON_GROUP = 'com.fasterxml.jackson' static final String JACKSON_NAME = 'jackson-bom' - static final String JACKSON_VERSION = '2.14.1' + static final String JACKSON_VERSION = '2.17.0' static final String SSLCONTEXT_GROUP = 'io.github.hakky54' static final String SSLCONTEXT_VERSION = '8.1.1' diff --git a/engine/chunk/src/main/java/io/deephaven/chunk/util/hashing/ObjectChunkDeepEquals.java b/engine/chunk/src/main/java/io/deephaven/chunk/util/hashing/ObjectChunkDeepEquals.java new file mode 100644 index 00000000000..985aca636f1 --- /dev/null +++ b/engine/chunk/src/main/java/io/deephaven/chunk/util/hashing/ObjectChunkDeepEquals.java @@ -0,0 +1,264 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit CharChunkEquals and run "./gradlew replicateHashing" to regenerate +// +// @formatter:off +package io.deephaven.chunk.util.hashing; + +import java.util.Objects; + +import io.deephaven.chunk.*; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.attributes.ChunkPositions; + +// region name +public class ObjectChunkDeepEquals implements ChunkEquals { + public static ObjectChunkDeepEquals INSTANCE = new ObjectChunkDeepEquals(); + // endregion name + + public static boolean equalReduce(ObjectChunk lhs, ObjectChunk rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + for (int ii = 0; ii < lhs.size(); ++ii) { + if (!eq(lhs.get(ii), rhs.get(ii))) { + return false; + } + } + return true; + } + + public static int firstDifference(ObjectChunk lhs, ObjectChunk rhs) { + int ii = 0; + for (ii = 0; ii < lhs.size() && ii < rhs.size(); ++ii) { + if (!eq(lhs.get(ii), rhs.get(ii))) { + return ii; + } + } + return ii; + } + + private static void equal(ObjectChunk lhs, ObjectChunk rhs, + WritableBooleanChunk destination) { + for (int ii = 0; ii < lhs.size(); ++ii) { + destination.set(ii, eq(lhs.get(ii), rhs.get(ii))); + } + destination.setSize(lhs.size()); + } + + private static void equalNext(ObjectChunk chunk, WritableBooleanChunk destination) { + for (int ii = 0; ii < chunk.size() - 1; ++ii) { + destination.set(ii, eq(chunk.get(ii), chunk.get(ii + 1))); + } + destination.setSize(chunk.size() - 1); + } + + private static void equal(ObjectChunk lhs, Object rhs, WritableBooleanChunk destination) { + for (int ii = 0; ii < lhs.size(); ++ii) { + destination.set(ii, eq(lhs.get(ii), rhs)); + } + destination.setSize(lhs.size()); + } + + public static void notEqual(ObjectChunk lhs, ObjectChunk rhs, + WritableBooleanChunk destination) { + for (int ii = 0; ii < lhs.size(); ++ii) { + destination.set(ii, neq(lhs.get(ii), rhs.get(ii))); + } + destination.setSize(lhs.size()); + } + + public static void notEqual(ObjectChunk lhs, Object rhs, WritableBooleanChunk destination) { + for (int ii = 0; ii < lhs.size(); ++ii) { + destination.set(ii, neq(lhs.get(ii), rhs)); + } + destination.setSize(lhs.size()); + } + + private static void andEqual(ObjectChunk lhs, ObjectChunk rhs, + WritableBooleanChunk destination) { + for (int ii = 0; ii < lhs.size(); ++ii) { + destination.set(ii, destination.get(ii) && eq(lhs.get(ii), rhs.get(ii))); + } + destination.setSize(lhs.size()); + } + + private static void andNotEqual(ObjectChunk lhs, ObjectChunk rhs, + WritableBooleanChunk destination) { + for (int ii = 0; ii < lhs.size(); ++ii) { + destination.set(ii, destination.get(ii) && neq(lhs.get(ii), rhs.get(ii))); + } + destination.setSize(lhs.size()); + } + + private static void andEqualNext(ObjectChunk chunk, WritableBooleanChunk destination) { + for (int ii = 0; ii < chunk.size() - 1; ++ii) { + destination.set(ii, destination.get(ii) && eq(chunk.get(ii), chunk.get(ii + 1))); + } + destination.setSize(chunk.size() - 1); + } + + private static void equalPairs(IntChunk chunkPositionsToCheckForEquality, + ObjectChunk valuesChunk, WritableBooleanChunk destinations) { + final int pairCount = chunkPositionsToCheckForEquality.size() / 2; + for (int ii = 0; ii < pairCount; ++ii) { + final int firstPosition = chunkPositionsToCheckForEquality.get(ii * 2); + final int secondPosition = chunkPositionsToCheckForEquality.get(ii * 2 + 1); + final boolean equals = eq(valuesChunk.get(firstPosition), valuesChunk.get(secondPosition)); + destinations.set(ii, equals); + } + destinations.setSize(pairCount); + } + + private static void andEqualPairs(IntChunk chunkPositionsToCheckForEquality, + ObjectChunk valuesChunk, WritableBooleanChunk destinations) { + final int pairCount = chunkPositionsToCheckForEquality.size() / 2; + for (int ii = 0; ii < pairCount; ++ii) { + if (destinations.get(ii)) { + final int firstPosition = chunkPositionsToCheckForEquality.get(ii * 2); + final int secondPosition = chunkPositionsToCheckForEquality.get(ii * 2 + 1); + final boolean equals = eq(valuesChunk.get(firstPosition), valuesChunk.get(secondPosition)); + destinations.set(ii, equals); + } + } + } + + private static void equalPermuted(IntChunk lhsPositions, IntChunk rhsPositions, + ObjectChunk lhs, ObjectChunk rhs, WritableBooleanChunk destinations) { + for (int ii = 0; ii < lhsPositions.size(); ++ii) { + final int lhsPosition = lhsPositions.get(ii); + final int rhsPosition = rhsPositions.get(ii); + final boolean equals = eq(lhs.get(lhsPosition), rhs.get(rhsPosition)); + destinations.set(ii, equals); + } + destinations.setSize(lhsPositions.size()); + } + + private static void andEqualPermuted(IntChunk lhsPositions, IntChunk rhsPositions, + ObjectChunk lhs, ObjectChunk rhs, WritableBooleanChunk destinations) { + for (int ii = 0; ii < lhsPositions.size(); ++ii) { + if (destinations.get(ii)) { + final int lhsPosition = lhsPositions.get(ii); + final int rhsPosition = rhsPositions.get(ii); + final boolean equals = eq(lhs.get(lhsPosition), rhs.get(rhsPosition)); + destinations.set(ii, equals); + } + } + destinations.setSize(lhsPositions.size()); + } + + private static void equalLhsPermuted(IntChunk lhsPositions, ObjectChunk lhs, + ObjectChunk rhs, WritableBooleanChunk destinations) { + for (int ii = 0; ii < lhsPositions.size(); ++ii) { + final int lhsPosition = lhsPositions.get(ii); + final boolean equals = eq(lhs.get(lhsPosition), rhs.get(ii)); + destinations.set(ii, equals); + } + destinations.setSize(lhsPositions.size()); + } + + private static void andEqualLhsPermuted(IntChunk lhsPositions, ObjectChunk lhs, + ObjectChunk rhs, WritableBooleanChunk destinations) { + for (int ii = 0; ii < lhsPositions.size(); ++ii) { + if (destinations.get(ii)) { + final int lhsPosition = lhsPositions.get(ii); + final boolean equals = eq(lhs.get(lhsPosition), rhs.get(ii)); + destinations.set(ii, equals); + } + } + destinations.setSize(lhsPositions.size()); + } + + @Override + public boolean equalReduce(Chunk lhs, Chunk rhs) { + return equalReduce(lhs.asObjectChunk(), rhs.asObjectChunk()); + } + + @Override + public void equal(Chunk lhs, Chunk rhs, WritableBooleanChunk destination) { + equal(lhs.asObjectChunk(), rhs.asObjectChunk(), destination); + } + + public static void equal(Chunk lhs, Object rhs, WritableBooleanChunk destination) { + equal(lhs.asObjectChunk(), rhs, destination); + } + + @Override + public void equalNext(Chunk chunk, WritableBooleanChunk destination) { + equalNext(chunk.asObjectChunk(), destination); + } + + @Override + public void andEqual(Chunk lhs, Chunk rhs, WritableBooleanChunk destination) { + andEqual(lhs.asObjectChunk(), rhs.asObjectChunk(), destination); + } + + @Override + public void andEqualNext(Chunk chunk, WritableBooleanChunk destination) { + andEqualNext(chunk.asObjectChunk(), destination); + } + + @Override + public void equalPermuted(IntChunk lhsPositions, IntChunk rhsPositions, + Chunk lhs, Chunk rhs, WritableBooleanChunk destination) { + equalPermuted(lhsPositions, rhsPositions, lhs.asObjectChunk(), rhs.asObjectChunk(), destination); + } + + @Override + public void equalLhsPermuted(IntChunk lhsPositions, Chunk lhs, + Chunk rhs, WritableBooleanChunk destination) { + equalLhsPermuted(lhsPositions, lhs.asObjectChunk(), rhs.asObjectChunk(), destination); + } + + @Override + public void andEqualPermuted(IntChunk lhsPositions, IntChunk rhsPositions, + Chunk lhs, Chunk rhs, WritableBooleanChunk destination) { + andEqualPermuted(lhsPositions, rhsPositions, lhs.asObjectChunk(), rhs.asObjectChunk(), destination); + } + + @Override + public void andEqualLhsPermuted(IntChunk lhsPositions, Chunk lhs, + Chunk rhs, WritableBooleanChunk destination) { + andEqualLhsPermuted(lhsPositions, lhs.asObjectChunk(), rhs.asObjectChunk(), destination); + } + + @Override + public void notEqual(Chunk lhs, Chunk rhs, WritableBooleanChunk destination) { + notEqual(lhs.asObjectChunk(), rhs.asObjectChunk(), destination); + } + + public static void notEqual(Chunk lhs, Object rhs, WritableBooleanChunk destination) { + notEqual(lhs.asObjectChunk(), rhs, destination); + } + + @Override + public void andNotEqual(Chunk lhs, Chunk rhs, WritableBooleanChunk destination) { + andNotEqual(lhs.asObjectChunk(), rhs.asObjectChunk(), destination); + } + + @Override + public void equalPairs(IntChunk chunkPositionsToCheckForEquality, Chunk valuesChunk, + WritableBooleanChunk destinations) { + equalPairs(chunkPositionsToCheckForEquality, valuesChunk.asObjectChunk(), destinations); + } + + @Override + public void andEqualPairs(IntChunk chunkPositionsToCheckForEquality, + Chunk valuesChunk, WritableBooleanChunk destinations) { + andEqualPairs(chunkPositionsToCheckForEquality, valuesChunk.asObjectChunk(), destinations); + } + + // region eq + static private boolean eq(Object lhs, Object rhs) { + return Objects.deepEquals(lhs, rhs); + } + // endregion eq + + // region neq + static private boolean neq(Object lhs, Object rhs) { + return !eq(lhs, rhs); + } + // endregion neq +} diff --git a/engine/processor/build.gradle b/engine/processor/build.gradle index 6e7740b5736..e83b218b6f7 100644 --- a/engine/processor/build.gradle +++ b/engine/processor/build.gradle @@ -8,6 +8,8 @@ dependencies { api project(':qst-type') api project(':engine-chunk') + Classpaths.inheritImmutables(project) + Classpaths.inheritJUnitPlatform(project) Classpaths.inheritAssertJ(project) testImplementation 'org.junit.jupiter:junit-jupiter' diff --git a/engine/processor/src/main/java/io/deephaven/processor/NamedObjectProcessor.java b/engine/processor/src/main/java/io/deephaven/processor/NamedObjectProcessor.java new file mode 100644 index 00000000000..96703818349 --- /dev/null +++ b/engine/processor/src/main/java/io/deephaven/processor/NamedObjectProcessor.java @@ -0,0 +1,86 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.processor; + +import io.deephaven.annotations.BuildableStyle; +import io.deephaven.qst.type.Type; +import org.immutables.value.Value.Check; +import org.immutables.value.Value.Immutable; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +@Immutable +@BuildableStyle +public abstract class NamedObjectProcessor { + + public static Builder builder() { + return ImmutableNamedObjectProcessor.builder(); + } + + public static NamedObjectProcessor of(ObjectProcessor processor, String... names) { + return NamedObjectProcessor.builder().processor(processor).addNames(names).build(); + } + + public static NamedObjectProcessor of(ObjectProcessor processor, Iterable names) { + return NamedObjectProcessor.builder().processor(processor).addAllNames(names).build(); + } + + /** + * The name for each output of {@link #processor()}. + */ + public abstract List names(); + + /** + * The object processor. + */ + public abstract ObjectProcessor processor(); + + public interface Builder { + Builder processor(ObjectProcessor processor); + + Builder addNames(String element); + + Builder addNames(String... elements); + + Builder addAllNames(Iterable elements); + + NamedObjectProcessor build(); + } + + public interface Provider extends ObjectProcessor.Provider { + + /** + * The name for each output of the processors. Equivalent to the named processors' + * {@link NamedObjectProcessor#names()}. + * + * @return the names + */ + List names(); + + /** + * Creates a named object processor that can process the {@code inputType}. This will successfully create a + * named processor when {@code inputType} is one of, or extends from one of, {@link #inputTypes()}. Otherwise, + * an {@link IllegalArgumentException} will be thrown. Equivalent to + * {@code NamedObjectProcessor.of(processor(inputType), names())}. + * + * @param inputType the input type + * @return the object processor + * @param the input type + */ + default NamedObjectProcessor named(Type inputType) { + return NamedObjectProcessor.of(processor(inputType), names()); + } + } + + @Check + final void checkSizes() { + if (names().size() != processor().outputSize()) { + throw new IllegalArgumentException( + String.format("Unmatched sizes; names().size()=%d, processor().outputSize()=%d", + names().size(), processor().outputSize())); + } + } +} diff --git a/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessor.java b/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessor.java index fcb166ec5f0..70e42e14f05 100644 --- a/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessor.java +++ b/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessor.java @@ -19,8 +19,8 @@ import io.deephaven.qst.type.ShortType; import io.deephaven.qst.type.Type; -import java.time.Instant; import java.util.List; +import java.util.Set; /** * An interface for processing data from one or more input objects into output chunks on a 1-to-1 input record to output @@ -141,6 +141,15 @@ static ChunkType chunkType(Type type) { return ObjectProcessorTypes.of(type); } + /** + * The number of outputs. Equivalent to {@code outputTypes().size()}. + * + * @return the number of outputs + */ + default int outputSize() { + return outputTypes().size(); + } + /** * The logical output types {@code this} instance processes. The size and types correspond to the expected size and * {@link io.deephaven.chunk.ChunkType chunk types} for {@link #processAll(ObjectChunk, List)} as specified by @@ -168,4 +177,44 @@ static ChunkType chunkType(Type type) { * at least {@code in.size()} */ void processAll(ObjectChunk in, List> out); + + /** + * An abstraction over {@link ObjectProcessor} that provides the same logical object processor for different input + * types. + */ + interface Provider { + + /** + * The supported input types for {@link #processor(Type)}. + * + * @return the supported input types + */ + Set> inputTypes(); + + /** + * The output types for the processors. Equivalent to the processors' {@link ObjectProcessor#outputTypes()}. + * + * @return the output types + */ + List> outputTypes(); + + /** + * The number of output types for the processors. Equivalent to the processors' + * {@link ObjectProcessor#outputSize()}. + * + * @return the number of output types + */ + int outputSize(); + + /** + * Creates an object processor that can process the {@code inputType}. This will successfully create a processor + * when {@code inputType} is one of, or extends from one of, {@link #inputTypes()}. Otherwise, an + * {@link IllegalArgumentException} will be thrown. + * + * @param inputType the input type + * @return the object processor + * @param the input type + */ + ObjectProcessor processor(Type inputType); + } } diff --git a/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessorRowLimited.java b/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessorRowLimited.java index 7dfd03f1249..53d2192e517 100644 --- a/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessorRowLimited.java +++ b/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessorRowLimited.java @@ -48,6 +48,11 @@ int rowLimit() { return rowLimit; } + @Override + public int outputSize() { + return delegate.outputSize(); + } + @Override public List> outputTypes() { return delegate.outputTypes(); diff --git a/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessorStrict.java b/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessorStrict.java index fe1f69277f4..30c84f38aea 100644 --- a/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessorStrict.java +++ b/engine/processor/src/main/java/io/deephaven/processor/ObjectProcessorStrict.java @@ -27,6 +27,16 @@ static ObjectProcessor create(ObjectProcessor delegate) { ObjectProcessorStrict(ObjectProcessor delegate) { this.delegate = Objects.requireNonNull(delegate); this.outputTypes = List.copyOf(delegate.outputTypes()); + if (delegate.outputSize() != outputTypes.size()) { + throw new IllegalArgumentException( + String.format("Inconsistent size. delegate.outputSize()=%d, delegate.outputTypes().size()=%d", + delegate.outputSize(), outputTypes.size())); + } + } + + @Override + public int outputSize() { + return delegate.outputSize(); } @Override @@ -40,12 +50,13 @@ public List> outputTypes() { @Override public void processAll(ObjectChunk in, List> out) { - final int numColumns = delegate.outputTypes().size(); + final int numColumns = delegate.outputSize(); if (numColumns != out.size()) { throw new IllegalArgumentException(String.format( - "Improper number of out chunks. Expected delegate.outputTypes().size() == out.size(). delegate.outputTypes().size()=%d, out.size()=%d", + "Improper number of out chunks. Expected delegate.outputSize() == out.size(). delegate.outputSize()=%d, out.size()=%d", numColumns, out.size())); } + final List> delegateOutputTypes = delegate.outputTypes(); final int[] originalSizes = new int[numColumns]; for (int chunkIx = 0; chunkIx < numColumns; ++chunkIx) { final WritableChunk chunk = out.get(chunkIx); @@ -54,7 +65,7 @@ public void processAll(ObjectChunk in, List> ou "out chunk does not have enough remaining capacity. chunkIx=%d, in.size()=%d, chunk.size()=%d, chunk.capacity()=%d", chunkIx, in.size(), chunk.size(), chunk.capacity())); } - final Type type = delegate.outputTypes().get(chunkIx); + final Type type = delegateOutputTypes.get(chunkIx); final ChunkType expectedChunkType = ObjectProcessor.chunkType(type); final ChunkType actualChunkType = chunk.getChunkType(); if (expectedChunkType != actualChunkType) { diff --git a/engine/processor/src/test/java/io/deephaven/processor/ObjectProcessorStrictTest.java b/engine/processor/src/test/java/io/deephaven/processor/ObjectProcessorStrictTest.java index eb79ed25583..562aeb368dc 100644 --- a/engine/processor/src/test/java/io/deephaven/processor/ObjectProcessorStrictTest.java +++ b/engine/processor/src/test/java/io/deephaven/processor/ObjectProcessorStrictTest.java @@ -103,7 +103,7 @@ public void testIncorrectChunkType() { } @Test - public void testNotEnoughOutputSize() { + public void testNotEnoughOutputOutputSize() { ObjectProcessor delegate = ObjectProcessor.noop(List.of(Type.intType()), false); ObjectProcessor strict = ObjectProcessor.strict(delegate); try ( @@ -172,6 +172,11 @@ public void testBadDelegateOutputTypes() { ObjectProcessor strict = ObjectProcessor.strict(new ObjectProcessor<>() { private final List> outputTypes = new ArrayList<>(List.of(Type.intType())); + @Override + public int outputSize() { + return 1; + } + @Override public List> outputTypes() { try { @@ -220,4 +225,30 @@ public void processAll(ObjectChunk in, List> out) { } } } + + @Test + public void testBadDelegateOutputSize() { + try { + ObjectProcessor.strict(new ObjectProcessor<>() { + @Override + public int outputSize() { + return 2; + } + + @Override + public List> outputTypes() { + return List.of(Type.intType()); + } + + @Override + public void processAll(ObjectChunk in, List> out) { + // ignore + } + }); + failBecauseExceptionWasNotThrown(IllegalAccessException.class); + } catch (IllegalArgumentException e) { + assertThat(e).hasMessageContaining( + "Inconsistent size. delegate.outputSize()=2, delegate.outputTypes().size()=1"); + } + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ring/RingTableTools.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ring/RingTableTools.java index 80bd1be94d7..45495357d15 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ring/RingTableTools.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ring/RingTableTools.java @@ -3,6 +3,9 @@ // package io.deephaven.engine.table.impl.sources.ring; +import io.deephaven.base.ArrayUtil; +import io.deephaven.base.MathUtil; +import io.deephaven.base.verify.Require; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableUpdate; @@ -43,6 +46,7 @@ public static Table of(Table parent, int capacity) { * @return the ring table */ public static Table of(Table parent, int capacity, boolean initialize) { + Require.leq(capacity, "capacity", ArrayUtil.MAX_ARRAY_SIZE); return QueryPerformanceRecorder.withNugget("RingTableTools.of", () -> { final BaseTable baseTable = (BaseTable) parent.coalesce(); final OperationSnapshotControl snapshotControl = @@ -56,7 +60,7 @@ public static Table of(Table parent, int capacity, boolean initialize) { * re-indexed, with an additional {@link Table#tail(long)} to restructure for {@code capacity}. * *

- * Logically equivalent to {@code of(parent, Integer.highestOneBit(capacity - 1) << 1, initialize).tail(capacity)}. + * Logically equivalent to {@code of(parent, MathUtil.roundUpPowerOf2(capacity), initialize).tail(capacity)}. * *

* This setup may be useful when consumers need to maximize random access fill speed from a ring table. @@ -66,11 +70,12 @@ public static Table of(Table parent, int capacity, boolean initialize) { * @param initialize if the resulting table should source initial data from the snapshot of {@code parent} * @return the ring table * @see #of(Table, int, boolean) + * @see MathUtil#roundUpPowerOf2(int) */ public static Table of2(Table parent, int capacity, boolean initialize) { + Require.leq(capacity, "capacity", MathUtil.MAX_POWER_OF_2); return QueryPerformanceRecorder.withNugget("RingTableTools.of2", () -> { - // todo: there is probably a better way to do this - final int capacityPowerOf2 = capacity == 1 ? 1 : Integer.highestOneBit(capacity - 1) << 1; + final int capacityPowerOf2 = MathUtil.roundUpPowerOf2(capacity); final BaseTable baseTable = (BaseTable) parent.coalesce(); final OperationSnapshotControl snapshotControl = baseTable.createSnapshotControlIfRefreshing(OperationSnapshotControl::new); diff --git a/extensions/bson-jackson/build.gradle b/extensions/bson-jackson/build.gradle new file mode 100644 index 00000000000..660ef1b207f --- /dev/null +++ b/extensions/bson-jackson/build.gradle @@ -0,0 +1,23 @@ +plugins { + id 'java-library' + id 'io.deephaven.project.register' +} + +dependencies { + api project(':extensions-json-jackson') + api project(':engine-processor') + api 'de.undercouch:bson4jackson:2.15.1' + + Classpaths.inheritImmutables(project) + compileOnly 'com.google.code.findbugs:jsr305:3.0.2' + + Classpaths.inheritJacksonPlatform(project, 'testImplementation') + Classpaths.inheritJUnitPlatform(project) + Classpaths.inheritAssertJ(project) + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'com.fasterxml.jackson.core:jackson-databind' +} + +test { + useJUnitPlatform() +} diff --git a/extensions/bson-jackson/gradle.properties b/extensions/bson-jackson/gradle.properties new file mode 100644 index 00000000000..c186bbfdde1 --- /dev/null +++ b/extensions/bson-jackson/gradle.properties @@ -0,0 +1 @@ +io.deephaven.project.ProjectType=JAVA_PUBLIC diff --git a/extensions/bson-jackson/src/main/java/io/deephaven/bson/jackson/JacksonBsonConfiguration.java b/extensions/bson-jackson/src/main/java/io/deephaven/bson/jackson/JacksonBsonConfiguration.java new file mode 100644 index 00000000000..663a7e02671 --- /dev/null +++ b/extensions/bson-jackson/src/main/java/io/deephaven/bson/jackson/JacksonBsonConfiguration.java @@ -0,0 +1,30 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.bson.jackson; + +import com.fasterxml.jackson.core.ObjectCodec; +import de.undercouch.bson4jackson.BsonFactory; + +import java.lang.reflect.InvocationTargetException; + +final class JacksonBsonConfiguration { + private static final BsonFactory DEFAULT_FACTORY; + + static { + // We'll attach an ObjectMapper if it's on the classpath, this allows parsing of AnyOptions + ObjectCodec objectCodec = null; + try { + final Class clazz = Class.forName("com.fasterxml.jackson.databind.ObjectMapper"); + objectCodec = (ObjectCodec) clazz.getDeclaredConstructor().newInstance(); + } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException + | InvocationTargetException e) { + // ignore + } + DEFAULT_FACTORY = new BsonFactory(objectCodec); + } + + static BsonFactory defaultFactory() { + return DEFAULT_FACTORY; + } +} diff --git a/extensions/bson-jackson/src/main/java/io/deephaven/bson/jackson/JacksonBsonProvider.java b/extensions/bson-jackson/src/main/java/io/deephaven/bson/jackson/JacksonBsonProvider.java new file mode 100644 index 00000000000..567d3a94bd4 --- /dev/null +++ b/extensions/bson-jackson/src/main/java/io/deephaven/bson/jackson/JacksonBsonProvider.java @@ -0,0 +1,33 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.bson.jackson; + +import de.undercouch.bson4jackson.BsonFactory; +import io.deephaven.json.Value; +import io.deephaven.json.jackson.JacksonProvider; + +public final class JacksonBsonProvider { + + /** + * Creates a jackson BSON provider using a default factory. + * + * @param options the object options + * @return the jackson BSON provider + * @see #of(Value, BsonFactory) + */ + public static JacksonProvider of(Value options) { + return of(options, JacksonBsonConfiguration.defaultFactory()); + } + + /** + * Creates a jackson BSON provider using the provided {@code factory}. + * + * @param options the object options + * @param factory the jackson BSON factory + * @return the jackson BSON provider + */ + public static JacksonProvider of(Value options, BsonFactory factory) { + return JacksonProvider.of(options, factory); + } +} diff --git a/extensions/bson-jackson/src/test/java/io/deephaven/bson/jackson/BsonTest.java b/extensions/bson-jackson/src/test/java/io/deephaven/bson/jackson/BsonTest.java new file mode 100644 index 00000000000..1ac72354c1a --- /dev/null +++ b/extensions/bson-jackson/src/test/java/io/deephaven/bson/jackson/BsonTest.java @@ -0,0 +1,39 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.bson.jackson; + +import com.fasterxml.jackson.databind.ObjectMapper; +import de.undercouch.bson4jackson.BsonFactory; +import io.deephaven.chunk.IntChunk; +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.IntValue; +import io.deephaven.json.ObjectValue; +import io.deephaven.json.StringValue; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static io.deephaven.bson.jackson.TestHelper.parse; + +public class BsonTest { + + private static final ObjectValue OBJECT_NAME_AGE_FIELD = ObjectValue.builder() + .putFields("name", StringValue.standard()) + .putFields("age", IntValue.standard()) + .build(); + + @Test + void bson() throws IOException { + final byte[] bsonExample = new ObjectMapper(new BsonFactory()).writeValueAsBytes(Map.of( + "name", "foo", + "age", 42)); + parse( + JacksonBsonProvider.of(OBJECT_NAME_AGE_FIELD).bytesProcessor(), + List.of(bsonExample), + ObjectChunk.chunkWrap(new String[] {"foo"}), + IntChunk.chunkWrap(new int[] {42})); + } +} diff --git a/extensions/bson-jackson/src/test/java/io/deephaven/bson/jackson/TestHelper.java b/extensions/bson-jackson/src/test/java/io/deephaven/bson/jackson/TestHelper.java new file mode 100644 index 00000000000..70564ee2e5c --- /dev/null +++ b/extensions/bson-jackson/src/test/java/io/deephaven/bson/jackson/TestHelper.java @@ -0,0 +1,73 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.bson.jackson; + +import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.ChunkType; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableObjectChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.util.hashing.ChunkEquals; +import io.deephaven.chunk.util.hashing.ObjectChunkDeepEquals; +import io.deephaven.processor.ObjectProcessor; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHelper { + + public static void parse(ObjectProcessor processor, List rows, Chunk... expectedCols) + throws IOException { + final List> out = processor + .outputTypes() + .stream() + .map(ObjectProcessor::chunkType) + .map(x -> x.makeWritableChunk(rows.size())) + .collect(Collectors.toList()); + try { + assertThat(out.size()).isEqualTo(expectedCols.length); + assertThat(out.stream().map(Chunk::getChunkType).collect(Collectors.toList())) + .isEqualTo(Stream.of(expectedCols).map(Chunk::getChunkType).collect(Collectors.toList())); + for (WritableChunk wc : out) { + wc.setSize(0); + } + try (final WritableObjectChunk in = WritableObjectChunk.makeWritableChunk(rows.size())) { + int i = 0; + for (T input : rows) { + in.set(i, input); + ++i; + } + try { + processor.processAll(in, out); + } catch (UncheckedIOException e) { + throw e.getCause(); + } + } + for (int i = 0; i < expectedCols.length; ++i) { + check(out.get(i), expectedCols[i]); + } + } finally { + for (WritableChunk wc : out) { + wc.close(); + } + } + } + + static void check(Chunk actual, Chunk expected) { + assertThat(actual.getChunkType()).isEqualTo(expected.getChunkType()); + assertThat(actual.size()).isEqualTo(expected.size()); + assertThat(getChunkEquals(actual).equalReduce(actual, expected)).isTrue(); + } + + private static ChunkEquals getChunkEquals(Chunk actual) { + return actual.getChunkType() == ChunkType.Object + ? ObjectChunkDeepEquals.INSTANCE + : ChunkEquals.makeEqual(actual.getChunkType()); + } +} diff --git a/extensions/json-jackson/build.gradle b/extensions/json-jackson/build.gradle new file mode 100644 index 00000000000..bd18f5a88b4 --- /dev/null +++ b/extensions/json-jackson/build.gradle @@ -0,0 +1,34 @@ +plugins { + id 'java-library' + id 'io.deephaven.project.register' +} + +dependencies { + api project(':engine-processor') + + Classpaths.inheritJacksonPlatform(project, 'api') + Classpaths.inheritJacksonPlatform(project, 'testImplementation') + + api 'com.fasterxml.jackson.core:jackson-core' + // https://github.com/FasterXML/jackson-core/issues/1229 + implementation 'ch.randelshofer:fastdoubleparser:1.0.0' + + api project(':extensions-json') + + implementation project(':table-api') // only needs NameValidator, might be worth refactoring? + + implementation project(':engine-query-constants') + implementation project(':engine-time') + Classpaths.inheritImmutables(project) + Classpaths.inheritAutoService(project) + compileOnly 'com.google.code.findbugs:jsr305:3.0.2' + + Classpaths.inheritJUnitPlatform(project) + Classpaths.inheritAssertJ(project) + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'com.fasterxml.jackson.core:jackson-databind' +} + +test { + useJUnitPlatform() +} diff --git a/extensions/json-jackson/gradle.properties b/extensions/json-jackson/gradle.properties new file mode 100644 index 00000000000..c186bbfdde1 --- /dev/null +++ b/extensions/json-jackson/gradle.properties @@ -0,0 +1 @@ +io.deephaven.project.ProjectType=JAVA_PUBLIC diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/AnyMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/AnyMixin.java new file mode 100644 index 00000000000..acaa2cc9fe2 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/AnyMixin.java @@ -0,0 +1,28 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.TreeNode; +import io.deephaven.json.AnyValue; +import io.deephaven.qst.type.Type; + +import java.io.IOException; + +final class AnyMixin extends GenericObjectMixin { + public AnyMixin(AnyValue options, JsonFactory factory) { + super(factory, options, Type.ofCustom(TreeNode.class)); + } + + @Override + public TreeNode parseValue(JsonParser parser) throws IOException { + return parser.readValueAsTree(); + } + + @Override + public TreeNode parseMissing(JsonParser parser) throws IOException { + return parser.getCodec().missingNode(); + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ArrayMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ArrayMixin.java new file mode 100644 index 00000000000..9f7ecd147e8 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ArrayMixin.java @@ -0,0 +1,112 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.json.ArrayValue; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Stream; + +final class ArrayMixin extends Mixin { + + private final Mixin element; + + public ArrayMixin(ArrayValue options, JsonFactory factory) { + super(factory, options); + element = Mixin.of(options.element(), factory); + } + + @Override + public int outputSize() { + return element.outputSize(); + } + + @Override + public Stream> paths() { + return element.paths(); + } + + @Override + public Stream> outputTypesImpl() { + return elementOutputTypes().map(Type::arrayType); + } + + @Override + public ValueProcessor processor(String context) { + return new ArrayMixinProcessor(); + } + + private Stream> elementOutputTypes() { + return element.outputTypesImpl(); + } + + private RepeaterProcessor elementRepeater() { + return element.repeaterProcessor(); + } + + @Override + RepeaterProcessor repeaterProcessor() { + // For example: + // double (element()) + // double[] (processor()) + // double[][] (repeater()) + // return new ArrayOfArrayRepeaterProcessor(allowMissing, allowNull); + return new ValueInnerRepeaterProcessor(new ArrayMixinProcessor()); + } + + private class ArrayMixinProcessor implements ValueProcessor { + + private final RepeaterProcessor elementProcessor; + + ArrayMixinProcessor() { + this.elementProcessor = elementRepeater(); + } + + @Override + public void setContext(List> out) { + elementProcessor.setContext(out); + } + + @Override + public void clearContext() { + elementProcessor.clearContext(); + } + + @Override + public int numColumns() { + return elementProcessor.numColumns(); + } + + @Override + public Stream> columnTypes() { + return elementProcessor.columnTypes(); + } + + @Override + public void processCurrentValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case START_ARRAY: + RepeaterProcessor.processArray(parser, elementProcessor); + return; + case VALUE_NULL: + checkNullAllowed(parser); + elementProcessor.processNullRepeater(parser); + return; + default: + throw unexpectedToken(parser); + } + } + + @Override + public void processMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + elementProcessor.processMissingRepeater(parser); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/BigDecimalMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/BigDecimalMixin.java new file mode 100644 index 00000000000..6d2d1fd3a8c --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/BigDecimalMixin.java @@ -0,0 +1,59 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.json.BigDecimalValue; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.math.BigDecimal; + +final class BigDecimalMixin extends GenericObjectMixin { + + public BigDecimalMixin(BigDecimalValue options, JsonFactory factory) { + super(factory, options, Type.ofCustom(BigDecimal.class)); + } + + @Override + public BigDecimal parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_NUMBER_INT: + case VALUE_NUMBER_FLOAT: + return parseFromNumber(parser); + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + @Override + public BigDecimal parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + private BigDecimal parseFromNumber(JsonParser parser) throws IOException { + checkNumberAllowed(parser); + return Parsing.parseDecimalAsBigDecimal(parser); + } + + private BigDecimal parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + return Parsing.parseStringAsBigDecimal(parser); + } + + private BigDecimal parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(null); + } + + private BigDecimal parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(null); + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/BigIntegerMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/BigIntegerMixin.java new file mode 100644 index 00000000000..f2585108fd2 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/BigIntegerMixin.java @@ -0,0 +1,67 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.json.BigIntegerValue; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.math.BigInteger; + +final class BigIntegerMixin extends GenericObjectMixin { + + public BigIntegerMixin(BigIntegerValue options, JsonFactory factory) { + super(factory, options, Type.ofCustom(BigInteger.class)); + } + + @Override + public BigInteger parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_NUMBER_INT: + return parseFromInt(parser); + case VALUE_NUMBER_FLOAT: + return parseFromDecimal(parser); + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + @Override + public BigInteger parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + private BigInteger parseFromInt(JsonParser parser) throws IOException { + checkNumberIntAllowed(parser); + return Parsing.parseIntAsBigInteger(parser); + } + + private BigInteger parseFromDecimal(JsonParser parser) throws IOException { + checkDecimalAllowed(parser); + return Parsing.parseDecimalAsBigInteger(parser); + } + + private BigInteger parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + return allowDecimal() + ? Parsing.parseDecimalStringAsBigInteger(parser) + : Parsing.parseStringAsBigInteger(parser); + } + + private BigInteger parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(null); + } + + private BigInteger parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(null); + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/BoolMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/BoolMixin.java new file mode 100644 index 00000000000..70a5ba39d17 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/BoolMixin.java @@ -0,0 +1,168 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.chunk.WritableByteChunk; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.json.BoolValue; +import io.deephaven.qst.type.Type; +import io.deephaven.util.BooleanUtils; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Stream; + +final class BoolMixin extends Mixin { + + private final Boolean onNull; + private final Boolean onMissing; + private final byte onNullByte; + private final byte onMissingByte; + + public BoolMixin(BoolValue options, JsonFactory factory) { + super(factory, options); + onNull = options.onNull().orElse(null); + onMissing = options.onMissing().orElse(null); + onNullByte = BooleanUtils.booleanAsByte(onNull); + onMissingByte = BooleanUtils.booleanAsByte(onMissing); + } + + @Override + public int outputSize() { + return 1; + } + + @Override + public Stream> paths() { + return Stream.of(List.of()); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.of(Type.booleanType().boxedType()); + } + + @Override + public ValueProcessor processor(String context) { + return new BoolMixinProcessor(); + } + + private byte parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_TRUE: + return BooleanUtils.TRUE_BOOLEAN_AS_BYTE; + case VALUE_FALSE: + return BooleanUtils.FALSE_BOOLEAN_AS_BYTE; + case VALUE_NULL: + return parseFromNull(parser); + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + } + throw unexpectedToken(parser); + } + + private byte parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new RepeaterGenericImpl<>(new ToBoolean(), null, null, + Type.booleanType().boxedType().arrayType()); + } + + final class ToBoolean implements ToObject { + @Override + public Boolean parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_TRUE: + return Boolean.TRUE; + case VALUE_FALSE: + return Boolean.FALSE; + case VALUE_NULL: + return parseFromNullBoolean(parser); + case VALUE_STRING: + case FIELD_NAME: + return parseFromStringBoolean(parser); + } + throw unexpectedToken(parser); + } + + @Override + public Boolean parseMissing(JsonParser parser) throws IOException { + return parseFromMissingBoolean(parser); + } + } + + private byte parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + if (!allowNull()) { + final byte res = Parsing.parseStringAsByteBool(parser, BooleanUtils.NULL_BOOLEAN_AS_BYTE); + if (res == BooleanUtils.NULL_BOOLEAN_AS_BYTE) { + throw nullNotAllowed(parser); + } + return res; + } + return Parsing.parseStringAsByteBool(parser, onNullByte); + } + + private byte parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return onNullByte; + } + + private byte parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return onMissingByte; + } + + private Boolean parseFromStringBoolean(JsonParser parser) throws IOException { + checkStringAllowed(parser); + if (!allowNull()) { + final Boolean result = Parsing.parseStringAsBoolean(parser, null); + if (result == null) { + throw nullNotAllowed(parser); + } + return result; + } + return Parsing.parseStringAsBoolean(parser, onNull); + } + + private Boolean parseFromNullBoolean(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return onNull; + } + + private Boolean parseFromMissingBoolean(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return onMissing; + } + + private class BoolMixinProcessor extends ValueProcessorMixinBase { + private WritableByteChunk out; + + @Override + public final void setContext(List> out) { + this.out = out.get(0).asWritableByteChunk(); + } + + @Override + public final void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ByteBufferInputStream.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ByteBufferInputStream.java new file mode 100644 index 00000000000..17906ad0f84 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ByteBufferInputStream.java @@ -0,0 +1,54 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Objects; + +final class ByteBufferInputStream extends InputStream { + + public static InputStream of(ByteBuffer buffer) { + if (buffer.hasArray()) { + return new ByteArrayInputStream(buffer.array(), buffer.arrayOffset() + buffer.position(), + buffer.remaining()); + } else { + return new ByteBufferInputStream(buffer.asReadOnlyBuffer()); + } + } + + private final ByteBuffer buffer; + + private ByteBufferInputStream(ByteBuffer buf) { + buffer = Objects.requireNonNull(buf); + } + + @Override + public int available() { + return buffer.remaining(); + } + + @Override + public int read() { + return buffer.hasRemaining() ? buffer.get() & 0xFF : -1; + } + + @Override + public int read(byte[] bytes, int off, int len) { + if (!buffer.hasRemaining()) { + return -1; + } + len = Math.min(len, buffer.remaining()); + buffer.get(bytes, off, len); + return len; + } + + @Override + public long skip(long n) { + n = Math.min(n, buffer.remaining()); + buffer.position(buffer.position() + (int) n); + return n; + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ByteMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ByteMixin.java new file mode 100644 index 00000000000..856491ab8cb --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ByteMixin.java @@ -0,0 +1,151 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.base.MathUtil; +import io.deephaven.chunk.WritableByteChunk; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.sized.SizedByteChunk; +import io.deephaven.json.ByteValue; +import io.deephaven.json.Value; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +final class ByteMixin extends Mixin { + public ByteMixin(ByteValue options, JsonFactory factory) { + super(factory, options); + } + + @Override + public int outputSize() { + return 1; + } + + @Override + public Stream> paths() { + return Stream.of(List.of()); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.of(Type.byteType()); + } + + @Override + public ValueProcessor processor(String context) { + return new ByteMixinProcessor(); + } + + private byte parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_NUMBER_INT: + return parseFromInt(parser); + case VALUE_NUMBER_FLOAT: + return parseFromDecimal(parser); + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + private byte parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new ByteRepeaterImpl(); + } + + final class ByteRepeaterImpl extends RepeaterProcessorBase { + private final SizedByteChunk chunk = new SizedByteChunk<>(0); + + public ByteRepeaterImpl() { + super(null, null, Type.byteType().arrayType()); + } + + @Override + public void processElementImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableByteChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, ByteMixin.this.parseValue(parser)); + chunk.setSize(newSize); + } + + @Override + public void processElementMissingImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableByteChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, ByteMixin.this.parseMissing(parser)); + chunk.setSize(newSize); + } + + @Override + public byte[] doneImpl(JsonParser parser, int length) { + final WritableByteChunk chunk = this.chunk.get(); + return Arrays.copyOfRange(chunk.array(), chunk.arrayOffset(), chunk.arrayOffset() + length); + } + } + + private byte parseFromInt(JsonParser parser) throws IOException { + checkNumberIntAllowed(parser); + return Parsing.parseIntAsByte(parser); + } + + private byte parseFromDecimal(JsonParser parser) throws IOException { + checkDecimalAllowed(parser); + return Parsing.parseDecimalAsByte(parser); + } + + private byte parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + return allowDecimal() + ? Parsing.parseDecimalStringAsByte(parser) + : Parsing.parseStringAsByte(parser); + } + + private byte parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(QueryConstants.NULL_BYTE); + } + + private byte parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(QueryConstants.NULL_BYTE); + } + + private class ByteMixinProcessor extends ValueProcessorMixinBase { + private WritableByteChunk out; + + @Override + public final void setContext(List> out) { + this.out = out.get(0).asWritableByteChunk(); + } + + @Override + public final void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/CharMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/CharMixin.java new file mode 100644 index 00000000000..f14b81583f1 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/CharMixin.java @@ -0,0 +1,134 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.base.MathUtil; +import io.deephaven.chunk.WritableCharChunk; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.sized.SizedCharChunk; +import io.deephaven.json.CharValue; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +final class CharMixin extends Mixin { + public CharMixin(CharValue options, JsonFactory factory) { + super(factory, options); + } + + @Override + public int outputSize() { + return 1; + } + + @Override + public Stream> paths() { + return Stream.of(List.of()); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.of(Type.charType()); + } + + @Override + public ValueProcessor processor(String context) { + return new CharMixinProcessor(); + } + + private char parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + private char parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new CharRepeaterImpl(); + } + + final class CharRepeaterImpl extends RepeaterProcessorBase { + private final SizedCharChunk chunk = new SizedCharChunk<>(0); + + public CharRepeaterImpl() { + super(null, null, Type.charType().arrayType()); + } + + @Override + public void processElementImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableCharChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, CharMixin.this.parseValue(parser)); + chunk.setSize(newSize); + } + + @Override + public void processElementMissingImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableCharChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, CharMixin.this.parseMissing(parser)); + chunk.setSize(newSize); + } + + @Override + public char[] doneImpl(JsonParser parser, int length) { + final WritableCharChunk chunk = this.chunk.get(); + return Arrays.copyOfRange(chunk.array(), chunk.arrayOffset(), chunk.arrayOffset() + length); + } + } + + private char parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + return Parsing.parseStringAsChar(parser); + } + + private char parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(QueryConstants.NULL_CHAR); + } + + private char parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(QueryConstants.NULL_CHAR); + } + + private class CharMixinProcessor extends ValueProcessorMixinBase { + private WritableCharChunk out; + + @Override + public void setContext(List> out) { + this.out = out.get(0).asWritableCharChunk(); + } + + @Override + public void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ContextAware.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ContextAware.java new file mode 100644 index 00000000000..7e28d086bb4 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ContextAware.java @@ -0,0 +1,20 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import io.deephaven.chunk.WritableChunk; +import io.deephaven.qst.type.Type; + +import java.util.List; +import java.util.stream.Stream; + +interface ContextAware { + void setContext(List> out); + + void clearContext(); + + int numColumns(); + + Stream> columnTypes(); +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ContextAwareDelegateBase.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ContextAwareDelegateBase.java new file mode 100644 index 00000000000..bfe6a248727 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ContextAwareDelegateBase.java @@ -0,0 +1,50 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import io.deephaven.chunk.WritableChunk; +import io.deephaven.qst.type.Type; + +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +abstract class ContextAwareDelegateBase implements ContextAware { + + private final Collection delegates; + private final int numColumns; + + public ContextAwareDelegateBase(Collection delegates) { + this.delegates = Objects.requireNonNull(delegates); + this.numColumns = delegates.stream().mapToInt(ContextAware::numColumns).sum(); + } + + @Override + public final void setContext(List> out) { + int ix = 0; + for (ContextAware delegate : delegates) { + final int numColumns = delegate.numColumns(); + delegate.setContext(out.subList(ix, ix + numColumns)); + ix += numColumns; + } + } + + @Override + public final void clearContext() { + for (ContextAware delegate : delegates) { + delegate.clearContext(); + } + } + + @Override + public final int numColumns() { + return numColumns; + } + + @Override + public final Stream> columnTypes() { + return delegates.stream().flatMap(ContextAware::columnTypes); + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/DoubleMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/DoubleMixin.java new file mode 100644 index 00000000000..bd1425a3784 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/DoubleMixin.java @@ -0,0 +1,145 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.base.MathUtil; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableDoubleChunk; +import io.deephaven.chunk.sized.SizedDoubleChunk; +import io.deephaven.json.DoubleValue; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +final class DoubleMixin extends Mixin { + + public DoubleMixin(DoubleValue options, JsonFactory factory) { + super(factory, options); + } + + @Override + public int outputSize() { + return 1; + } + + @Override + public Stream> paths() { + return Stream.of(List.of()); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.of(Type.doubleType()); + } + + @Override + public ValueProcessor processor(String context) { + return new DoubleMixinProcessor(); + } + + private double parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_NUMBER_INT: + case VALUE_NUMBER_FLOAT: + return parseFromNumber(parser); + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + private double parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new DoubleRepeaterImpl(); + } + + final class DoubleRepeaterImpl extends RepeaterProcessorBase { + private final SizedDoubleChunk chunk = new SizedDoubleChunk<>(0); + + public DoubleRepeaterImpl() { + super(null, null, Type.doubleType().arrayType()); + } + + @Override + public void processElementImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableDoubleChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, DoubleMixin.this.parseValue(parser)); + chunk.setSize(newSize); + } + + @Override + public void processElementMissingImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableDoubleChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, DoubleMixin.this.parseMissing(parser)); + chunk.setSize(newSize); + } + + @Override + public double[] doneImpl(JsonParser parser, int length) { + final WritableDoubleChunk chunk = this.chunk.get(); + return Arrays.copyOfRange(chunk.array(), chunk.arrayOffset(), chunk.arrayOffset() + length); + } + } + + private double parseFromNumber(JsonParser parser) throws IOException { + checkNumberAllowed(parser); + // TODO: improve after https://github.com/FasterXML/jackson-core/issues/1229 + return Parsing.parseNumberAsDouble(parser); + } + + private double parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + return Parsing.parseStringAsDouble(parser); + } + + private double parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(QueryConstants.NULL_DOUBLE); + } + + private double parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(QueryConstants.NULL_DOUBLE); + } + + final class DoubleMixinProcessor extends ValueProcessorMixinBase { + + private WritableDoubleChunk out; + + @Override + public void setContext(List> out) { + this.out = out.get(0).asWritableDoubleChunk(); + } + + @Override + public void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/FieldProcessor.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/FieldProcessor.java new file mode 100644 index 00000000000..6a0b75e4a77 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/FieldProcessor.java @@ -0,0 +1,35 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import java.io.IOException; + +import static io.deephaven.json.jackson.Parsing.assertCurrentToken; + +interface FieldProcessor { + + static void processFields(JsonParser parser, FieldProcessor fieldProcess) throws IOException { + while (parser.hasToken(JsonToken.FIELD_NAME)) { + final String fieldName = parser.currentName(); + parser.nextToken(); + fieldProcess.process(fieldName, parser); + parser.nextToken(); + } + assertCurrentToken(parser, JsonToken.END_OBJECT); + } + + static void skipFields(JsonParser parser) throws IOException { + while (parser.hasToken(JsonToken.FIELD_NAME)) { + parser.nextToken(); + parser.skipChildren(); + parser.nextToken(); + } + assertCurrentToken(parser, JsonToken.END_OBJECT); + } + + void process(String fieldName, JsonParser parser) throws IOException; +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/FloatMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/FloatMixin.java new file mode 100644 index 00000000000..18ec8cde73a --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/FloatMixin.java @@ -0,0 +1,144 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.base.MathUtil; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableFloatChunk; +import io.deephaven.chunk.sized.SizedFloatChunk; +import io.deephaven.json.FloatValue; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +final class FloatMixin extends Mixin { + + public FloatMixin(FloatValue options, JsonFactory factory) { + super(factory, options); + } + + @Override + public int outputSize() { + return 1; + } + + @Override + public Stream> paths() { + return Stream.of(List.of()); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.of(Type.floatType()); + } + + @Override + public ValueProcessor processor(String context) { + return new FloatMixinProcessor(); + } + + private float parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_NUMBER_INT: + case VALUE_NUMBER_FLOAT: + return parseFromNumber(parser); + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + private float parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new FloatRepeaterImpl(); + } + + final class FloatRepeaterImpl extends RepeaterProcessorBase { + private final SizedFloatChunk chunk = new SizedFloatChunk<>(0); + + public FloatRepeaterImpl() { + super(null, null, Type.floatType().arrayType()); + } + + @Override + public void processElementImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableFloatChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, FloatMixin.this.parseValue(parser)); + chunk.setSize(newSize); + } + + @Override + public void processElementMissingImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableFloatChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, FloatMixin.this.parseMissing(parser)); + chunk.setSize(newSize); + } + + @Override + public float[] doneImpl(JsonParser parser, int length) { + final WritableFloatChunk chunk = this.chunk.get(); + return Arrays.copyOfRange(chunk.array(), chunk.arrayOffset(), chunk.arrayOffset() + length); + } + } + + private float parseFromNumber(JsonParser parser) throws IOException { + checkNumberAllowed(parser); + return Parsing.parseNumberAsFloat(parser); + } + + private float parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + return Parsing.parseStringAsFloat(parser); + } + + private float parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(QueryConstants.NULL_FLOAT); + } + + private float parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(QueryConstants.NULL_FLOAT); + } + + final class FloatMixinProcessor extends ValueProcessorMixinBase { + + private WritableFloatChunk out; + + @Override + public void setContext(List> out) { + this.out = out.get(0).asWritableFloatChunk(); + } + + @Override + public void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/GenericObjectMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/GenericObjectMixin.java new file mode 100644 index 00000000000..0a87f2f566a --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/GenericObjectMixin.java @@ -0,0 +1,76 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableObjectChunk; +import io.deephaven.json.Value; +import io.deephaven.qst.type.GenericType; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +abstract class GenericObjectMixin extends Mixin implements ToObject { + private final GenericType type; + + public GenericObjectMixin(JsonFactory factory, T options, GenericType type) { + super(factory, options); + this.type = Objects.requireNonNull(type); + } + + @Override + public final int outputSize() { + return 1; + } + + @Override + final Stream> paths() { + return Stream.of(List.of()); + } + + @Override + final Stream> outputTypesImpl() { + return Stream.of(type); + } + + @Override + final ValueProcessor processor(String context) { + return new GenericObjectMixinProcessor(); + } + + @Override + final RepeaterProcessor repeaterProcessor() { + return new RepeaterGenericImpl<>(this, null, null, type.arrayType()); + } + + private class GenericObjectMixinProcessor extends ValueProcessorMixinBase { + + private WritableObjectChunk out; + + @Override + public final void setContext(List> out) { + this.out = out.get(0).asWritableObjectChunk(); + } + + @Override + public final void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/InstantMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/InstantMixin.java new file mode 100644 index 00000000000..7084213bbcf --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/InstantMixin.java @@ -0,0 +1,146 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableLongChunk; +import io.deephaven.json.InstantValue; +import io.deephaven.qst.type.Type; +import io.deephaven.time.DateTimeUtils; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.util.List; +import java.util.stream.Stream; + +final class InstantMixin extends Mixin { + + private final long onNull; + private final long onMissing; + + public InstantMixin(InstantValue options, JsonFactory factory) { + super(factory, options); + onNull = DateTimeUtils.epochNanos(options.onNull().orElse(null)); + onMissing = DateTimeUtils.epochNanos(options.onMissing().orElse(null)); + } + + @Override + public int outputSize() { + return 1; + } + + @Override + public Stream> paths() { + return Stream.of(List.of()); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.of(Type.instantType()); + } + + @Override + public ValueProcessor processor(String context) { + return new InstantMixinProcessor(); + } + + private long parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + private long parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new RepeaterGenericImpl<>(new ToObjectImpl(), null, null, + Type.instantType().arrayType()); + } + + class ToObjectImpl implements ToObject { + @Override + public Instant parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_STRING: + case FIELD_NAME: + return parseFromStringToInstant(parser); + case VALUE_NULL: + return parseFromNullToInstant(parser); + } + throw unexpectedToken(parser); + } + + @Override + public Instant parseMissing(JsonParser parser) throws IOException { + return parseFromMissingToInstant(parser); + } + } + + private long parseFromString(JsonParser parser) throws IOException { + final TemporalAccessor accessor = options.dateTimeFormatter().parse(Parsing.textAsCharSequence(parser)); + final long epochSeconds = accessor.getLong(ChronoField.INSTANT_SECONDS); + final int nanoOfSecond = accessor.get(ChronoField.NANO_OF_SECOND); + return epochSeconds * 1_000_000_000L + nanoOfSecond; + } + + private long parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return onNull; + } + + private long parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return onMissing; + } + + private Instant parseFromStringToInstant(JsonParser parser) throws IOException { + return Instant.from(options.dateTimeFormatter().parse(Parsing.textAsCharSequence(parser))); + } + + private Instant parseFromNullToInstant(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(null); + } + + private Instant parseFromMissingToInstant(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(null); + } + + private class InstantMixinProcessor extends ValueProcessorMixinBase { + private WritableLongChunk out; + + @Override + public final void setContext(List> out) { + this.out = out.get(0).asWritableLongChunk(); + } + + @Override + public final void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/InstantNumberMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/InstantNumberMixin.java new file mode 100644 index 00000000000..f8bd7befdb7 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/InstantNumberMixin.java @@ -0,0 +1,178 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableLongChunk; +import io.deephaven.json.InstantNumberValue; +import io.deephaven.qst.type.Type; +import io.deephaven.time.DateTimeUtils; + +import java.io.IOException; +import java.math.BigInteger; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +final class InstantNumberMixin extends Mixin { + + private final long onNull; + private final long onMissing; + + public InstantNumberMixin(InstantNumberValue options, JsonFactory factory) { + super(factory, options); + onNull = DateTimeUtils.epochNanos(options.onNull().orElse(null)); + onMissing = DateTimeUtils.epochNanos(options.onMissing().orElse(null)); + } + + @Override + public int outputSize() { + return 1; + } + + @Override + public Stream> paths() { + return Stream.of(List.of()); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.of(Type.instantType()); + } + + @Override + public ValueProcessor processor(String context) { + return new InstantNumberMixinProcessor(longFunction()); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new RepeaterGenericImpl<>(new ObjectImpl(), null, null, + Type.instantType().arrayType()); + } + + private LongImpl longFunction() { + switch (options.format()) { + case EPOCH_SECONDS: + return new LongImpl(9); + case EPOCH_MILLIS: + return new LongImpl(6); + case EPOCH_MICROS: + return new LongImpl(3); + case EPOCH_NANOS: + return new LongImpl(0); + default: + throw new IllegalStateException(); + } + } + + private class LongImpl { + + private final int scaled; + private final int mult; + + LongImpl(int scaled) { + this.scaled = scaled; + this.mult = BigInteger.valueOf(10).pow(scaled).intValueExact(); + } + + private long parseFromInt(JsonParser parser) throws IOException { + return mult * Parsing.parseIntAsLong(parser); + } + + private long parseFromDecimal(JsonParser parser) throws IOException { + // We need to parse w/ BigDecimal in the case of VALUE_NUMBER_FLOAT, otherwise we might lose accuracy + // jshell> (long)(1703292532.123456789 * 1000000000) + // $4 ==> 1703292532123456768 + // See InstantNumberOptionsTest + return Parsing.parseDecimalAsScaledLong(parser, scaled); + } + + private long parseFromString(JsonParser parser) throws IOException { + return mult * Parsing.parseStringAsLong(parser); + } + + private long parseFromDecimalString(JsonParser parser) throws IOException { + return Parsing.parseDecimalStringAsScaledLong(parser, scaled); + } + + public final long parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_NUMBER_INT: + checkNumberIntAllowed(parser); + return parseFromInt(parser); + case VALUE_NUMBER_FLOAT: + checkDecimalAllowed(parser); + return parseFromDecimal(parser); + case VALUE_STRING: + case FIELD_NAME: + checkStringAllowed(parser); + return allowDecimal() + ? parseFromDecimalString(parser) + : parseFromString(parser); + case VALUE_NULL: + checkNullAllowed(parser); + return onNull; + } + throw unexpectedToken(parser); + } + + public final long parseMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return onMissing; + } + } + + private class ObjectImpl implements ToObject { + + private final LongImpl longImpl; + + public ObjectImpl() { + this.longImpl = longFunction(); + } + + @Override + public Instant parseValue(JsonParser parser) throws IOException { + return DateTimeUtils.epochNanosToInstant(longImpl.parseValue(parser)); + } + + @Override + public Instant parseMissing(JsonParser parser) throws IOException { + return DateTimeUtils.epochNanosToInstant(longImpl.parseValue(parser)); + } + } + + private class InstantNumberMixinProcessor extends ValueProcessorMixinBase { + private final LongImpl impl; + + private WritableLongChunk out; + + public InstantNumberMixinProcessor(LongImpl impl) { + this.impl = Objects.requireNonNull(impl); + } + + @Override + public final void setContext(List> out) { + this.out = out.get(0).asWritableLongChunk(); + } + + @Override + public final void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(impl.parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(impl.parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/IntMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/IntMixin.java new file mode 100644 index 00000000000..40cd67fcfc5 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/IntMixin.java @@ -0,0 +1,152 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.base.MathUtil; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableIntChunk; +import io.deephaven.chunk.sized.SizedIntChunk; +import io.deephaven.json.IntValue; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +final class IntMixin extends Mixin { + + public IntMixin(IntValue options, JsonFactory factory) { + super(factory, options); + } + + @Override + public int outputSize() { + return 1; + } + + @Override + public Stream> paths() { + return Stream.of(List.of()); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.of(Type.intType()); + } + + @Override + public ValueProcessor processor(String context) { + return new IntMixinProcessor(); + } + + private int parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_NUMBER_INT: + return parseFromInt(parser); + case VALUE_NUMBER_FLOAT: + return parseFromDecimal(parser); + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + private int parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new IntRepeaterImpl(); + } + + final class IntRepeaterImpl extends RepeaterProcessorBase { + private final SizedIntChunk chunk = new SizedIntChunk<>(0); + + public IntRepeaterImpl() { + super(null, null, Type.intType().arrayType()); + } + + @Override + public void processElementImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableIntChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, IntMixin.this.parseValue(parser)); + chunk.setSize(newSize); + } + + @Override + public void processElementMissingImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableIntChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, IntMixin.this.parseMissing(parser)); + chunk.setSize(newSize); + } + + @Override + public int[] doneImpl(JsonParser parser, int length) { + final WritableIntChunk chunk = this.chunk.get(); + return Arrays.copyOfRange(chunk.array(), chunk.arrayOffset(), chunk.arrayOffset() + length); + } + } + + private int parseFromInt(JsonParser parser) throws IOException { + checkNumberIntAllowed(parser); + return Parsing.parseIntAsInt(parser); + } + + private int parseFromDecimal(JsonParser parser) throws IOException { + checkDecimalAllowed(parser); + return Parsing.parseDecimalAsInt(parser); + } + + private int parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + return allowDecimal() + ? Parsing.parseDecimalStringAsInt(parser) + : Parsing.parseStringAsInt(parser); + } + + private int parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(QueryConstants.NULL_INT); + } + + private int parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(QueryConstants.NULL_INT); + } + + final class IntMixinProcessor extends ValueProcessorMixinBase { + + private WritableIntChunk out; + + @Override + public void setContext(List> out) { + this.out = out.get(0).asWritableIntChunk(); + } + + @Override + public void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/JacksonConfiguration.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/JacksonConfiguration.java new file mode 100644 index 00000000000..c1de008ec88 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/JacksonConfiguration.java @@ -0,0 +1,56 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonFactoryBuilder; +import com.fasterxml.jackson.core.ObjectCodec; +import com.fasterxml.jackson.core.StreamReadFeature; + +import java.lang.reflect.InvocationTargetException; + +public final class JacksonConfiguration { + + private static final JsonFactory DEFAULT_FACTORY; + + static { + // We'll attach an ObjectMapper if it's on the classpath, this allows parsing of AnyOptions + ObjectCodec objectCodec = null; + try { + final Class clazz = Class.forName("com.fasterxml.jackson.databind.ObjectMapper"); + objectCodec = (ObjectCodec) clazz.getDeclaredConstructor().newInstance(); + } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException + | InvocationTargetException e) { + // ignore + } + DEFAULT_FACTORY = defaultFactoryBuilder().build().setCodec(objectCodec); + } + + /** + * Constructs a Deephaven-configured json factory builder. This currently includes + * {@link StreamReadFeature#USE_FAST_DOUBLE_PARSER}, {@link StreamReadFeature#USE_FAST_BIG_NUMBER_PARSER}, and + * {@link StreamReadFeature#INCLUDE_SOURCE_IN_LOCATION}. The specific configuration may change in the future. + * + * @return the Deephaven-configured json factory builder + */ + public static JsonFactoryBuilder defaultFactoryBuilder() { + return new JsonFactoryBuilder() + .enable(StreamReadFeature.USE_FAST_DOUBLE_PARSER) + .enable(StreamReadFeature.USE_FAST_BIG_NUMBER_PARSER) + .enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION); + } + + // Not currently public, but javadoc still useful to ensure internal callers don't modify. + /** + * Returns a Deephaven-configured json factory singleton. Callers should not modify the returned factory in any way. + * This has been constructed as the singleton-equivalent of {@code defaultFactoryBuilder().build()}, with an + * ObjectMapper set as the codec if it is on the classpath. + * + * @return the Deephaven-configured json factory singleton + * @see #defaultFactoryBuilder() + */ + static JsonFactory defaultFactory() { + return DEFAULT_FACTORY; + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/JacksonProvider.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/JacksonProvider.java new file mode 100644 index 00000000000..e70cd0578ef --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/JacksonProvider.java @@ -0,0 +1,291 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.TreeNode; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.json.AnyValue; +import io.deephaven.json.ArrayValue; +import io.deephaven.json.DoubleValue; +import io.deephaven.json.IntValue; +import io.deephaven.json.LongValue; +import io.deephaven.json.ObjectEntriesValue; +import io.deephaven.json.ObjectValue; +import io.deephaven.json.StringValue; +import io.deephaven.json.TupleValue; +import io.deephaven.json.TypedObjectValue; +import io.deephaven.json.Value; +import io.deephaven.processor.NamedObjectProcessor; +import io.deephaven.processor.ObjectProcessor; +import io.deephaven.qst.type.Type; +import io.deephaven.util.annotations.FinalDefault; + +import java.io.File; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.function.Function; + +/** + * A {@link Value JSON value} {@link ObjectProcessor processor} implementation using + * Jackson>. + * + *

+ * This implementation allows users to efficiently parse / destructure a + * JSON value (from a supported {@link JacksonProvider#getInputTypes() + * input type}) into {@link WritableChunk writable chunks} according to the type(s) as specified by its {@link Value + * Value type}. This is done using the Jackson streaming + * API {@link JsonParser} (as opposed to the databind / object mapping API, which must first create intermediate + * objects). + * + *

+ * The "simple" types are self-explanatory. For example, the {@link StringValue} represents a {@link String} output + * type, and (by default) expects a JSON string as input; the {@link IntValue} represents an {@code int} output type, + * and (by default) expects a JSON number as input. The allowed JSON input types can be specified via + * {@link Value#allowedTypes() allowed types}; users are encouraged to use the strictest type they can according to how + * their JSON data is serialized. + * + *

+ * The most common "complex" type is {@link ObjectValue}, which expects to parse a JSON object of known fields. The + * object contains {@link ObjectValue#fields()}, which represent other {@link Value values}. The fields are recursively + * resolved and flattened into the {@link ObjectProcessor#outputTypes()}. For example, a JSON object, which itself + * contains another JSON object + * + * 

+ * {
+ *   "city": "Plymouth",
+ *   "point": {
+ *       "latitude": 45.018269,
+ *       "longitude": -93.473892
+ *   }
+ * }
+ *
+ * + * when represented with structuring as one might expect ({@link ObjectValue}({@link StringValue}, + * {@link ObjectValue}({@link DoubleValue}, {@link DoubleValue}))), will produce {@link ObjectProcessor#outputTypes() + * output types} representing {@code [String, double, double]}. Furthermore, the field names and delimiter "_" will be + * used by default to provide the {@link NamedObjectProcessor#names() names} + * {@code ["city", "point_latitude", "point_longitude"]}. + * + *

+ * The {@link ArrayValue} represents a variable-length array, which expects to parse a JSON array where each element is + * expected to have the same {@link ArrayValue#element() element type}. (This is in contrast to JSON arrays more + * generally, where each element of the array can be a different JSON value type.) The output type will be the output + * type(s) of the element type as the component type of a native array, {@link Type#arrayType()}. For example, if we + * used the previous example as the {@link ArrayValue#element() array component type}, it will produce + * {@link ObjectProcessor#outputTypes() output types} representing {@code [String[], double[], double[]]} (the + * {@link NamedObjectProcessor#names() names} will remain unchanged). + * + *

+ * The {@link TupleValue} represents a fixed number of {@link TupleValue#namedValues() value types}, which expects to + * parse a fixed-length JSON array where each element corresponds to the same-indexed value type. The values are + * recursively resolved and flattened into the {@link ObjectProcessor#outputTypes()}; for example, the earlier example's + * data could be re-represented as the JSON array + * + * 

+ * ["Plymouth", 45.018269, -93.473892]
+ *
+ * + * and structured as one might expect ({@link TupleValue}({@link StringValue}, {@link DoubleValue}, + * {@link DoubleValue})), and will produce {@link ObjectProcessor#outputTypes() output types} representing + * {@code [String, double, double]}. Even though no field names are present in the JSON value, users may set + * {@link TupleValue#namedValues() names} for each element (and will otherwise inherit integer-indexed default names). + * + *

+ * The {@link TypedObjectValue} represents a union of {@link ObjectValue object values} where the first field is + * type-discriminating. For example, the following might be modelled as a type-discriminated object with + * type-discriminating field "type", shared "symbol" {@link StringValue}, "quote" object of "bid" {@link DoubleValue} + * and an "ask" {@link DoubleValue}, and "trade" object containing a "price" {@link DoubleValue} and a "size" + * {@link LongValue}. + * + * 

+ * {
+ *   "type": "quote",
+ *   "symbol": "BAR",
+ *   "bid": 10.01,
+ *   "ask": 10.05
+ * }
+ * {
+ *   "type": "trade",
+ *   "symbol": "FOO",
+ *   "price": 70.03,
+ *   "size": 42
+ * }
+ *
+ * + * The {@link ObjectProcessor#outputTypes() output types} are first the type-discriminating field, then the shared + * fields (if any), followed by the individual {@link ObjectValue object value} fields; with the above example, that + * would result in {@link ObjectProcessor#outputTypes() output types} + * {@code [String, String, double, double, double long]} and {@link NamedObjectProcessor#names() names} + * {@code ["type", "symbol", "quote_bid", "quote_ask", "trade_price", "trade_size"]}. + * + *

+ * The {@link ObjectEntriesValue} represents a variable-length object, which expects to parse a JSON object where each + * key-value entry has a common {@link ObjectEntriesValue#value() value type}. The output type will be the key and value + * element types as a component of native arrays ({@link Type#arrayType()}). For example, a JSON object, whose values + * are also JSON objects + * + * 

+ * {
+ *   "Plymouth": {
+ *       "latitude": 45.018269,
+ *       "longitude": -93.473892
+ *   },
+ *   "New York": {
+ *       "latitude": 40.730610,
+ *       "longitude": -73.935242
+ *   }
+ * }
+ *
+ * + * when represented with structuring as one might expect ({@link ObjectEntriesValue}({@link StringValue}, + * {@link ObjectValue}({@link DoubleValue}, {@link DoubleValue}))), will produce {@link ObjectProcessor#outputTypes() + * output types} representing {@code [String[], double[], double[]]}, and {@link NamedObjectProcessor#names() names} + * {@code ["Key", "latitude", "longitude"]}. + * + *

+ * The {@link AnyValue} type represents a {@link TreeNode} output; this requires that the Jackson databinding API be + * available on the classpath. This is useful for initial modelling and debugging purposes. + */ +public interface JacksonProvider extends NamedObjectProcessor.Provider { + + /** + * Creates a jackson provider using a default factory. Equivalent to + * {@code of(options, JacksonConfiguration.defaultFactoryBuilder().build())}. + * + * @param options the object options + * @return the jackson provider + * @see #of(Value, JsonFactory) + * @see JacksonConfiguration#defaultFactoryBuilder() + */ + static JacksonProvider of(Value options) { + return of(options, JacksonConfiguration.defaultFactory()); + } + + /** + * Creates a jackson provider using the provided {@code factory}. + * + * @param options the object options + * @param factory the jackson factory + * @return the jackson provider + */ + static JacksonProvider of(Value options, JsonFactory factory) { + return Mixin.of(options, factory); + } + + /** + * The supported types. Includes {@link String}, {@code byte[]}, {@code char[]}, {@link File}, {@link Path}, + * {@link URL}, {@link ByteBuffer}, and {@link CharBuffer}. + * + * @return the supported types + */ + static Set> getInputTypes() { + return Set.of( + Type.stringType(), + Type.byteType().arrayType(), + Type.charType().arrayType(), + Type.ofCustom(File.class), + Type.ofCustom(Path.class), + Type.ofCustom(URL.class), + Type.ofCustom(ByteBuffer.class), + Type.ofCustom(CharBuffer.class)); + } + + /** + * The supported types. Equivalent to {@link #getInputTypes()}. + * + * @return the supported types + */ + @Override + @FinalDefault + default Set> inputTypes() { + return getInputTypes(); + } + + /** + * Creates an object processor based on the {@code inputType} with a default {@link JsonFactory}. + * + * @param inputType the input type + * @return the object processor + * @param the input type + * @see #stringProcessor() + * @see #bytesProcessor() + * @see #charsProcessor() + * @see #fileProcessor() + * @see #pathProcessor() + * @see #urlProcessor() + * @see #byteBufferProcessor() + * @see #charBufferProcessor() + */ + @Override + ObjectProcessor processor(Type inputType); + + List names(Function, String> f); + + /** + * Creates a {@link String} json object processor. + * + * @return the object processor + * @see JsonFactory#createParser(String) + */ + ObjectProcessor stringProcessor(); + + /** + * Creates a {@code byte[]} json object processor. + * + * @return the object processor + * @see JsonFactory#createParser(byte[]) + */ + ObjectProcessor bytesProcessor(); + + /** + * Creates a {@code char[]} json object processor. + * + * @return the object processor + * @see JsonFactory#createParser(char[]) + */ + ObjectProcessor charsProcessor(); + + /** + * Creates a {@link File} json object processor. + * + * @return the object processor + * @see JsonFactory#createParser(File) + */ + ObjectProcessor fileProcessor(); + + /** + * Creates a {@link Path} json object processor. + * + * @return the object processor + */ + ObjectProcessor pathProcessor(); + + /** + * Creates a {@link URL} json object processor. + * + * @return the object processor + * @see JsonFactory#createParser(URL) + */ + ObjectProcessor urlProcessor(); + + /** + * Creates a {@link ByteBuffer} json object processor. + * + * @return the object processor + */ + ObjectProcessor byteBufferProcessor(); + + /** + * Creates a {@link CharBuffer} json object processor. + * + * @return the object processor + */ + ObjectProcessor charBufferProcessor(); +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/JacksonSource.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/JacksonSource.java new file mode 100644 index 00000000000..adb9b278ce0 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/JacksonSource.java @@ -0,0 +1,72 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadFeature; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; + +final class JacksonSource { + + public static JsonParser of(JsonFactory factory, String content) throws IOException { + return factory.createParser(content); + } + + public static JsonParser of(JsonFactory factory, File file) throws IOException { + return factory.createParser(file); + } + + public static JsonParser of(JsonFactory factory, Path path) throws IOException { + if (FileSystems.getDefault() == path.getFileSystem()) { + return of(factory, path.toFile()); + } + if (!factory.isEnabled(StreamReadFeature.AUTO_CLOSE_SOURCE)) { + throw new RuntimeException(String.format("Unable to create Path-based parser when '%s' is not enabled", + StreamReadFeature.AUTO_CLOSE_SOURCE)); + } + // jackson buffers internally + return factory.createParser(Files.newInputStream(path)); + } + + public static JsonParser of(JsonFactory factory, InputStream inputStream) throws IOException { + return factory.createParser(inputStream); + } + + public static JsonParser of(JsonFactory factory, URL url) throws IOException { + return factory.createParser(url); + } + + public static JsonParser of(JsonFactory factory, byte[] array, int offset, int len) throws IOException { + return factory.createParser(array, offset, len); + } + + public static JsonParser of(JsonFactory factory, ByteBuffer buffer) throws IOException { + if (buffer.hasArray()) { + return of(factory, buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); + } + return of(factory, ByteBufferInputStream.of(buffer)); + } + + public static JsonParser of(JsonFactory factory, char[] array, int offset, int len) throws IOException { + return factory.createParser(array, offset, len); + } + + public static JsonParser of(JsonFactory factory, CharBuffer buffer) throws IOException { + if (buffer.hasArray()) { + return of(factory, buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); + } + // We could be more efficient here with CharBufferReader. Surprised it's not build into JDK. + return of(factory, buffer.toString()); + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/LocalDateMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/LocalDateMixin.java new file mode 100644 index 00000000000..2ad69c30932 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/LocalDateMixin.java @@ -0,0 +1,52 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.json.LocalDateValue; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.time.LocalDate; +import java.time.temporal.TemporalAccessor; + +final class LocalDateMixin extends GenericObjectMixin { + + public LocalDateMixin(LocalDateValue options, JsonFactory factory) { + super(factory, options, Type.ofCustom(LocalDate.class)); + } + + @Override + public LocalDate parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + @Override + public LocalDate parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + private LocalDate parseFromString(JsonParser parser) throws IOException { + final TemporalAccessor accessor = options.dateTimeFormatter().parse(Parsing.textAsCharSequence(parser)); + return LocalDate.from(accessor); + } + + private LocalDate parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(null); + } + + private LocalDate parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(null); + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/LongMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/LongMixin.java new file mode 100644 index 00000000000..c7a69e7f47f --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/LongMixin.java @@ -0,0 +1,151 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.base.MathUtil; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableLongChunk; +import io.deephaven.chunk.sized.SizedLongChunk; +import io.deephaven.json.LongValue; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +final class LongMixin extends Mixin { + + public LongMixin(LongValue options, JsonFactory config) { + super(config, options); + } + + @Override + public int outputSize() { + return 1; + } + + @Override + public Stream> paths() { + return Stream.of(List.of()); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.of(Type.longType()); + } + + @Override + public ValueProcessor processor(String context) { + return new LongMixinProcessor(); + } + + private long parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_NUMBER_INT: + return parseFromInt(parser); + case VALUE_NUMBER_FLOAT: + return parseFromDecimal(parser); + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + private long parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new LongRepeaterImpl(); + } + + final class LongRepeaterImpl extends RepeaterProcessorBase { + private final SizedLongChunk chunk = new SizedLongChunk<>(0); + + public LongRepeaterImpl() { + super(null, null, Type.longType().arrayType()); + } + + @Override + public void processElementImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableLongChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, LongMixin.this.parseValue(parser)); + chunk.setSize(newSize); + } + + @Override + public void processElementMissingImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableLongChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, LongMixin.this.parseMissing(parser)); + chunk.setSize(newSize); + } + + @Override + public long[] doneImpl(JsonParser parser, int length) { + final WritableLongChunk chunk = this.chunk.get(); + return Arrays.copyOfRange(chunk.array(), chunk.arrayOffset(), chunk.arrayOffset() + length); + } + } + + private long parseFromInt(JsonParser parser) throws IOException { + checkNumberIntAllowed(parser); + return Parsing.parseIntAsLong(parser); + } + + private long parseFromDecimal(JsonParser parser) throws IOException { + checkDecimalAllowed(parser); + return Parsing.parseDecimalAsLong(parser); + } + + private long parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + return allowDecimal() + ? Parsing.parseDecimalStringAsLong(parser) + : Parsing.parseStringAsLong(parser); + } + + private long parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(QueryConstants.NULL_LONG); + } + + private long parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(QueryConstants.NULL_LONG); + } + + private class LongMixinProcessor extends ValueProcessorMixinBase { + private WritableLongChunk out; + + @Override + public void setContext(List> out) { + this.out = out.get(0).asWritableLongChunk(); + } + + @Override + public void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/Mixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/Mixin.java new file mode 100644 index 00000000000..e0b6c89f640 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/Mixin.java @@ -0,0 +1,521 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.api.util.NameValidator; +import io.deephaven.json.AnyValue; +import io.deephaven.json.ArrayValue; +import io.deephaven.json.BigDecimalValue; +import io.deephaven.json.BigIntegerValue; +import io.deephaven.json.BoolValue; +import io.deephaven.json.ByteValue; +import io.deephaven.json.CharValue; +import io.deephaven.json.DoubleValue; +import io.deephaven.json.FloatValue; +import io.deephaven.json.InstantNumberValue; +import io.deephaven.json.InstantValue; +import io.deephaven.json.IntValue; +import io.deephaven.json.JsonValueTypes; +import io.deephaven.json.LocalDateValue; +import io.deephaven.json.LongValue; +import io.deephaven.json.ObjectField; +import io.deephaven.json.ObjectEntriesValue; +import io.deephaven.json.ObjectValue; +import io.deephaven.json.ShortValue; +import io.deephaven.json.SkipValue; +import io.deephaven.json.StringValue; +import io.deephaven.json.TupleValue; +import io.deephaven.json.TypedObjectValue; +import io.deephaven.json.Value; +import io.deephaven.processor.ObjectProcessor; +import io.deephaven.qst.type.Type; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +abstract class Mixin implements JacksonProvider { + + static final Function, String> TO_COLUMN_NAME = Mixin::toColumnName; + + public static String toColumnName(List path) { + return path.isEmpty() ? "Value" : String.join("_", path); + } + + static Mixin of(Value options, JsonFactory factory) { + return options.walk(new MixinImpl(factory)); + } + + private final JsonFactory factory; + final T options; + + Mixin(JsonFactory factory, T options) { + this.factory = Objects.requireNonNull(factory); + this.options = Objects.requireNonNull(options); + } + + @Override + public final List> outputTypes() { + return outputTypesImpl().collect(Collectors.toList()); + } + + @Override + public final List names() { + return names(TO_COLUMN_NAME); + } + + @Override + public final List names(Function, String> f) { + return Arrays.asList(NameValidator.legalizeColumnNames(paths().map(f).toArray(String[]::new), true)); + } + + @SuppressWarnings("unchecked") + @Override + public final ObjectProcessor processor(Type inputType) { + final Class clazz = inputType.clazz(); + if (String.class.isAssignableFrom(clazz)) { + return (ObjectProcessor) stringProcessor(); + } + if (byte[].class.isAssignableFrom(clazz)) { + return (ObjectProcessor) bytesProcessor(); + } + if (char[].class.isAssignableFrom(clazz)) { + return (ObjectProcessor) charsProcessor(); + } + if (File.class.isAssignableFrom(clazz)) { + return (ObjectProcessor) fileProcessor(); + } + if (Path.class.isAssignableFrom(clazz)) { + return (ObjectProcessor) pathProcessor(); + } + if (URL.class.isAssignableFrom(clazz)) { + return (ObjectProcessor) urlProcessor(); + } + if (ByteBuffer.class.isAssignableFrom(clazz)) { + return (ObjectProcessor) byteBufferProcessor(); + } + if (CharBuffer.class.isAssignableFrom(clazz)) { + return (ObjectProcessor) charBufferProcessor(); + } + throw new IllegalArgumentException("Unable to create JSON processor from type " + inputType); + } + + @Override + public final ObjectProcessor stringProcessor() { + return new StringIn(); + } + + @Override + public final ObjectProcessor bytesProcessor() { + return new BytesIn(); + } + + @Override + public final ObjectProcessor charsProcessor() { + return new CharsIn(); + } + + @Override + public final ObjectProcessor fileProcessor() { + return new FileIn(); + } + + @Override + public final ObjectProcessor pathProcessor() { + return new PathIn(); + } + + @Override + public final ObjectProcessor urlProcessor() { + return new URLIn(); + } + + @Override + public final ObjectProcessor byteBufferProcessor() { + return new ByteBufferIn(); + } + + @Override + public final ObjectProcessor charBufferProcessor() { + return new CharBufferIn(); + } + + abstract ValueProcessor processor(String context); + + abstract RepeaterProcessor repeaterProcessor(); + + abstract Stream> paths(); + + abstract Stream> outputTypesImpl(); + + static List prefixWith(String prefix, List path) { + return Stream.concat(Stream.of(prefix), path.stream()).collect(Collectors.toList()); + } + + static Stream> prefixWithKeys(Map> fields) { + final List>> paths = new ArrayList<>(fields.size()); + for (Entry> e : fields.entrySet()) { + final Stream> prefixedPaths = e.getValue().paths().map(x -> prefixWith(e.getKey().name(), x)); + paths.add(prefixedPaths); + } + return paths.stream().flatMap(Function.identity()); + } + + static Stream> prefixWithKeysAndSkip(Map> fields, int skip) { + final List>> paths = new ArrayList<>(fields.size()); + for (Entry> e : fields.entrySet()) { + final Stream> prefixedPaths = + e.getValue().paths().map(x -> prefixWith(e.getKey(), x)).skip(skip); + paths.add(prefixedPaths); + } + return paths.stream().flatMap(Function.identity()); + } + + private abstract class ObjectProcessorMixin extends ObjectProcessorJsonValue { + public ObjectProcessorMixin() { + super(Mixin.this.processor("")); + } + } + + private class StringIn extends ObjectProcessorMixin { + @Override + protected JsonParser createParser(String in) throws IOException { + return JacksonSource.of(factory, in); + } + } + + private class BytesIn extends ObjectProcessorMixin { + @Override + protected JsonParser createParser(byte[] in) throws IOException { + return JacksonSource.of(factory, in, 0, in.length); + } + } + + private class ByteBufferIn extends ObjectProcessorMixin { + @Override + protected JsonParser createParser(ByteBuffer in) throws IOException { + return JacksonSource.of(factory, in); + } + } + + private class CharBufferIn extends ObjectProcessorMixin { + @Override + protected JsonParser createParser(CharBuffer in) throws IOException { + return JacksonSource.of(factory, in); + } + } + + private class CharsIn extends ObjectProcessorMixin { + @Override + protected JsonParser createParser(char[] in) throws IOException { + return JacksonSource.of(factory, in, 0, in.length); + } + } + + private class FileIn extends ObjectProcessorMixin { + @Override + protected JsonParser createParser(File in) throws IOException { + return JacksonSource.of(factory, in); + } + } + + private class PathIn extends ObjectProcessorMixin { + @Override + protected JsonParser createParser(Path in) throws IOException { + return JacksonSource.of(factory, in); + } + } + + private class URLIn extends ObjectProcessorMixin { + @Override + protected JsonParser createParser(URL in) throws IOException { + return JacksonSource.of(factory, in); + } + } + + private static class MixinImpl implements Value.Visitor> { + private final JsonFactory factory; + + public MixinImpl(JsonFactory factory) { + this.factory = Objects.requireNonNull(factory); + } + + @Override + public StringMixin visit(StringValue _string) { + return new StringMixin(_string, factory); + } + + @Override + public Mixin visit(BoolValue _bool) { + return new BoolMixin(_bool, factory); + } + + @Override + public Mixin visit(ByteValue _byte) { + return new ByteMixin(_byte, factory); + } + + @Override + public Mixin visit(CharValue _char) { + return new CharMixin(_char, factory); + } + + @Override + public Mixin visit(ShortValue _short) { + return new ShortMixin(_short, factory); + } + + @Override + public IntMixin visit(IntValue _int) { + return new IntMixin(_int, factory); + } + + @Override + public LongMixin visit(LongValue _long) { + return new LongMixin(_long, factory); + } + + @Override + public FloatMixin visit(FloatValue _float) { + return new FloatMixin(_float, factory); + } + + @Override + public DoubleMixin visit(DoubleValue _double) { + return new DoubleMixin(_double, factory); + } + + @Override + public ObjectMixin visit(ObjectValue object) { + return new ObjectMixin(object, factory); + } + + @Override + public Mixin visit(ObjectEntriesValue objectKv) { + return new ObjectEntriesMixin(objectKv, factory); + } + + @Override + public InstantMixin visit(InstantValue instant) { + return new InstantMixin(instant, factory); + } + + @Override + public InstantNumberMixin visit(InstantNumberValue instantNumber) { + return new InstantNumberMixin(instantNumber, factory); + } + + @Override + public BigIntegerMixin visit(BigIntegerValue bigInteger) { + return new BigIntegerMixin(bigInteger, factory); + } + + @Override + public BigDecimalMixin visit(BigDecimalValue bigDecimal) { + return new BigDecimalMixin(bigDecimal, factory); + } + + @Override + public SkipMixin visit(SkipValue skip) { + return new SkipMixin(skip, factory); + } + + @Override + public TupleMixin visit(TupleValue tuple) { + return new TupleMixin(tuple, factory); + } + + @Override + public TypedObjectMixin visit(TypedObjectValue typedObject) { + return new TypedObjectMixin(typedObject, factory); + } + + @Override + public LocalDateMixin visit(LocalDateValue localDate) { + return new LocalDateMixin(localDate, factory); + } + + @Override + public ArrayMixin visit(ArrayValue array) { + return new ArrayMixin(array, factory); + } + + @Override + public AnyMixin visit(AnyValue any) { + return new AnyMixin(any, factory); + } + } + + final boolean allowNull() { + return options.allowedTypes().contains(JsonValueTypes.NULL); + } + + final boolean allowMissing() { + return options.allowMissing(); + } + + final boolean allowNumberInt() { + return options.allowedTypes().contains(JsonValueTypes.INT); + } + + final boolean allowDecimal() { + return options.allowedTypes().contains(JsonValueTypes.DECIMAL); + } + + final void checkNumberAllowed(JsonParser parser) throws IOException { + if (!allowNumberInt() && !allowDecimal()) { + throw new ValueAwareException("Number not allowed", parser.currentLocation(), options); + } + } + + final void checkNumberIntAllowed(JsonParser parser) throws IOException { + if (!allowNumberInt()) { + throw new ValueAwareException("Number int not allowed", parser.currentLocation(), options); + } + } + + final void checkDecimalAllowed(JsonParser parser) throws IOException { + if (!allowDecimal()) { + throw new ValueAwareException("Decimal not allowed", parser.currentLocation(), options); + } + } + + final void checkBoolAllowed(JsonParser parser) throws IOException { + if (!options.allowedTypes().contains(JsonValueTypes.BOOL)) { + throw new ValueAwareException("Bool not allowed", parser.currentLocation(), options); + } + } + + final void checkStringAllowed(JsonParser parser) throws IOException { + if (!options.allowedTypes().contains(JsonValueTypes.STRING)) { + throw new ValueAwareException("String not allowed", parser.currentLocation(), options); + } + } + + final void checkObjectAllowed(JsonParser parser) throws IOException { + if (!options.allowedTypes().contains(JsonValueTypes.OBJECT)) { + throw new ValueAwareException("Object not allowed", parser.currentLocation(), options); + } + } + + final void checkArrayAllowed(JsonParser parser) throws IOException { + if (!options.allowedTypes().contains(JsonValueTypes.ARRAY)) { + throw new ValueAwareException("Array not allowed", parser.currentLocation(), options); + } + } + + final void checkNullAllowed(JsonParser parser) throws IOException { + if (!allowNull()) { + throw nullNotAllowed(parser); + } + } + + final ValueAwareException nullNotAllowed(JsonParser parser) { + return new ValueAwareException("Null not allowed", parser.currentLocation(), options); + } + + final void checkMissingAllowed(JsonParser parser) throws IOException { + if (!allowMissing()) { + throw new ValueAwareException("Missing not allowed", parser.currentLocation(), options); + } + } + + final IOException unexpectedToken(JsonParser parser) throws ValueAwareException { + final String msg; + switch (parser.currentToken()) { + case VALUE_TRUE: + case VALUE_FALSE: + msg = "Bool not expected"; + break; + case START_OBJECT: + msg = "Object not expected"; + break; + case START_ARRAY: + msg = "Array not expected"; + break; + case VALUE_NUMBER_INT: + msg = "Number int not expected"; + break; + case VALUE_NUMBER_FLOAT: + msg = "Decimal not expected"; + break; + case FIELD_NAME: + msg = "Field name not expected"; + break; + case VALUE_STRING: + msg = "String not expected"; + break; + case VALUE_NULL: + msg = "Null not expected"; + break; + default: + msg = parser.currentToken() + " not expected"; + } + throw new ValueAwareException(msg, parser.currentLocation(), options); + } + + abstract class ValueProcessorMixinBase implements ValueProcessor { + @Override + public final int numColumns() { + return Mixin.this.outputSize(); + } + + @Override + public final Stream> columnTypes() { + return Mixin.this.outputTypesImpl(); + } + + @Override + public final void processCurrentValue(JsonParser parser) throws IOException { + try { + processCurrentValueImpl(parser); + } catch (ValueAwareException e) { + if (options.equals(e.value())) { + throw e; + } else { + throw wrap(parser, e, "Unable to process current value"); + } + } catch (IOException e) { + throw wrap(parser, e, "Unable to process current value"); + } + } + + @Override + public final void processMissing(JsonParser parser) throws IOException { + try { + processMissingImpl(parser); + } catch (ValueAwareException e) { + if (options.equals(e.value())) { + throw e; + } else { + throw wrap(parser, e, "Unable to process missing value"); + } + } catch (IOException e) { + throw wrap(parser, e, "Unable to process missing value"); + } + } + + protected abstract void processCurrentValueImpl(JsonParser parser) throws IOException; + + protected abstract void processMissingImpl(JsonParser parser) throws IOException; + + private ValueAwareException wrap(JsonParser parser, IOException e, String msg) { + return new ValueAwareException(msg, parser.currentLocation(), e, options); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ObjectEntriesMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ObjectEntriesMixin.java new file mode 100644 index 00000000000..75c3267168d --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ObjectEntriesMixin.java @@ -0,0 +1,105 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.json.ObjectEntriesValue; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Stream; + +final class ObjectEntriesMixin extends Mixin { + private final Mixin key; + private final Mixin value; + + public ObjectEntriesMixin(ObjectEntriesValue options, JsonFactory factory) { + super(factory, options); + key = Mixin.of(options.key(), factory); + value = Mixin.of(options.value(), factory); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.concat(key.outputTypesImpl(), value.outputTypesImpl()).map(Type::arrayType); + } + + @Override + public int outputSize() { + return key.outputSize() + value.outputSize(); + } + + @Override + public Stream> paths() { + final Stream> keyPath = + key.outputSize() == 1 && key.paths().findFirst().orElseThrow().isEmpty() + ? Stream.of(List.of("Key")) + : key.paths(); + final Stream> valuePath = + value.outputSize() == 1 && value.paths().findFirst().orElseThrow().isEmpty() + ? Stream.of(List.of("Value")) + : value.paths(); + return Stream.concat(keyPath, valuePath); + } + + @Override + public ValueProcessor processor(String context) { + return new ObjectEntriesMixinProcessor(); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new ValueInnerRepeaterProcessor(new ObjectEntriesMixinProcessor()); + } + + private class ObjectEntriesMixinProcessor extends ValueProcessorMixinBase { + + private final RepeaterProcessor keyProcessor; + private final RepeaterProcessor valueProcessor; + + ObjectEntriesMixinProcessor() { + this.keyProcessor = key.repeaterProcessor(); + this.valueProcessor = value.repeaterProcessor(); + } + + @Override + public void setContext(List> out) { + final int keySize = keyProcessor.numColumns(); + keyProcessor.setContext(out.subList(0, keySize)); + valueProcessor.setContext(out.subList(keySize, keySize + valueProcessor.numColumns())); + } + + @Override + public void clearContext() { + keyProcessor.clearContext(); + valueProcessor.clearContext(); + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case START_OBJECT: + RepeaterProcessor.processObjectKeyValues(parser, keyProcessor, valueProcessor); + return; + case VALUE_NULL: + checkNullAllowed(parser); + keyProcessor.processNullRepeater(parser); + valueProcessor.processNullRepeater(parser); + return; + default: + throw unexpectedToken(parser); + } + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + keyProcessor.processMissingRepeater(parser); + valueProcessor.processMissingRepeater(parser); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ObjectMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ObjectMixin.java new file mode 100644 index 00000000000..ea14baf40bf --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ObjectMixin.java @@ -0,0 +1,448 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import io.deephaven.json.ObjectField; +import io.deephaven.json.ObjectField.RepeatedBehavior; +import io.deephaven.json.ObjectValue; +import io.deephaven.json.jackson.RepeaterProcessor.Context; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Stream; + +final class ObjectMixin extends Mixin { + + private final Map> mixins; + private final int numOutputs; + + public ObjectMixin(ObjectValue options, JsonFactory factory) { + super(factory, options); + final LinkedHashMap> map = new LinkedHashMap<>(options.fields().size()); + for (ObjectField field : options.fields()) { + map.put(field, Mixin.of(field.options(), factory)); + } + mixins = Collections.unmodifiableMap(map); + numOutputs = mixins.values().stream().mapToInt(Mixin::outputSize).sum(); + } + + @Override + public Stream> outputTypesImpl() { + return mixins.values().stream().flatMap(Mixin::outputTypesImpl); + } + + @Override + public int outputSize() { + return numOutputs; + } + + @Override + public Stream> paths() { + return prefixWithKeys(mixins); + } + + @Override + public ValueProcessor processor(String context) { + return processor(context, false); + } + + public ValueProcessor processor(String context, boolean isDiscriminated) { + final Map processors = new LinkedHashMap<>(mixins.size()); + int ix = 0; + for (Entry> e : mixins.entrySet()) { + final ObjectField field = e.getKey(); + final Mixin opts = e.getValue(); + final int numTypes = opts.outputSize(); + final ValueProcessor fieldProcessor = opts.processor(context + "/" + field.name()); + processors.put(field, fieldProcessor); + ix += numTypes; + } + if (ix != outputSize()) { + throw new IllegalStateException(); + } + return processorImpl(processors, isDiscriminated); + } + + @Override + RepeaterProcessor repeaterProcessor() { + final Map processors = new LinkedHashMap<>(mixins.size()); + int ix = 0; + for (Entry> e : mixins.entrySet()) { + final ObjectField field = e.getKey(); + final Mixin opts = e.getValue(); + final int numTypes = opts.outputSize(); + final RepeaterProcessor fieldProcessor = opts.repeaterProcessor(); + processors.put(field, fieldProcessor); + ix += numTypes; + } + if (ix != outputSize()) { + throw new IllegalStateException(); + } + return new ObjectValueRepeaterProcessor(processors); + } + + private boolean allCaseSensitive() { + return options.fields().stream().allMatch(ObjectField::caseSensitive); + } + + ObjectValueFieldProcessor processorImpl(Map fields, boolean isDiscriminatedObject) { + return new ObjectValueFieldProcessor(fields, isDiscriminatedObject); + } + + final class ObjectValueFieldProcessor extends ContextAwareDelegateBase implements ValueProcessor, FieldProcessor { + private final Map fields; + private final Map map; + private final boolean isDiscriminated; + + ObjectValueFieldProcessor(Map fields, boolean isDiscriminated) { + super(fields.values()); + this.fields = fields; + this.map = allCaseSensitive() + ? new HashMap<>() + : new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + this.isDiscriminated = isDiscriminated; + for (Entry e : fields.entrySet()) { + final ObjectField field = e.getKey(); + map.put(field.name(), field); + for (String alias : field.aliases()) { + map.put(alias, field); + } + } + this.visited = new HashSet<>(fields.size()); + } + + private ObjectField lookupField(String fieldName) { + final ObjectField field = map.get(fieldName); + if (field == null) { + return null; + } + if (!field.caseSensitive()) { + return field; + } + // Need to handle the case where some fields are case-insensitive, but this one is _not_. + if (field.name().equals(fieldName) || field.aliases().contains(fieldName)) { + return field; + } + return null; + } + + private ValueProcessor processor(ObjectField options) { + return Objects.requireNonNull(fields.get(options)); + } + + @Override + public void processCurrentValue(JsonParser parser) throws IOException { + // In the normal case, we expect to be at START_OBJECT or VALUE_NULL. + // In the discriminated case, we expect to already be inside an object (FIELD_NAME or END_OBJECT). + switch (parser.currentToken()) { + case START_OBJECT: + if (isDiscriminated) { + throw unexpectedToken(parser); + } + if (parser.nextToken() == JsonToken.END_OBJECT) { + processEmptyObject(parser); + return; + } + if (!parser.hasToken(JsonToken.FIELD_NAME)) { + throw new IllegalStateException(); + } + processObjectFields(parser); + return; + case VALUE_NULL: + if (isDiscriminated) { + throw unexpectedToken(parser); + } + processNullObject(parser); + return; + case FIELD_NAME: + if (!isDiscriminated) { + throw unexpectedToken(parser); + } + processObjectFields(parser); + return; + case END_OBJECT: + if (!isDiscriminated) { + throw unexpectedToken(parser); + } + processEmptyObject(parser); + return; + default: + throw unexpectedToken(parser); + } + } + + @Override + public void processMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + for (Entry entry : fields.entrySet()) { + processMissingField(entry.getKey(), entry.getValue(), parser); + } + } + + private void processNullObject(JsonParser parser) throws IOException { + checkNullAllowed(parser); + for (Entry entry : fields.entrySet()) { + processField(entry.getKey(), entry.getValue(), parser); + } + } + + private void processEmptyObject(JsonParser parser) throws IOException { + // This logic should be equivalent to processObjectFields, but where we know there are no fields + for (Entry entry : fields.entrySet()) { + processMissingField(entry.getKey(), entry.getValue(), parser); + } + } + + // ----------------------------------------------------------------------------------------------------------- + + private final Set visited; + + private void processObjectFields(JsonParser parser) throws IOException { + try { + FieldProcessor.processFields(parser, this); + if (visited.size() == fields.size()) { + // All fields visited, none missing + return; + } + for (Entry e : fields.entrySet()) { + if (!visited.contains(e.getKey())) { + processMissingField(e.getKey(), e.getValue(), parser); + } + } + } finally { + visited.clear(); + } + } + + @Override + public void process(String fieldName, JsonParser parser) throws IOException { + final ObjectField field = lookupField(fieldName); + if (field == null) { + if (!options.allowUnknownFields()) { + throw new ValueAwareException(String.format("Unknown field '%s' not allowed", fieldName), + parser.currentLocation(), options); + } + parser.skipChildren(); + } else if (visited.add(field)) { + // First time seeing field + processField(field, processor(field), parser); + } else if (field.repeatedBehavior() == RepeatedBehavior.USE_FIRST) { + parser.skipChildren(); + } else { + throw new ValueAwareException(String.format("Field '%s' has already been visited", fieldName), + parser.currentLocation(), options); + } + } + + // ----------------------------------------------------------------------------------------------------------- + + private void processField(ObjectField field, ValueProcessor processor, JsonParser parser) + throws ValueAwareException { + try { + processor.processCurrentValue(parser); + } catch (IOException | RuntimeException e) { + throw new ValueAwareException(String.format("Unable to process field '%s'", field.name()), + parser.currentLocation(), e, options); + } + } + + private void processMissingField(ObjectField field, ValueProcessor processor, JsonParser parser) + throws ValueAwareException { + try { + processor.processMissing(parser); + } catch (IOException | RuntimeException e) { + throw new ValueAwareException(String.format("Unable to process field '%s'", field.name()), + parser.currentLocation(), e, options); + } + } + } + + final class ObjectValueRepeaterProcessor extends ContextAwareDelegateBase + implements RepeaterProcessor, Context, FieldProcessor { + private final Map fields; + private final Map contexts; + private final Map map; + + public ObjectValueRepeaterProcessor(Map fields) { + super(fields.values()); + this.fields = Objects.requireNonNull(fields); + contexts = new LinkedHashMap<>(fields.size()); + for (Entry e : fields.entrySet()) { + contexts.put(e.getKey(), e.getValue().context()); + } + this.map = allCaseSensitive() + ? new HashMap<>() + : new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (Entry e : fields.entrySet()) { + final ObjectField field = e.getKey(); + map.put(field.name(), field); + for (String alias : field.aliases()) { + map.put(alias, field); + } + } + this.visited = new HashSet<>(fields.size()); + } + + private Collection processors() { + return fields.values(); + } + + @Override + public Context context() { + return this; + } + + @Override + public void processNullRepeater(JsonParser parser) throws IOException { + for (RepeaterProcessor p : processors()) { + p.processNullRepeater(parser); + } + } + + @Override + public void processMissingRepeater(JsonParser parser) throws IOException { + for (RepeaterProcessor p : processors()) { + p.processMissingRepeater(parser); + } + } + + private ObjectField lookupField(String fieldName) { + final ObjectField field = map.get(fieldName); + if (field == null) { + return null; + } + if (!field.caseSensitive()) { + return field; + } + // Need to handle the case where some fields are case-insensitive, but this one is _not_. + if (field.name().equals(fieldName) || field.aliases().contains(fieldName)) { + return field; + } + return null; + } + + private Context context(ObjectField o) { + return Objects.requireNonNull(contexts.get(o)); + } + + @Override + public void start(JsonParser parser) throws IOException { + for (Context value : contexts.values()) { + value.start(parser); + } + } + + @Override + public void processElement(JsonParser parser) throws IOException { + // Not supporting TypedObjectValue array (TypedObjectMixin#repeaterProcessor) yet, so don't need + // discrimination support here. + switch (parser.currentToken()) { + case START_OBJECT: + if (parser.nextToken() == JsonToken.END_OBJECT) { + processEmptyObject(parser); + return; + } + if (!parser.hasToken(JsonToken.FIELD_NAME)) { + throw new IllegalStateException(); + } + processObjectFields(parser); + return; + case VALUE_NULL: + processNullObject(parser); + return; + default: + throw unexpectedToken(parser); + } + } + + private void processNullObject(JsonParser parser) throws IOException { + // element is null + // pass-through JsonToken.VALUE_NULL + for (Context context : contexts.values()) { + context.processElement(parser); + } + } + + private void processEmptyObject(JsonParser parser) throws IOException { + // This logic should be equivalent to processObjectFields, but where we know there are no fields + for (Context context : contexts.values()) { + context.processElementMissing(parser); + } + } + + // ----------------------------------------------------------------------------------------------------------- + + private final Set visited; + + private void processObjectFields(JsonParser parser) throws IOException { + try { + FieldProcessor.processFields(parser, this); + if (visited.size() == fields.size()) { + // All fields visited, none missing + return; + } + for (Entry e : contexts.entrySet()) { + if (!visited.contains(e.getKey())) { + e.getValue().processElementMissing(parser); + } + } + } finally { + visited.clear(); + } + } + + @Override + public void process(String fieldName, JsonParser parser) throws IOException { + final ObjectField field = lookupField(fieldName); + if (field == null) { + if (!options.allowUnknownFields()) { + throw new ValueAwareException( + String.format("Unexpected field '%s' and allowUnknownFields == false", fieldName), + parser.currentLocation(), options); + } + parser.skipChildren(); + } else if (visited.add(field)) { + // First time seeing field + context(field).processElement(parser); + } else if (field.repeatedBehavior() == RepeatedBehavior.USE_FIRST) { + parser.skipChildren(); + } else { + throw new ValueAwareException( + String.format("Field '%s' has already been visited and repeatedBehavior == %s", fieldName, + field.repeatedBehavior()), + parser.currentLocation(), options); + } + } + + // ----------------------------------------------------------------------------------------------------------- + + @Override + public void processElementMissing(JsonParser parser) throws IOException { + for (Context context : contexts.values()) { + context.processElementMissing(parser); + } + } + + @Override + public void done(JsonParser parser) throws IOException { + for (Context context : contexts.values()) { + context.done(parser); + } + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ObjectProcessorJsonValue.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ObjectProcessorJsonValue.java new file mode 100644 index 00000000000..8268ab3b348 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ObjectProcessorJsonValue.java @@ -0,0 +1,57 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.processor.ObjectProcessor; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +abstract class ObjectProcessorJsonValue implements ObjectProcessor { + + protected abstract JsonParser createParser(X in) throws IOException; + + private final ValueProcessor processor; + + ObjectProcessorJsonValue(ValueProcessor processor) { + this.processor = Objects.requireNonNull(processor); + } + + @Override + public final int outputSize() { + return processor.numColumns(); + } + + @Override + public final List> outputTypes() { + return processor.columnTypes().collect(Collectors.toList()); + } + + @Override + public final void processAll(ObjectChunk in, List> out) { + processor.setContext(out); + try { + processAllImpl(in); + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + processor.clearContext(); + } + } + + void processAllImpl(ObjectChunk in) throws IOException { + for (int i = 0; i < in.size(); ++i) { + try (final JsonParser parser = createParser(in.get(i))) { + ValueProcessor.processFullJson(parser, processor); + } + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/Parsing.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/Parsing.java new file mode 100644 index 00000000000..b50e306ec89 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/Parsing.java @@ -0,0 +1,302 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import ch.randelshofer.fastdoubleparser.JavaBigDecimalParser; +import ch.randelshofer.fastdoubleparser.JavaBigIntegerParser; +import ch.randelshofer.fastdoubleparser.JavaDoubleParser; +import ch.randelshofer.fastdoubleparser.JavaFloatParser; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.core.StreamReadFeature; +import io.deephaven.util.BooleanUtils; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.CharBuffer; + +final class Parsing { + + static void assertNoCurrentToken(JsonParser parser) { + if (parser.hasCurrentToken()) { + throw new IllegalStateException( + String.format("Expected no current token. actual=%s", parser.currentToken())); + } + } + + static void assertNextToken(JsonParser parser, JsonToken expected) throws IOException { + final JsonToken actual = parser.nextToken(); + if (actual != expected) { + throw new IllegalStateException( + String.format("Unexpected next token. expected=%s, actual=%s", expected, actual)); + } + } + + static void assertCurrentToken(JsonParser parser, JsonToken expected) { + if (!parser.hasToken(expected)) { + throw new IllegalStateException( + String.format("Unexpected current token. expected=%s, actual=%s", expected, parser.currentToken())); + } + } + + static CharSequence textAsCharSequence(JsonParser parser) throws IOException { + return parser.hasTextCharacters() + ? CharBuffer.wrap(parser.getTextCharacters(), parser.getTextOffset(), parser.getTextLength()) + : parser.getText(); + } + + static byte parseStringAsByteBool(JsonParser parser, byte onNull) throws IOException { + final String text = parser.getText().trim(); + if ("true".equalsIgnoreCase(text)) { + return BooleanUtils.TRUE_BOOLEAN_AS_BYTE; + } + if ("false".equalsIgnoreCase(text)) { + return BooleanUtils.FALSE_BOOLEAN_AS_BYTE; + } + if ("null".equalsIgnoreCase(text)) { + return onNull; + } + throw new IOException(String.format("Unexpected string as boolean '%s'", text)); + } + + static Boolean parseStringAsBoolean(JsonParser parser, Boolean onNull) throws IOException { + final String text = parser.getText().trim(); + if ("true".equalsIgnoreCase(text)) { + return Boolean.TRUE; + } + if ("false".equalsIgnoreCase(text)) { + return Boolean.FALSE; + } + if ("null".equalsIgnoreCase(text)) { + return onNull; + } + throw new IOException(String.format("Unexpected string as boolean '%s'", text)); + } + + static char parseStringAsChar(JsonParser parser) throws IOException { + if (parser.hasTextCharacters()) { + final int textLength = parser.getTextLength(); + if (textLength != 1) { + throw new IOException( + String.format("Expected char to be string of length 1, is instead %d", textLength)); + } + return parser.getTextCharacters()[parser.getTextOffset()]; + } + final String text = parser.getText(); + if (text.length() != 1) { + throw new IOException( + String.format("Expected char to be string of length 1, is instead %d", text.length())); + } + return text.charAt(0); + } + + // --------------------------------------------------------------------------- + + static byte parseIntAsByte(JsonParser parser) throws IOException { + return parser.getByteValue(); + } + + static byte parseDecimalAsByte(JsonParser parser) throws IOException { + return parser.getByteValue(); + } + + static byte parseDecimalStringAsByte(JsonParser parser) throws IOException { + // parse as float then cast to byte; no loss of whole number part (32 bit -> 8 bit) if in range + return (byte) parseStringAsFloat(parser); + } + + static byte parseStringAsByte(JsonParser parser) throws IOException { + return (byte) parseStringAsInt(parser); + } + + // --------------------------------------------------------------------------- + + + static short parseIntAsShort(JsonParser parser) throws IOException { + return parser.getShortValue(); + } + + static short parseDecimalAsShort(JsonParser parser) throws IOException { + // parse as double then cast to short; no loss of whole number part (64 bit -> 16 bit) + return parser.getShortValue(); + } + + static short parseDecimalStringAsShort(JsonParser parser) throws IOException { + // parse as float then cast to short; no loss of whole number part (32 bit -> 16 bit) if in range + return (short) parseStringAsFloat(parser); + } + + static short parseStringAsShort(JsonParser parser) throws IOException { + return (short) parseStringAsInt(parser); + } + + // --------------------------------------------------------------------------- + + static int parseIntAsInt(JsonParser parser) throws IOException { + return parser.getIntValue(); + } + + static int parseDecimalAsInt(JsonParser parser) throws IOException { + // parse as double then cast to int; no loss of whole number part (64 bit -> 32 bit) + return parser.getIntValue(); + } + + // --------------------------------------------------------------------------- + + static long parseIntAsLong(JsonParser parser) throws IOException { + return parser.getLongValue(); + } + + static long parseDecimalAsLong(JsonParser parser) throws IOException { + // Parsing as double then casting to long will lose precision. + // (long)9223372036854775806.0 == 9223372036854775807 + return parser.getDecimalValue().longValue(); + } + + static long parseDecimalAsScaledLong(JsonParser parser, int n) throws IOException { + return parser.getDecimalValue().scaleByPowerOfTen(n).longValue(); + } + + // --------------------------------------------------------------------------- + + static String parseStringAsString(JsonParser parser) throws IOException { + return parser.getText(); + } + + static String parseIntAsString(JsonParser parser) throws IOException { + return parser.getText(); + } + + static String parseDecimalAsString(JsonParser parser) throws IOException { + return parser.getText(); + } + + static String parseBoolAsString(JsonParser parser) throws IOException { + return parser.getText(); + } + + // --------------------------------------------------------------------------- + + static float parseNumberAsFloat(JsonParser parser) throws IOException { + // TODO: improve after https://github.com/FasterXML/jackson-core/issues/1229 + return parser.getFloatValue(); + } + + static double parseNumberAsDouble(JsonParser parser) throws IOException { + // TODO: improve after https://github.com/FasterXML/jackson-core/issues/1229 + return parser.getDoubleValue(); + } + + // --------------------------------------------------------------------------- + + static int parseStringAsInt(JsonParser parser) throws IOException { + if (parser.hasTextCharacters()) { + final int len = parser.getTextLength(); + final CharSequence cs = CharBuffer.wrap(parser.getTextCharacters(), parser.getTextOffset(), len); + return Integer.parseInt(cs, 0, len, 10); + } else { + return Integer.parseInt(parser.getText()); + } + } + + static int parseDecimalStringAsInt(JsonParser parser) throws IOException { + // parse as double then cast to int; no loss of whole number part (64 bit -> 32 bit) + return (int) parseStringAsDouble(parser); + } + + static long parseStringAsLong(JsonParser parser) throws IOException { + if (parser.hasTextCharacters()) { + final int len = parser.getTextLength(); + final CharSequence cs = CharBuffer.wrap(parser.getTextCharacters(), parser.getTextOffset(), len); + return Long.parseLong(cs, 0, len, 10); + } else { + return Long.parseLong(parser.getText()); + } + } + + static long parseDecimalStringAsLong(JsonParser parser) throws IOException { + // To ensure 64-bit in cases where the string is a decimal, we need BigDecimal + return parseStringAsBigDecimal(parser).longValue(); + } + + static long parseDecimalStringAsScaledLong(JsonParser parser, int n) throws IOException { + // To ensure 64-bit in cases where the string is a decimal, we need BigDecimal + return parseStringAsBigDecimal(parser).scaleByPowerOfTen(n).longValue(); + } + + static float parseStringAsFloat(JsonParser parser) throws IOException { + // TODO: improve after https://github.com/FasterXML/jackson-core/issues/1229 + return parser.isEnabled(StreamReadFeature.USE_FAST_DOUBLE_PARSER) + ? parseStringAsFloatFast(parser) + : Float.parseFloat(parser.getText()); + } + + static double parseStringAsDouble(JsonParser parser) throws IOException { + // TODO: improve after https://github.com/FasterXML/jackson-core/issues/1229 + return parser.isEnabled(StreamReadFeature.USE_FAST_DOUBLE_PARSER) + ? parseStringAsDoubleFast(parser) + : Double.parseDouble(parser.getText()); + } + + // --------------------------------------------------------------------------- + + static BigInteger parseIntAsBigInteger(JsonParser parser) throws IOException { + return parser.getBigIntegerValue(); + } + + static BigInteger parseDecimalAsBigInteger(JsonParser parser) throws IOException { + return parser.getBigIntegerValue(); + } + + static BigInteger parseStringAsBigInteger(JsonParser parser) throws IOException { + // TODO: improve after https://github.com/FasterXML/jackson-core/issues/1229 + return parser.isEnabled(StreamReadFeature.USE_FAST_BIG_NUMBER_PARSER) + ? parseStringAsBigIntegerFast(parser) + : new BigInteger(parser.getText()); + } + + static BigInteger parseDecimalStringAsBigInteger(JsonParser parser) throws IOException { + return parseStringAsBigDecimal(parser).toBigInteger(); + } + + // --------------------------------------------------------------------------- + + static BigDecimal parseDecimalAsBigDecimal(JsonParser parser) throws IOException { + return parser.getDecimalValue(); + } + + static BigDecimal parseStringAsBigDecimal(JsonParser parser) throws IOException { + // TODO: improve after https://github.com/FasterXML/jackson-core/issues/1229 + return parser.isEnabled(StreamReadFeature.USE_FAST_BIG_NUMBER_PARSER) + ? parseStringAsBigDecimalFast(parser) + : new BigDecimal(parser.getText()); + } + + // --------------------------------------------------------------------------- + + private static float parseStringAsFloatFast(JsonParser p) throws IOException { + return p.hasTextCharacters() + ? JavaFloatParser.parseFloat(p.getTextCharacters(), p.getTextOffset(), p.getTextLength()) + : JavaFloatParser.parseFloat(p.getText()); + } + + private static double parseStringAsDoubleFast(JsonParser p) throws IOException { + return p.hasTextCharacters() + ? JavaDoubleParser.parseDouble(p.getTextCharacters(), p.getTextOffset(), p.getTextLength()) + : JavaDoubleParser.parseDouble(p.getText()); + } + + private static BigInteger parseStringAsBigIntegerFast(JsonParser p) throws IOException { + return p.hasTextCharacters() + ? JavaBigIntegerParser.parseBigInteger(p.getTextCharacters(), p.getTextOffset(), p.getTextLength()) + : JavaBigIntegerParser.parseBigInteger(p.getText()); + } + + private static BigDecimal parseStringAsBigDecimalFast(JsonParser p) throws IOException { + return p.hasTextCharacters() + ? JavaBigDecimalParser.parseBigDecimal(p.getTextCharacters(), p.getTextOffset(), p.getTextLength()) + : JavaBigDecimalParser.parseBigDecimal(p.getText()); + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/RepeaterGenericImpl.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/RepeaterGenericImpl.java new file mode 100644 index 00000000000..0f2ba69774e --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/RepeaterGenericImpl.java @@ -0,0 +1,49 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.base.MathUtil; +import io.deephaven.chunk.WritableObjectChunk; +import io.deephaven.chunk.sized.SizedObjectChunk; +import io.deephaven.qst.type.NativeArrayType; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +final class RepeaterGenericImpl extends RepeaterProcessorBase { + private final ToObject toObject; + private final SizedObjectChunk chunk; + + public RepeaterGenericImpl(ToObject toObject, T[] onMissing, T[] onNull, NativeArrayType arrayType) { + super(onMissing, onNull, arrayType); + this.toObject = Objects.requireNonNull(toObject); + chunk = new SizedObjectChunk<>(0); + } + + @Override + public void processElementImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableObjectChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, toObject.parseValue(parser)); + chunk.setSize(newSize); + } + + @Override + public void processElementMissingImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableObjectChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, toObject.parseMissing(parser)); + chunk.setSize(newSize); + } + + @Override + public T[] doneImpl(JsonParser parser, int length) { + final WritableObjectChunk chunk = this.chunk.get(); + final T[] result = Arrays.copyOfRange(chunk.array(), chunk.arrayOffset(), chunk.arrayOffset() + length); + chunk.fillWithNullValue(0, length); + return result; + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/RepeaterProcessor.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/RepeaterProcessor.java new file mode 100644 index 00000000000..2118ba40bb4 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/RepeaterProcessor.java @@ -0,0 +1,63 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import java.io.IOException; + +// generalized for array vs object-kv +interface RepeaterProcessor extends ContextAware { + + static void processArray( + JsonParser parser, + RepeaterProcessor processor) throws IOException { + Parsing.assertCurrentToken(parser, JsonToken.START_ARRAY); + final Context context = processor.context(); + context.start(parser); + while (parser.nextToken() != JsonToken.END_ARRAY) { + context.processElement(parser); + } + context.done(parser); + } + + static void processObjectKeyValues( + JsonParser parser, + RepeaterProcessor keyProcessor, + RepeaterProcessor valueProcessor) throws IOException { + Parsing.assertCurrentToken(parser, JsonToken.START_OBJECT); + final Context keyContext = keyProcessor.context(); + final Context valueContext = valueProcessor.context(); + keyContext.start(parser); + valueContext.start(parser); + while (parser.nextToken() != JsonToken.END_OBJECT) { + Parsing.assertCurrentToken(parser, JsonToken.FIELD_NAME); + keyContext.processElement(parser); + parser.nextToken(); + valueContext.processElement(parser); + } + keyContext.done(parser); + valueContext.done(parser); + } + + void processNullRepeater(JsonParser parser) throws IOException; + + void processMissingRepeater(JsonParser parser) throws IOException; + + Context context(); + + interface Context { + + void start(JsonParser parser) throws IOException; + + void processElement(JsonParser parser) throws IOException; + + // While traditional arrays can't have missing elements, when an object is an array, a field may be missing: + // [ { "foo": 1, "bar": 2 }, {"bar": 3} ] + void processElementMissing(JsonParser parser) throws IOException; + + void done(JsonParser parser) throws IOException; + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/RepeaterProcessorBase.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/RepeaterProcessorBase.java new file mode 100644 index 00000000000..f9ecc19d752 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/RepeaterProcessorBase.java @@ -0,0 +1,99 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableObjectChunk; +import io.deephaven.json.jackson.RepeaterProcessor.Context; +import io.deephaven.qst.type.NativeArrayType; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +abstract class RepeaterProcessorBase implements RepeaterProcessor, Context { + + private final T onMissing; + private final T onNull; + + private final NativeArrayType arrayType; + + private WritableObjectChunk out; + private int ix; + + public RepeaterProcessorBase(T onMissing, T onNull, NativeArrayType arrayType) { + this.onMissing = onMissing; + this.onNull = onNull; + this.arrayType = Objects.requireNonNull(arrayType); + } + + public void startImpl(JsonParser parser) throws IOException {} + + public abstract void processElementImpl(JsonParser parser, int index) throws IOException; + + public abstract void processElementMissingImpl(JsonParser parser, int index) throws IOException; + + public abstract T doneImpl(JsonParser parser, int length) throws IOException; + + @Override + public final void setContext(List> out) { + this.out = out.get(0).asWritableObjectChunk(); + } + + @Override + public final void clearContext() { + out = null; + } + + @Override + public final int numColumns() { + return 1; + } + + @Override + public Stream> columnTypes() { + return Stream.of(arrayType); + } + + @Override + public final Context context() { + return this; + } + + @Override + public final void processMissingRepeater(JsonParser parser) throws IOException { + out.add(onMissing); + } + + @Override + public final void processNullRepeater(JsonParser parser) throws IOException { + out.add(onNull); + } + + @Override + public final void start(JsonParser parser) throws IOException { + startImpl(parser); + ix = 0; + } + + @Override + public final void processElement(JsonParser parser) throws IOException { + processElementImpl(parser, ix); + ++ix; + } + + @Override + public final void processElementMissing(JsonParser parser) throws IOException { + processElementMissingImpl(parser, ix); + ++ix; + } + + @Override + public final void done(JsonParser parser) throws IOException { + out.add(doneImpl(parser, ix)); + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ShortMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ShortMixin.java new file mode 100644 index 00000000000..786491b1863 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ShortMixin.java @@ -0,0 +1,151 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.base.MathUtil; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableShortChunk; +import io.deephaven.chunk.sized.SizedShortChunk; +import io.deephaven.json.ShortValue; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +final class ShortMixin extends Mixin { + public ShortMixin(ShortValue options, JsonFactory factory) { + super(factory, options); + } + + @Override + public int outputSize() { + return 1; + } + + @Override + public Stream> paths() { + return Stream.of(List.of()); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.of(Type.shortType()); + } + + @Override + public ValueProcessor processor(String context) { + return new ShortMixinProcessor(); + } + + private short parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_NUMBER_INT: + return parseFromInt(parser); + case VALUE_NUMBER_FLOAT: + return parseFromDecimal(parser); + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + private short parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new ShortRepeaterImpl(); + } + + final class ShortRepeaterImpl extends RepeaterProcessorBase { + private final SizedShortChunk chunk = new SizedShortChunk<>(0); + + public ShortRepeaterImpl() { + super(null, null, Type.shortType().arrayType()); + } + + @Override + public void processElementImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableShortChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, ShortMixin.this.parseValue(parser)); + chunk.setSize(newSize); + } + + @Override + public void processElementMissingImpl(JsonParser parser, int index) throws IOException { + final int newSize = index + 1; + final WritableShortChunk chunk = this.chunk.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + chunk.set(index, ShortMixin.this.parseMissing(parser)); + chunk.setSize(newSize); + } + + @Override + public short[] doneImpl(JsonParser parser, int length) { + final WritableShortChunk chunk = this.chunk.get(); + return Arrays.copyOfRange(chunk.array(), chunk.arrayOffset(), chunk.arrayOffset() + length); + } + } + + private short parseFromInt(JsonParser parser) throws IOException { + checkNumberIntAllowed(parser); + return Parsing.parseIntAsShort(parser); + } + + private short parseFromDecimal(JsonParser parser) throws IOException { + checkDecimalAllowed(parser); + return Parsing.parseDecimalAsShort(parser); + } + + private short parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + return allowDecimal() + ? Parsing.parseDecimalStringAsShort(parser) + : Parsing.parseStringAsShort(parser); + } + + private short parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(QueryConstants.NULL_SHORT); + } + + private short parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(QueryConstants.NULL_SHORT); + } + + final class ShortMixinProcessor extends ValueProcessorMixinBase { + + private WritableShortChunk out; + + @Override + public void setContext(List> out) { + this.out = out.get(0).asWritableShortChunk(); + } + + @Override + public void clearContext() { + out = null; + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + out.add(parseValue(parser)); + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + out.add(parseMissing(parser)); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/SkipMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/SkipMixin.java new file mode 100644 index 00000000000..b291a0000d9 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/SkipMixin.java @@ -0,0 +1,159 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.json.SkipValue; +import io.deephaven.json.jackson.RepeaterProcessor.Context; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Stream; + +final class SkipMixin extends Mixin implements ValueProcessor { + + public SkipMixin(SkipValue options, JsonFactory factory) { + super(factory, options); + } + + @Override + public int outputSize() { + return 0; + } + + @Override + public int numColumns() { + return 0; + } + + @Override + public Stream> paths() { + return Stream.empty(); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.empty(); + } + + @Override + public ValueProcessor processor(String context) { + return this; + } + + @Override + public void setContext(List> out) { + + } + + @Override + public void clearContext() { + + } + + @Override + RepeaterProcessor repeaterProcessor() { + return new SkipArray(); + } + + @Override + public Stream> columnTypes() { + return Stream.empty(); + } + + @Override + public void processCurrentValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case START_OBJECT: + checkObjectAllowed(parser); + parser.skipChildren(); + break; + case START_ARRAY: + checkArrayAllowed(parser); + parser.skipChildren(); + break; + case VALUE_STRING: + case FIELD_NAME: + checkStringAllowed(parser); + break; + case VALUE_NUMBER_INT: + checkNumberIntAllowed(parser); + break; + case VALUE_NUMBER_FLOAT: + checkDecimalAllowed(parser); + break; + case VALUE_TRUE: + case VALUE_FALSE: + checkBoolAllowed(parser); + break; + case VALUE_NULL: + checkNullAllowed(parser); + break; + default: + throw unexpectedToken(parser); + } + } + + @Override + public void processMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + } + + private final class SkipArray implements RepeaterProcessor, Context { + + @Override + public Context context() { + return this; + } + + @Override + public void processNullRepeater(JsonParser parser) throws IOException {} + + @Override + public void processMissingRepeater(JsonParser parser) throws IOException {} + + @Override + public void setContext(List> out) { + + } + + @Override + public void clearContext() { + + } + + @Override + public int numColumns() { + return 0; + } + + @Override + public Stream> columnTypes() { + return Stream.empty(); + } + + @Override + public void start(JsonParser parser) throws IOException { + + } + + @Override + public void processElement(JsonParser parser) throws IOException { + processCurrentValue(parser); + } + + @Override + public void processElementMissing(JsonParser parser) throws IOException { + processMissing(parser); + } + + @Override + public void done(JsonParser parser) throws IOException { + + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/StringMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/StringMixin.java new file mode 100644 index 00000000000..19c64504430 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/StringMixin.java @@ -0,0 +1,72 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.json.StringValue; +import io.deephaven.qst.type.Type; + +import java.io.IOException; + +final class StringMixin extends GenericObjectMixin { + + public StringMixin(StringValue options, JsonFactory factory) { + super(factory, options, Type.stringType()); + } + + @Override + public String parseValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case VALUE_STRING: + case FIELD_NAME: + return parseFromString(parser); + case VALUE_NUMBER_INT: + return parseFromInt(parser); + case VALUE_NUMBER_FLOAT: + return parseFromDecimal(parser); + case VALUE_TRUE: + case VALUE_FALSE: + return parseFromBool(parser); + case VALUE_NULL: + return parseFromNull(parser); + } + throw unexpectedToken(parser); + } + + @Override + public String parseMissing(JsonParser parser) throws IOException { + return parseFromMissing(parser); + } + + private String parseFromString(JsonParser parser) throws IOException { + checkStringAllowed(parser); + return Parsing.parseStringAsString(parser); + } + + private String parseFromInt(JsonParser parser) throws IOException { + checkNumberIntAllowed(parser); + return Parsing.parseIntAsString(parser); + } + + private String parseFromDecimal(JsonParser parser) throws IOException { + checkDecimalAllowed(parser); + return Parsing.parseDecimalAsString(parser); + } + + private String parseFromBool(JsonParser parser) throws IOException { + checkBoolAllowed(parser); + return Parsing.parseBoolAsString(parser); + } + + private String parseFromNull(JsonParser parser) throws IOException { + checkNullAllowed(parser); + return options.onNull().orElse(null); + } + + private String parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + return options.onMissing().orElse(null); + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ToObject.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ToObject.java new file mode 100644 index 00000000000..8f91cc089ce --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ToObject.java @@ -0,0 +1,15 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonParser; + +import java.io.IOException; + +interface ToObject { + + T parseValue(JsonParser parser) throws IOException; + + T parseMissing(JsonParser parser) throws IOException; +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/TupleMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/TupleMixin.java new file mode 100644 index 00000000000..a520ea12703 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/TupleMixin.java @@ -0,0 +1,265 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import io.deephaven.json.TupleValue; +import io.deephaven.json.Value; +import io.deephaven.json.jackson.RepeaterProcessor.Context; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Stream; + +import static io.deephaven.json.jackson.Parsing.assertCurrentToken; + +final class TupleMixin extends Mixin { + + private final Map> mixins; + private final int numColumns; + + public TupleMixin(TupleValue options, JsonFactory factory) { + super(factory, options); + final LinkedHashMap> map = new LinkedHashMap<>(options.namedValues().size()); + for (Entry e : options.namedValues().entrySet()) { + map.put(e.getKey(), Mixin.of(e.getValue(), factory)); + } + mixins = Collections.unmodifiableMap(map); + numColumns = mixins.values().stream().mapToInt(Mixin::outputSize).sum(); + } + + @Override + public int outputSize() { + return numColumns; + } + + @Override + public Stream> paths() { + if (mixins.size() == 1) { + return mixins.values().iterator().next().paths(); + } + final List>> prefixed = new ArrayList<>(); + for (Entry> e : mixins.entrySet()) { + prefixed.add(e.getValue().paths().map(x -> prefixWith(e.getKey(), x))); + } + return prefixed.stream().flatMap(Function.identity()); + } + + @Override + public Stream> outputTypesImpl() { + return mixins.values().stream().flatMap(Mixin::outputTypesImpl); + } + + @Override + public ValueProcessor processor(String context) { + final List processors = new ArrayList<>(options.namedValues().size()); + int ix = 0; + for (Entry> e : mixins.entrySet()) { + final Mixin mixin = e.getValue(); + final int numTypes = mixin.outputSize(); + final ValueProcessor processor = mixin.processor(context + "[" + e.getKey() + "]"); + processors.add(processor); + ix += numTypes; + } + if (ix != outputSize()) { + throw new IllegalStateException(); + } + return new TupleProcessor(processors); + } + + @Override + RepeaterProcessor repeaterProcessor() { + final List processors = new ArrayList<>(mixins.size()); + int ix = 0; + for (Entry> e : mixins.entrySet()) { + final Mixin mixin = e.getValue(); + final int numTypes = mixin.outputSize(); + final RepeaterProcessor processor = mixin.repeaterProcessor(); + processors.add(processor); + ix += numTypes; + } + if (ix != outputSize()) { + throw new IllegalStateException(); + } + return new TupleArrayProcessor(processors); + } + + private class TupleProcessor extends ContextAwareDelegateBase implements ValueProcessor { + private final List values; + + public TupleProcessor(List values) { + super(values); + this.values = Objects.requireNonNull(values); + } + + @Override + public void processCurrentValue(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case START_ARRAY: + processTuple(parser); + return; + case VALUE_NULL: + processNullTuple(parser); + return; + default: + throw unexpectedToken(parser); + } + } + + @Override + public void processMissing(JsonParser parser) throws IOException { + parseFromMissing(parser); + } + + private void processTuple(JsonParser parser) throws IOException { + int ix = 0; + for (ValueProcessor value : values) { + parser.nextToken(); + try { + value.processCurrentValue(parser); + } catch (IOException | RuntimeException e) { + throw new ValueAwareException(String.format("Unable to process tuple ix %d", ix), + parser.currentLocation(), e, options); + } + ++ix; + } + parser.nextToken(); + assertCurrentToken(parser, JsonToken.END_ARRAY); + } + + private void processNullTuple(JsonParser parser) throws IOException { + checkNullAllowed(parser); + int ix = 0; + // Note: we are treating a null tuple the same as a tuple of null objects + // null ~= [null, ..., null] + for (ValueProcessor value : values) { + // Note: _not_ incrementing to nextToken + try { + value.processCurrentValue(parser); + } catch (IOException | RuntimeException e) { + throw new ValueAwareException(String.format("Unable to process tuple ix %d", ix), + parser.currentLocation(), e, options); + } + ++ix; + } + } + + private void parseFromMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + int ix = 0; + // Note: we are treating a missing tuple the same as a tuple of missing objects (which, is technically + // impossible w/ native json, but it's the semantics we are exposing). + // ~= [, ..., ] + for (ValueProcessor value : values) { + try { + value.processMissing(parser); + } catch (IOException | RuntimeException e) { + throw new ValueAwareException(String.format("Unable to process tuple ix %d", ix), + parser.currentLocation(), e, options); + } + ++ix; + } + } + } + + final class TupleArrayProcessor extends ContextAwareDelegateBase implements RepeaterProcessor, Context { + private final List values; + private final List contexts; + + public TupleArrayProcessor(List values) { + super(values); + this.values = Objects.requireNonNull(values); + this.contexts = new ArrayList<>(values.size()); + for (RepeaterProcessor value : values) { + contexts.add(value.context()); + } + } + + @Override + public Context context() { + return this; + } + + @Override + public void processNullRepeater(JsonParser parser) throws IOException { + for (RepeaterProcessor value : values) { + value.processNullRepeater(parser); + } + } + + @Override + public void processMissingRepeater(JsonParser parser) throws IOException { + for (RepeaterProcessor value : values) { + value.processMissingRepeater(parser); + } + } + + @Override + public void start(JsonParser parser) throws IOException { + for (Context context : contexts) { + context.start(parser); + } + } + + @Override + public void processElement(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case START_ARRAY: + processTuple(parser); + break; + case VALUE_NULL: + processNullTuple(parser); + break; + default: + throw unexpectedToken(parser); + } + } + + private void processTuple(JsonParser parser) throws IOException { + for (Context context : contexts) { + parser.nextToken(); + context.processElement(parser); + } + parser.nextToken(); + assertCurrentToken(parser, JsonToken.END_ARRAY); + } + + private void processNullTuple(JsonParser parser) throws IOException { + checkNullAllowed(parser); + // Note: we are treating a null tuple the same as a tuple of null objects + // null ~= [null, ..., null] + for (Context context : contexts) { + context.processElement(parser); + } + } + + @Override + public void processElementMissing(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + // Note: we are treating a missing tuple the same as a tuple of missing objects (which, is technically + // impossible w/ native json, but it's the semantics we are exposing). + // ~= [, ..., ] + for (Context context : contexts) { + context.processElementMissing(parser); + } + } + + @Override + public void done(JsonParser parser) throws IOException { + for (Context context : contexts) { + context.done(parser); + } + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/TypedObjectMixin.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/TypedObjectMixin.java new file mode 100644 index 00000000000..62ea03ecc8c --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/TypedObjectMixin.java @@ -0,0 +1,315 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableObjectChunk; +import io.deephaven.json.ObjectField; +import io.deephaven.json.ObjectValue; +import io.deephaven.json.StringValue; +import io.deephaven.json.TypedObjectValue; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Stream; + +final class TypedObjectMixin extends Mixin { + + private final TreeSet typeFieldAliases; + private final Map> sharedFields; + private final Map combinedFields; + private final int numSharedColumns; + private final int numSpecificColumns; + + public TypedObjectMixin(TypedObjectValue options, JsonFactory factory) { + super(factory, options); + if (!(options.typeField().options() instanceof StringValue)) { + throw new IllegalArgumentException("Only string-valued type fields are currently supported"); + } + final Object onNull = options.onNull().orElse(null); + if (onNull != null && !(onNull instanceof String)) { + throw new IllegalArgumentException("Only String onNull values are currently supported"); + } + final Object onMissing = options.onMissing().orElse(null); + if (onMissing != null && !(onMissing instanceof String)) { + throw new IllegalArgumentException("Only String onMissing values are currently supported"); + } + typeFieldAliases = options.typeField().caseSensitive() ? null : new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + { + final LinkedHashMap> map = new LinkedHashMap<>(options.sharedFields().size()); + for (ObjectField sharedField : options.sharedFields()) { + map.put(sharedField, Mixin.of(sharedField.options(), factory)); + } + sharedFields = Collections.unmodifiableMap(map); + } + { + final LinkedHashMap map = new LinkedHashMap<>(options.objects().size()); + for (Entry e : options.objects().entrySet()) { + map.put((String) e.getKey(), new ObjectMixin(combinedObject(e.getValue()), factory)); + } + combinedFields = Collections.unmodifiableMap(map); + } + numSharedColumns = sharedFields.values().stream().mapToInt(Mixin::outputSize).sum(); + numSpecificColumns = + combinedFields.values().stream().mapToInt(ObjectMixin::outputSize).map(x -> x - numSharedColumns).sum(); + } + + @Override + public int outputSize() { + return 1 + numSharedColumns + numSpecificColumns; + } + + @Override + public Stream> paths() { + return Stream.concat( + Stream.of(List.of(options.typeField().name())), + Stream.concat( + prefixWithKeys(sharedFields), + prefixWithKeysAndSkip(combinedFields, numSharedColumns))); + } + + @Override + public Stream> outputTypesImpl() { + return Stream.concat( + Stream.of(Type.stringType()), + Stream.concat( + sharedFields.values().stream().flatMap(Mixin::outputTypesImpl), + combinedFields.values().stream().map(Mixin::outputTypesImpl) + .flatMap(x -> x.skip(numSharedColumns)))); + } + + @Override + public ValueProcessor processor(String context) { + final Map processors = new LinkedHashMap<>(combinedFields.size()); + for (Entry e : combinedFields.entrySet()) { + final String type = e.getKey(); + final ValueProcessor processor = e.getValue().processor(context + "[" + type + "]", true); + processors.put(type, new Processor(processor)); + } + return new DiscriminatedProcessor(processors); + } + + @Override + RepeaterProcessor repeaterProcessor() { + throw new UnsupportedOperationException(); + } + + private ObjectValue combinedObject(ObjectValue objectOpts) { + final Set sharedFields = options.sharedFields(); + if (sharedFields.isEmpty()) { + return objectOpts; + } + return ObjectValue.builder() + .allowUnknownFields(objectOpts.allowUnknownFields()) + .allowMissing(objectOpts.allowMissing()) + .allowedTypes(objectOpts.allowedTypes()) + .addAllFields(sharedFields) + .addAllFields(objectOpts.fields()) + .build(); + } + + private boolean matchesTypeField(String name) { + if (options.typeField().caseSensitive()) { + return options.typeField().name().equals(name) || options.typeField().aliases().contains(name); + } else { + return options.typeField().name().equalsIgnoreCase(name) || typeFieldAliases.contains(name); + } + } + + private String parseTypeField(JsonParser parser) throws IOException { + final String firstFieldName = parser.currentName(); + if (!matchesTypeField(firstFieldName)) { + throw new ValueAwareException(String.format("Expected the first field to be '%s', is '%s'", + options.typeField().name(), firstFieldName), parser.currentLocation(), options); + } + switch (parser.nextToken()) { + case VALUE_STRING: + case FIELD_NAME: + return parser.getText(); + case VALUE_NUMBER_INT: + return parser.getText(); + case VALUE_NULL: + return null; + default: + throw unexpectedToken(parser); + } + } + + private static class Processor { + private final ValueProcessor combinedProcessor; + private final List> buffer; + private List> specificOut; + + Processor(ValueProcessor combinedProcessor) { + this.combinedProcessor = Objects.requireNonNull(combinedProcessor); + this.buffer = new ArrayList<>(combinedProcessor.numColumns()); + } + + void setContext(List> sharedOut, List> specificOut) { + this.specificOut = Objects.requireNonNull(specificOut); + buffer.clear(); + buffer.addAll(sharedOut); + buffer.addAll(specificOut); + combinedProcessor.setContext(buffer); + } + + void clearContext() { + combinedProcessor.clearContext(); + buffer.clear(); + specificOut = null; + } + + ValueProcessor combinedProcessor() { + return combinedProcessor; + } + + void notApplicable() { + // only skip specific fields + for (WritableChunk wc : specificOut) { + addNullValue(wc); + } + } + } + + private class DiscriminatedProcessor extends ValueProcessorMixinBase { + + private final Map combinedProcessors; + + private WritableObjectChunk typeChunk; + private List> sharedChunks; + + public DiscriminatedProcessor(Map combinedProcessors) { + this.combinedProcessors = Objects.requireNonNull(combinedProcessors); + } + + @Override + public void setContext(List> out) { + typeChunk = out.get(0).asWritableObjectChunk(); + sharedChunks = out.subList(1, 1 + numSharedColumns); + int outIx = 1 + sharedChunks.size(); + for (Processor combinedProcessor : combinedProcessors.values()) { + final int numColumns = combinedProcessor.combinedProcessor().numColumns(); + final int numSpecificColumns = numColumns - numSharedColumns; + final List> specificChunks = out.subList(outIx, outIx + numSpecificColumns); + combinedProcessor.setContext(sharedChunks, specificChunks); + outIx += numSpecificColumns; + } + } + + @Override + public void clearContext() { + typeChunk = null; + sharedChunks = null; + for (Processor combinedProcessor : combinedProcessors.values()) { + combinedProcessor.clearContext(); + } + } + + @Override + protected void processCurrentValueImpl(JsonParser parser) throws IOException { + switch (parser.currentToken()) { + case START_OBJECT: + if (parser.nextToken() == JsonToken.END_OBJECT) { + processEmptyObject(parser); + return; + } + if (!parser.hasToken(JsonToken.FIELD_NAME)) { + throw new IllegalStateException(); + } + processObjectFields(parser); + return; + case VALUE_NULL: + processNullObject(parser); + return; + default: + throw unexpectedToken(parser); + } + } + + @Override + protected void processMissingImpl(JsonParser parser) throws IOException { + checkMissingAllowed(parser); + typeChunk.add((String) options.onMissing().orElse(null)); + // We are _not_ trying to pass along the potential "on missing" value for each individual chunk; the + // individual columns may have "allowMissing = false", but as a higher level of control with + // TypedObjectValue, we have already verified that we want to allow missing. As such, the discriminating + // factor will be the typeChunk. + for (WritableChunk sharedChunk : sharedChunks) { + addNullValue(sharedChunk); + } + for (Processor processor : combinedProcessors.values()) { + processor.notApplicable(); + } + } + + private void processNullObject(JsonParser parser) throws IOException { + checkNullAllowed(parser); + typeChunk.add((String) options.onNull().orElse(null)); + // We are _not_ trying to pass along the potential "on null" value for each individual chunk; the + // individual columns may have "allowNull = false", but as a higher level of control with + // TypedObjectValue, we have already verified that we want to allow null. As such, the discriminating + // factor will be the typeChunk. + for (WritableChunk sharedChunk : sharedChunks) { + addNullValue(sharedChunk); + } + for (Processor processor : combinedProcessors.values()) { + processor.notApplicable(); + } + } + + private void processEmptyObject(JsonParser parser) throws IOException { + throw new ValueAwareException("Expected a non-empty object", parser.currentLocation(), options); + } + + private void processObjectFields(JsonParser parser) throws IOException { + final String typeFieldValue = parseTypeField(parser); + typeChunk.add(typeFieldValue); + parser.nextToken(); + boolean foundProcessor = false; + for (Entry e : combinedProcessors.entrySet()) { + final String processorType = e.getKey(); + final Processor processor = e.getValue(); + // Note: we are not supporting case-insensitive _value_ matching at this point in time. We do allow the + // field _names_ to be case insensitive (see ObjectField#caseSensitive). + // See io.deephaven.json.TypedObjectValueTest#caseSensitiveDiscriminator + if (processorType.equals(typeFieldValue)) { + processor.combinedProcessor().processCurrentValue(parser); + foundProcessor = true; + } else { + processor.notApplicable(); + } + } + if (!foundProcessor) { + if (!options.allowUnknownTypes()) { + throw new ValueAwareException(String.format("Unknown type '%s' not allowed", typeFieldValue), + parser.currentLocation(), options); + } + for (WritableChunk sharedChunk : sharedChunks) { + addNullValue(sharedChunk); + } + // We need to skip all the fields. parser.skipChildren() is not applicable here because we are already + // inside the object (as opposed to at START_OBJECT). + FieldProcessor.skipFields(parser); + } + } + } + + private static void addNullValue(WritableChunk writableChunk) { + final int size = writableChunk.size(); + writableChunk.fillWithNullValue(size, 1); + writableChunk.setSize(size + 1); + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ValueAwareException.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ValueAwareException.java new file mode 100644 index 00000000000..90027d1e783 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ValueAwareException.java @@ -0,0 +1,34 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonLocation; +import com.fasterxml.jackson.core.JsonProcessingException; +import io.deephaven.json.Value; + +import java.util.Objects; + +public class ValueAwareException extends JsonProcessingException { + + private final Value value; + + public ValueAwareException(String msg, JsonLocation loc, Value valueContext) { + super(msg, loc); + this.value = Objects.requireNonNull(valueContext); + } + + public ValueAwareException(String msg, JsonLocation loc, Throwable cause, Value valueContext) { + super(msg, loc, cause); + this.value = Objects.requireNonNull(valueContext); + } + + public Value value() { + return value; + } + + @Override + protected String getMessageSuffix() { + return " for " + value; + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ValueInnerRepeaterProcessor.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ValueInnerRepeaterProcessor.java new file mode 100644 index 00000000000..589909e80da --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ValueInnerRepeaterProcessor.java @@ -0,0 +1,147 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonParser; +import io.deephaven.base.MathUtil; +import io.deephaven.chunk.ChunkType; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableObjectChunk; +import io.deephaven.chunk.sized.SizedObjectChunk; +import io.deephaven.json.jackson.RepeaterProcessor.Context; +import io.deephaven.processor.ObjectProcessor; +import io.deephaven.qst.type.Type; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +final class ValueInnerRepeaterProcessor implements RepeaterProcessor, Context { + + private final ValueProcessor innerProcessor; + private final List> innerChunks; + private final List> sizedObjectChunks; + + private int ix; + + private List> out; + + public ValueInnerRepeaterProcessor(ValueProcessor innerProcessor) { + final List> innerChunks = innerProcessor.columnTypes() + .map(Type::arrayType) + .map(ObjectProcessor::chunkType) + .peek(ValueInnerRepeaterProcessor::checkObjectChunk) + .map(o -> o.makeWritableChunk(1)) + .peek(wc -> wc.setSize(0)) + .collect(Collectors.toList()); + // we _know_ these are all object chunks, given NativeArrayType + // noinspection unchecked,rawtypes + this.innerChunks = (List>) (List) innerChunks; + this.innerProcessor = innerProcessor; + this.innerProcessor.setContext(innerChunks); + sizedObjectChunks = IntStream.range(0, innerProcessor.numColumns()) + .mapToObj(i -> new SizedObjectChunk<>(0)) + .collect(Collectors.toList()); + } + + static void checkObjectChunk(ChunkType chunkType) { + if (chunkType != ChunkType.Object) { + throw new IllegalStateException(); + } + } + + @Override + public void setContext(List> out) { + // noinspection unchecked,rawtypes + this.out = (List>) (List) Objects.requireNonNull(out); + } + + @Override + public void clearContext() { + out = null; + } + + @Override + public int numColumns() { + return sizedObjectChunks.size(); + } + + @Override + public Stream> columnTypes() { + return innerProcessor.columnTypes().map(Type::arrayType); + } + + @Override + public void processNullRepeater(JsonParser parser) throws IOException { + for (WritableObjectChunk wc : out) { + wc.add(null); + } + } + + @Override + public void processMissingRepeater(JsonParser parser) throws IOException { + for (WritableObjectChunk wc : out) { + wc.add(null); + } + } + + @Override + public Context context() { + return this; + } + + @Override + public void start(JsonParser parser) throws IOException { + // Note: we are setting the context once up-front + // innerProcessor.setContext(innerChunks); + ix = 0; + } + + @Override + public void processElement(JsonParser parser) throws IOException { + innerProcessor.processCurrentValue(parser); + processImpl(); + } + + @Override + public void processElementMissing(JsonParser parser) throws IOException { + innerProcessor.processMissing(parser); + processImpl(); + } + + private void processImpl() { + final int newSize = ix + 1; + final int L = sizedObjectChunks.size(); + for (int i = 0; i < L; ++i) { + // noinspection unchecked + final WritableObjectChunk from = (WritableObjectChunk) innerChunks.get(i); + // noinspection unchecked + final SizedObjectChunk to = (SizedObjectChunk) sizedObjectChunks.get(i); + // we _could_ consider doing this in a chunked fashion. doing in simple fashion to initially test + to.ensureCapacityPreserve(MathUtil.roundUpArraySize(newSize)); + to.get().set(ix, from.get(0)); + to.get().setSize(newSize); + from.set(0, null); + from.setSize(0); + } + ix = newSize; + } + + @Override + public void done(JsonParser parser) throws IOException { + // Note: we are setting the context once up-front + // innerProcessor.clearContext(); + final int L = sizedObjectChunks.size(); + for (int i = 0; i < L; ++i) { + final WritableObjectChunk from = sizedObjectChunks.get(i).get(); + final Object[] objects = Arrays.copyOfRange(from.array(), from.arrayOffset(), from.arrayOffset() + ix); + from.fillWithNullValue(0, ix); + out.get(i).add(objects); + } + } +} diff --git a/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ValueProcessor.java b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ValueProcessor.java new file mode 100644 index 00000000000..865f62160f2 --- /dev/null +++ b/extensions/json-jackson/src/main/java/io/deephaven/json/jackson/ValueProcessor.java @@ -0,0 +1,59 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import java.io.IOException; + +import static io.deephaven.json.jackson.Parsing.assertNextToken; +import static io.deephaven.json.jackson.Parsing.assertNoCurrentToken; + +interface ValueProcessor extends ContextAware { + + static void processFullJson(JsonParser parser, ValueProcessor processor) throws IOException { + assertNoCurrentToken(parser); + final JsonToken startToken = parser.nextToken(); + if (startToken == null) { + processor.processMissing(parser); + return; + } + processor.processCurrentValue(parser); + // note: AnyOptions impl which is based on com.fasterxml.jackson.core.JsonParser.readValueAsTree + // clears out the token, so we can't necessarily check it. + // parser.getLastClearedToken() + // assertCurrentToken(parser, endToken(startToken)); + assertNextToken(parser, null); + } + + // semantically _similar_ to + // com.fasterxml.jackson.databind.JsonDeserializer.deserialize(com.fasterxml.jackson.core.JsonParser, + // com.fasterxml.jackson.databind.DeserializationContext), + // but not functional (want to destructure efficiently) + + /** + * Called when the JSON value is present; the current token should be one of {@link JsonToken#START_OBJECT}, + * {@link JsonToken#START_ARRAY}, {@link JsonToken#VALUE_STRING}, {@link JsonToken#VALUE_NUMBER_INT}, + * {@link JsonToken#VALUE_NUMBER_FLOAT}, {@link JsonToken#VALUE_TRUE}, {@link JsonToken#VALUE_FALSE}, or + * {@link JsonToken#VALUE_NULL}. When the current token is {@link JsonToken#START_OBJECT}, the implementation must + * end with the corresponding {@link JsonToken#END_OBJECT} as the current token; when the current token is + * {@link JsonToken#START_ARRAY}, the implementation must end with the corresponding {@link JsonToken#END_ARRAY} as + * the current token; otherwise, the implementation must not change the current token. + * + * @param parser the parser + * @throws IOException if an IOException occurs + */ + void processCurrentValue(JsonParser parser) throws IOException; + + /** + * Called when the JSON value is missing; the current token may or may not be {@code null}. For example, if a + * field is missing from a JSON object, it's likely that missing values will be notified when the current token is + * {@link JsonToken#END_OBJECT}. Implementations must not modify the state of {@code parser}. + * + * @param parser the parser + * @throws IOException if an IOException occurs + */ + void processMissing(JsonParser parser) throws IOException; +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/ArrayTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/ArrayTest.java new file mode 100644 index 00000000000..b350a70b32c --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/ArrayTest.java @@ -0,0 +1,102 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.InstantNumberValue.Format; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.Instant; +import java.util.Map; + +import static io.deephaven.json.TestHelper.parse; + +public class ArrayTest { + + @Test + void primitive() throws IOException { + // [1.1, null, 3.3] + parse(DoubleValue.standard().array(), + "[1.1, null, 3.3]", + ObjectChunk.chunkWrap(new Object[] {new double[] {1.1, QueryConstants.NULL_DOUBLE, 3.3}})); + } + + @Test + void bool() throws IOException { + // [true, false, null] + parse(BoolValue.standard().array(), + "[true, false, null]", + ObjectChunk.chunkWrap(new Object[] {new Boolean[] {true, false, null}})); + } + + @Test + void tuple() throws IOException { + // [[1, 1.1], null, [3, 3.3]] + parse(TupleValue.of(IntValue.standard(), DoubleValue.standard()).array(), + "[[1, 1.1], null, [3, 3.3]]", + ObjectChunk.chunkWrap(new Object[] {new int[] {1, QueryConstants.NULL_INT, 3}}), + ObjectChunk.chunkWrap(new Object[] {new double[] {1.1, QueryConstants.NULL_DOUBLE, 3.3}})); + } + + @Test + void object() throws IOException { + // [{"int": 1, "double": 1.1}, null, {}, {"int": 4, "double": 4.4}] + parse(ObjectValue.builder() + .putFields("int", IntValue.standard()) + .putFields("double", DoubleValue.standard()) + .build() + .array(), + "[{\"int\": 1, \"double\": 1.1}, null, {}, {\"int\": 4, \"double\": 4.4}]", + ObjectChunk + .chunkWrap(new Object[] {new int[] {1, QueryConstants.NULL_INT, QueryConstants.NULL_INT, 4}}), + ObjectChunk.chunkWrap(new Object[] { + new double[] {1.1, QueryConstants.NULL_DOUBLE, QueryConstants.NULL_DOUBLE, 4.4}})); + } + + // missing feature + @Disabled + @Test + void typedObject() throws IOException { + // [ {"type": "int", "value": 42}, {"type": "string", "value": "foo"} ] + parse(TypedObjectValue.builder() + .typeFieldName("type") + .putObjects("int", ObjectValue.standard(Map.of("value", IntValue.standard()))) + .putObjects("string", ObjectValue.standard(Map.of("value", StringValue.standard()))) + .build() + .array(), + "[ {\"type\": \"int\", \"value\": 42}, {\"type\": \"string\", \"value\": \"foo\"} ]", + ObjectChunk.chunkWrap(new Object[] {new String[] {"int", "string"}}), + ObjectChunk + .chunkWrap(new Object[] {new int[] {42, QueryConstants.NULL_INT}}), + ObjectChunk.chunkWrap(new Object[] {new String[] {null, "foo"}})); + } + + @Test + void instant() throws IOException { + // ["2009-02-13T23:31:30.123456789Z", null] + parse(InstantValue.standard().array(), + "[\"2009-02-13T23:31:30.123456789Z\", null]", + ObjectChunk.chunkWrap( + new Object[] {new Instant[] {Instant.parse("2009-02-13T23:31:30.123456789Z"), null}})); + } + + @Test + void instantNumber() throws IOException { + // [1234567, null] + parse(Format.EPOCH_SECONDS.standard(false).array(), + "[1234567, null]", + ObjectChunk.chunkWrap(new Object[] {new Instant[] {Instant.ofEpochSecond(1234567), null}})); + } + + @Test + void tupleSkip() throws IOException { + // [[1, 1], [2, 2.2], [3, "foo"], [4, true], [5, false], [6, {}], [7, []], [8, null], null] + parse(TupleValue.of(IntValue.standard(), SkipValue.lenient()).array(), + "[[1, 1], [2, 2.2], [3, \"foo\"], [4, true], [5, false], [6, {}], [7, []], [8, null], null]", + ObjectChunk.chunkWrap(new Object[] {new int[] {1, 2, 3, 4, 5, 6, 7, 8, QueryConstants.NULL_INT}})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/BigDecimalValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/BigDecimalValueTest.java new file mode 100644 index 00000000000..05db216d2b5 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/BigDecimalValueTest.java @@ -0,0 +1,144 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class BigDecimalValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(BigDecimalValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.ofCustom(BigDecimal.class)); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.ofCustom(BigDecimal.class)); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(BigDecimalValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.ofCustom(BigDecimal.class).arrayType()); + assertThat(provider.stringProcessor().outputTypes()) + .containsExactly(Type.ofCustom(BigDecimal.class).arrayType()); + } + + @Test + void standard() throws IOException { + parse(BigDecimalValue.standard(), "42.42", ObjectChunk.chunkWrap(new BigDecimal[] {new BigDecimal("42.42")})); + } + + @Test + void standardMissing() throws IOException { + parse(BigDecimalValue.standard(), "", ObjectChunk.chunkWrap(new BigDecimal[] {null})); + } + + @Test + void standardNull() throws IOException { + parse(BigDecimalValue.standard(), "null", ObjectChunk.chunkWrap(new BigDecimal[] {null})); + } + + @Test + void customMissing() throws IOException { + parse(BigDecimalValue.builder().onMissing(BigDecimal.valueOf(-1)).build(), "", + ObjectChunk.chunkWrap(new BigDecimal[] {BigDecimal.valueOf(-1)})); + } + + @Test + void customNull() throws IOException { + parse(BigDecimalValue.builder().onNull(BigDecimal.valueOf(-2)).build(), "null", + ObjectChunk.chunkWrap(new BigDecimal[] {BigDecimal.valueOf(-2)})); + } + + @Test + void strict() throws IOException { + parse(BigDecimalValue.strict(), "42.42", ObjectChunk.chunkWrap(new BigDecimal[] {new BigDecimal("42.42")})); + } + + @Test + void strictMissing() { + try { + process(BigDecimalValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(BigDecimalValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void standardString() { + try { + process(BigDecimalValue.standard(), "\"42\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not allowed"); + } + } + + @Test + void standardTrue() { + try { + process(BigDecimalValue.standard(), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(BigDecimalValue.standard(), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardObject() { + try { + process(BigDecimalValue.standard(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(BigDecimalValue.standard(), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void lenientString() throws IOException { + parse(BigDecimalValue.lenient(), List.of("\"42.42\"", "\"43.999\""), + ObjectChunk.chunkWrap(new BigDecimal[] {new BigDecimal("42.42"), new BigDecimal("43.999")})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/BigIntegerValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/BigIntegerValueTest.java new file mode 100644 index 00000000000..f57d1d217b0 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/BigIntegerValueTest.java @@ -0,0 +1,167 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class BigIntegerValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(BigIntegerValue.standard(false)); + assertThat(provider.outputTypes()).containsExactly(Type.ofCustom(BigInteger.class)); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.ofCustom(BigInteger.class)); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(BigIntegerValue.standard(false).array()); + assertThat(provider.outputTypes()).containsExactly(Type.ofCustom(BigInteger.class).arrayType()); + assertThat(provider.stringProcessor().outputTypes()) + .containsExactly(Type.ofCustom(BigInteger.class).arrayType()); + } + + @Test + void standard() throws IOException { + parse(BigIntegerValue.standard(false), "42", ObjectChunk.chunkWrap(new BigInteger[] {BigInteger.valueOf(42)})); + } + + @Test + void standardMissing() throws IOException { + parse(BigIntegerValue.standard(false), "", ObjectChunk.chunkWrap(new BigInteger[] {null})); + } + + @Test + void standardNull() throws IOException { + parse(BigIntegerValue.standard(false), "null", ObjectChunk.chunkWrap(new BigInteger[] {null})); + } + + @Test + void customMissing() throws IOException { + parse(BigIntegerValue.builder().onMissing(BigInteger.valueOf(-1)).build(), "", + ObjectChunk.chunkWrap(new BigInteger[] {BigInteger.valueOf(-1)})); + } + + @Test + void customNull() throws IOException { + parse(BigIntegerValue.builder().onNull(BigInteger.valueOf(-2)).build(), "null", + ObjectChunk.chunkWrap(new BigInteger[] {BigInteger.valueOf(-2)})); + } + + @Test + void strict() throws IOException { + parse(BigIntegerValue.strict(false), "42", ObjectChunk.chunkWrap(new BigInteger[] {BigInteger.valueOf(42)})); + } + + @Test + void strictMissing() { + try { + process(BigIntegerValue.strict(false), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(BigIntegerValue.strict(false), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void standardString() { + try { + process(BigIntegerValue.standard(false), "\"42\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not allowed"); + } + } + + @Test + void standardTrue() { + try { + process(BigIntegerValue.standard(false), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(BigIntegerValue.standard(false), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardDecimal() { + try { + process(BigIntegerValue.standard(false), "42.42"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Decimal not allowed"); + } + } + + @Test + void standardObject() { + try { + process(BigIntegerValue.standard(false), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(BigIntegerValue.standard(false), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void lenientString() throws IOException { + parse(BigIntegerValue.lenient(false), List.of("\"42\"", "\"43\""), + ObjectChunk.chunkWrap(new BigInteger[] {BigInteger.valueOf(42), BigInteger.valueOf(43)})); + } + + @Test + void allowDecimal() throws IOException { + parse(BigIntegerValue.standard(true), List.of("42.42", "43.999"), + ObjectChunk.chunkWrap(new BigInteger[] {BigInteger.valueOf(42), BigInteger.valueOf(43)})); + } + + @Test + void allowDecimalString() throws IOException { + parse(BigIntegerValue.lenient(true), List.of("\"42.42\"", "\"43.999\""), + ObjectChunk.chunkWrap(new BigInteger[] {BigInteger.valueOf(42), BigInteger.valueOf(43)})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/BoolArrayTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/BoolArrayTest.java new file mode 100644 index 00000000000..e5efb746df9 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/BoolArrayTest.java @@ -0,0 +1,30 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static io.deephaven.json.TestHelper.parse; + +public class BoolArrayTest { + + @Test + void standard() throws IOException { + parse(BoolValue.standard().array(), "[true, null, false]", + ObjectChunk.chunkWrap(new Object[] {new Boolean[] {true, null, false}})); + } + + @Test + void standardMissing() throws IOException { + parse(BoolValue.standard().array(), "", ObjectChunk.chunkWrap(new Object[] {null})); + } + + @Test + void standardNull() throws IOException { + parse(BoolValue.standard().array(), "null", ObjectChunk.chunkWrap(new Object[] {null})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/BoolValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/BoolValueTest.java new file mode 100644 index 00000000000..29adc524f5b --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/BoolValueTest.java @@ -0,0 +1,152 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ByteChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.BooleanUtils; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class BoolValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(BoolValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.booleanType().boxedType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.booleanType().boxedType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(BoolValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.booleanType().boxedType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()) + .containsExactly(Type.booleanType().boxedType().arrayType()); + } + + @Test + void standard() throws IOException { + parse(BoolValue.standard(), List.of("true", "false"), ByteChunk + .chunkWrap(new byte[] {BooleanUtils.TRUE_BOOLEAN_AS_BYTE, BooleanUtils.FALSE_BOOLEAN_AS_BYTE})); + } + + @Test + void standardMissing() throws IOException { + parse(BoolValue.standard(), "", ByteChunk.chunkWrap(new byte[] {BooleanUtils.NULL_BOOLEAN_AS_BYTE})); + } + + @Test + void standardNull() throws IOException { + parse(BoolValue.standard(), "null", ByteChunk.chunkWrap(new byte[] {BooleanUtils.NULL_BOOLEAN_AS_BYTE})); + } + + @Test + void customMissing() throws IOException { + parse(BoolValue.builder().onMissing(true).build(), "", + ByteChunk.chunkWrap(new byte[] {BooleanUtils.TRUE_BOOLEAN_AS_BYTE})); + parse(BoolValue.builder().onMissing(false).build(), "", + ByteChunk.chunkWrap(new byte[] {BooleanUtils.FALSE_BOOLEAN_AS_BYTE})); + } + + @Test + void customNull() throws IOException { + parse(BoolValue.builder().onNull(true).build(), "null", + ByteChunk.chunkWrap(new byte[] {BooleanUtils.TRUE_BOOLEAN_AS_BYTE})); + parse(BoolValue.builder().onNull(false).build(), "null", + ByteChunk.chunkWrap(new byte[] {BooleanUtils.FALSE_BOOLEAN_AS_BYTE})); + } + + @Test + void strict() throws IOException { + parse(BoolValue.strict(), List.of("true", "false"), ByteChunk + .chunkWrap(new byte[] {BooleanUtils.TRUE_BOOLEAN_AS_BYTE, BooleanUtils.FALSE_BOOLEAN_AS_BYTE})); + } + + @Test + void strictMissing() { + try { + process(BoolValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(BoolValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void standardString() { + try { + process(BoolValue.standard(), "\"42\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not allowed"); + } + } + + @Test + void standardInt() { + try { + process(BoolValue.standard(), "42"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Number int not expected"); + } + } + + @Test + void standardDecimal() { + try { + process(BoolValue.standard(), "42.0"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Decimal not expected"); + } + } + + @Test + void standardObject() { + try { + process(BoolValue.standard(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(BoolValue.standard(), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void lenientString() throws IOException { + parse(BoolValue.lenient(), List.of("\"true\"", "\"false\"", "\"null\""), + ByteChunk.chunkWrap(new byte[] {BooleanUtils.TRUE_BOOLEAN_AS_BYTE, BooleanUtils.FALSE_BOOLEAN_AS_BYTE, + BooleanUtils.NULL_BOOLEAN_AS_BYTE})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/ByteValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/ByteValueTest.java new file mode 100644 index 00000000000..244e4ac2324 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/ByteValueTest.java @@ -0,0 +1,212 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ByteChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class ByteValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(ByteValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.byteType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.byteType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(ByteValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.byteType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.byteType().arrayType()); + } + + @Test + void standard() throws IOException { + parse(ByteValue.standard(), "42", ByteChunk.chunkWrap(new byte[] {42})); + } + + @Test + void standardMissing() throws IOException { + parse(ByteValue.standard(), "", ByteChunk.chunkWrap(new byte[] {QueryConstants.NULL_BYTE})); + } + + @Test + void standardNull() throws IOException { + parse(ByteValue.standard(), "null", ByteChunk.chunkWrap(new byte[] {QueryConstants.NULL_BYTE})); + } + + @Test + void customMissing() throws IOException { + parse(ByteValue.builder().onMissing((byte) -1).build(), "", ByteChunk.chunkWrap(new byte[] {-1})); + } + + @Test + void customNull() throws IOException { + parse(ByteValue.builder().onNull((byte) -2).build(), "null", ByteChunk.chunkWrap(new byte[] {-2})); + } + + @Test + void strict() throws IOException { + parse(ByteValue.strict(), "42", ByteChunk.chunkWrap(new byte[] {42})); + } + + @Test + void strictMissing() { + try { + process(ByteValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(ByteValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void standardUnderflow() { + try { + process(ByteValue.standard(), "-129"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process current value for ByteValue"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining( + "Numeric value (-129) out of range of Java byte"); + } + } + + @Test + void standardOverflow() { + // Jackson has non-standard byte processing + try { + process(ByteValue.standard(), "256"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process current value for ByteValue"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining( + "Numeric value (256) out of range of Java byte"); + } + } + + @Test + void standardString() { + try { + process(ByteValue.standard(), "\"42\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not allowed"); + } + } + + @Test + void standardTrue() { + try { + process(ByteValue.standard(), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(ByteValue.standard(), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardDecimal() { + try { + process(ByteValue.standard(), "42.0"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Decimal not allowed"); + } + } + + @Test + void standardObject() { + try { + process(ByteValue.standard(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(ByteValue.standard(), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void lenientString() throws IOException { + parse(ByteValue.lenient(), List.of("\"42\"", "\"43\""), ByteChunk.chunkWrap(new byte[] {42, 43})); + } + + @Test + void allowDecimal() throws IOException { + parse(ByteValue.builder() + .allowedTypes(JsonValueTypes.INT, JsonValueTypes.DECIMAL) + .build(), List.of("42.42", "43.999"), ByteChunk.chunkWrap(new byte[] {42, 43})); + } + + @Test + void allowDecimalString() throws IOException { + parse(ByteValue.builder() + .allowedTypes(JsonValueTypes.STRING, JsonValueTypes.INT, JsonValueTypes.DECIMAL) + .build(), + List.of("\"42.42\"", "\"43.999\""), ByteChunk.chunkWrap(new byte[] {42, 43})); + } + + @Test + void decimalStringLimitsNearMinValue() throws IOException { + for (int i = 0; i < 100; ++i) { + parse(ByteValue.builder().allowedTypes(JsonValueTypes.STRING, JsonValueTypes.DECIMAL, JsonValueTypes.INT) + .build(), + List.of(String.format("\"%d.0\"", Byte.MIN_VALUE + i)), + ByteChunk.chunkWrap(new byte[] {(byte) (Byte.MIN_VALUE + i)})); + } + } + + @Test + void decimalStringLimitsNearMaxValue() throws IOException { + for (int i = 0; i < 100; ++i) { + parse(ByteValue.builder().allowedTypes(JsonValueTypes.STRING, JsonValueTypes.DECIMAL, JsonValueTypes.INT) + .build(), + List.of(String.format("\"%d.0\"", Byte.MAX_VALUE - i)), + ByteChunk.chunkWrap(new byte[] {(byte) (Byte.MAX_VALUE - i)})); + } + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/CharValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/CharValueTest.java new file mode 100644 index 00000000000..bd0fea9eda3 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/CharValueTest.java @@ -0,0 +1,158 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.CharChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class CharValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(CharValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.charType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.charType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(CharValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.charType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.charType().arrayType()); + } + + @Test + void standard() throws IOException { + parse(CharValue.standard(), "\"c\"", CharChunk.chunkWrap(new char[] {'c'})); + } + + @Test + void standardMissing() throws IOException { + parse(CharValue.standard(), "", CharChunk.chunkWrap(new char[] {QueryConstants.NULL_CHAR})); + } + + @Test + void standardNull() throws IOException { + parse(CharValue.standard(), "null", CharChunk.chunkWrap(new char[] {QueryConstants.NULL_CHAR})); + } + + @Test + void customMissing() throws IOException { + parse(CharValue.builder().onMissing('m').build(), "", CharChunk.chunkWrap(new char[] {'m'})); + } + + @Test + void customNull() throws IOException { + parse(CharValue.builder().onNull('n').build(), "null", CharChunk.chunkWrap(new char[] {'n'})); + } + + @Test + void strict() throws IOException { + parse(CharValue.strict(), "\"c\"", CharChunk.chunkWrap(new char[] {'c'})); + } + + @Test + void strictMissing() { + try { + process(CharValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(CharValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + + @Test + void standardInt() { + try { + process(CharValue.standard(), "42"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Number int not expected"); + } + } + + @Test + void standardDecimal() { + try { + process(CharValue.standard(), "42.42"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Decimal not expected"); + } + } + + + @Test + void standardTrue() { + try { + process(CharValue.standard(), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(CharValue.standard(), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardObject() { + try { + process(CharValue.standard(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(CharValue.standard(), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void stringTooBig() { + try { + process(CharValue.standard(), "\"ABC\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process current value for CharValue"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining("Expected char to be string of length 1"); + } + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/DoubleArrayTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/DoubleArrayTest.java new file mode 100644 index 00000000000..060b1504261 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/DoubleArrayTest.java @@ -0,0 +1,31 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static io.deephaven.json.TestHelper.parse; + +public class DoubleArrayTest { + + @Test + void standard() throws IOException { + parse(DoubleValue.standard().array(), "[42.1, null, 43.2]", + ObjectChunk.chunkWrap(new Object[] {new double[] {42.1, QueryConstants.NULL_DOUBLE, 43.2}})); + } + + @Test + void standardMissing() throws IOException { + parse(DoubleValue.standard().array(), "", ObjectChunk.chunkWrap(new Object[] {null})); + } + + @Test + void standardNull() throws IOException { + parse(DoubleValue.standard().array(), "null", ObjectChunk.chunkWrap(new Object[] {null})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/DoubleValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/DoubleValueTest.java new file mode 100644 index 00000000000..53092c41f7f --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/DoubleValueTest.java @@ -0,0 +1,135 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.DoubleChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class DoubleValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(DoubleValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.doubleType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.doubleType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(DoubleValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.doubleType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.doubleType().arrayType()); + } + + @Test + void standard() throws IOException { + parse(DoubleValue.standard(), List.of("42", "42.42"), DoubleChunk.chunkWrap(new double[] {42, 42.42})); + } + + @Test + void standardMissing() throws IOException { + parse(DoubleValue.standard(), "", DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE})); + } + + @Test + void standardNull() throws IOException { + parse(DoubleValue.standard(), "null", DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE})); + } + + @Test + void customMissing() throws IOException { + parse(DoubleValue.builder().onMissing(-1.0).build(), "", DoubleChunk.chunkWrap(new double[] {-1})); + } + + @Test + void strict() throws IOException { + parse(DoubleValue.strict(), "42", DoubleChunk.chunkWrap(new double[] {42})); + } + + @Test + void strictMissing() { + try { + process(DoubleValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(DoubleValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void standardString() { + try { + process(DoubleValue.standard(), "\"42\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not allowed"); + } + } + + @Test + void standardTrue() { + try { + process(DoubleValue.standard(), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(DoubleValue.standard(), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardObject() { + try { + process(DoubleValue.standard(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(DoubleValue.standard(), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void lenientString() throws IOException { + parse(DoubleValue.lenient(), List.of("\"42\"", "\"42.42\""), DoubleChunk.chunkWrap(new double[] {42, 42.42})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/FloatValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/FloatValueTest.java new file mode 100644 index 00000000000..f5465dcb2ce --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/FloatValueTest.java @@ -0,0 +1,135 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.FloatChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class FloatValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(FloatValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.floatType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.floatType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(FloatValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.floatType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.floatType().arrayType()); + } + + @Test + void standard() throws IOException { + parse(FloatValue.standard(), List.of("42", "42.42"), FloatChunk.chunkWrap(new float[] {42, 42.42f})); + } + + @Test + void standardMissing() throws IOException { + parse(FloatValue.standard(), "", FloatChunk.chunkWrap(new float[] {QueryConstants.NULL_FLOAT})); + } + + @Test + void standardNull() throws IOException { + parse(FloatValue.standard(), "null", FloatChunk.chunkWrap(new float[] {QueryConstants.NULL_FLOAT})); + } + + @Test + void customMissing() throws IOException { + parse(FloatValue.builder().onMissing(-1.0f).build(), "", FloatChunk.chunkWrap(new float[] {-1})); + } + + @Test + void strict() throws IOException { + parse(FloatValue.strict(), "42", FloatChunk.chunkWrap(new float[] {42})); + } + + @Test + void strictMissing() { + try { + process(FloatValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(FloatValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void standardString() { + try { + process(FloatValue.standard(), "\"42\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not allowed"); + } + } + + @Test + void standardTrue() { + try { + process(FloatValue.standard(), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(FloatValue.standard(), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardObject() { + try { + process(FloatValue.standard(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(FloatValue.standard(), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void lenientString() throws IOException { + parse(FloatValue.lenient(), List.of("\"42\"", "\"42.42\""), FloatChunk.chunkWrap(new float[] {42, 42.42f})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/InstantNumberTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/InstantNumberTest.java new file mode 100644 index 00000000000..73fa3a3128d --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/InstantNumberTest.java @@ -0,0 +1,123 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.LongChunk; +import io.deephaven.json.InstantNumberValue.Format; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static io.deephaven.json.TestHelper.parse; +import static org.assertj.core.api.Assertions.assertThat; + +public class InstantNumberTest { + private static final long WITH_SECONDS = 1703292532000000000L; + private static final long WITH_MILLIS = 1703292532123000000L; + private static final long WITH_MICROS = 1703292532123456000L; + private static final long WITH_NANOS = 1703292532123456789L; + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(Format.EPOCH_SECONDS.standard(false)); + assertThat(provider.outputTypes()).containsExactly(Type.instantType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.instantType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(Format.EPOCH_SECONDS.standard(false).array()); + assertThat(provider.outputTypes()).containsExactly(Type.instantType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.instantType().arrayType()); + } + + @Test + void epochSeconds() throws IOException { + parse(Format.EPOCH_SECONDS.standard(false), "1703292532", LongChunk.chunkWrap(new long[] {WITH_SECONDS})); + } + + @Test + void epochSecondsDecimal() throws IOException { + parse(Format.EPOCH_SECONDS.standard(true), "1703292532.123456789", + LongChunk.chunkWrap(new long[] {WITH_NANOS})); + } + + @Test + void epochSecondsString() throws IOException { + parse(Format.EPOCH_SECONDS.lenient(false), "\"1703292532\"", LongChunk.chunkWrap(new long[] {WITH_SECONDS})); + } + + @Test + void epochSecondsStringDecimal() throws IOException { + parse(Format.EPOCH_SECONDS.lenient(true), "\"1703292532.123456789\"", + LongChunk.chunkWrap(new long[] {WITH_NANOS})); + } + + @Test + void epochMillis() throws IOException { + parse(Format.EPOCH_MILLIS.standard(false), "1703292532123", LongChunk.chunkWrap(new long[] {WITH_MILLIS})); + } + + @Test + void epochMillisDecimal() throws IOException { + parse(Format.EPOCH_MILLIS.standard(true), "1703292532123.456789", LongChunk.chunkWrap(new long[] {WITH_NANOS})); + } + + @Test + void epochMillisString() throws IOException { + parse(Format.EPOCH_MILLIS.lenient(false), "\"1703292532123\"", LongChunk.chunkWrap(new long[] {WITH_MILLIS})); + } + + @Test + void epochMillisStringDecimal() throws IOException { + parse(Format.EPOCH_MILLIS.lenient(true), "\"1703292532123.456789\"", + LongChunk.chunkWrap(new long[] {WITH_NANOS})); + } + + @Test + void epochMicros() throws IOException { + parse(Format.EPOCH_MICROS.standard(false), "1703292532123456", LongChunk.chunkWrap(new long[] {WITH_MICROS})); + } + + @Test + void epochMicrosDecimal() throws IOException { + parse(Format.EPOCH_MICROS.standard(true), "1703292532123456.789", LongChunk.chunkWrap(new long[] {WITH_NANOS})); + } + + @Test + void epochMicrosString() throws IOException { + parse(Format.EPOCH_MICROS.lenient(false), "\"1703292532123456\"", + LongChunk.chunkWrap(new long[] {WITH_MICROS})); + } + + @Test + void epochMicrosStringDecimal() throws IOException { + parse(Format.EPOCH_MICROS.lenient(true), "\"1703292532123456.789\"", + LongChunk.chunkWrap(new long[] {WITH_NANOS})); + } + + @Test + void epochNanos_() throws IOException { + parse(Format.EPOCH_NANOS.standard(false), "1703292532123456789", LongChunk.chunkWrap(new long[] {WITH_NANOS})); + } + + @Test + void epochNanosDecimal() throws IOException { + parse(Format.EPOCH_NANOS.standard(true), "1703292532123456789.0", LongChunk.chunkWrap(new long[] {WITH_NANOS})); + } + + @Test + void epochNanosString() throws IOException { + parse(Format.EPOCH_NANOS.lenient(false), "\"1703292532123456789\"", + LongChunk.chunkWrap(new long[] {WITH_NANOS})); + } + + @Test + void epochNanosStringDecimal() throws IOException { + parse(Format.EPOCH_NANOS.lenient(true), "\"1703292532123456789.0\"", + LongChunk.chunkWrap(new long[] {WITH_NANOS})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/InstantValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/InstantValueTest.java new file mode 100644 index 00000000000..7b85dcf7cdb --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/InstantValueTest.java @@ -0,0 +1,91 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.LongChunk; +import io.deephaven.json.InstantNumberValue.Format; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.Instant; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class InstantValueTest { + + private static final String XYZ_STR = "2009-02-13T23:31:30.123456789"; + private static final long XYZ_NANOS = 1234567890L * 1_000_000_000 + 123456789; + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(InstantValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.instantType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.instantType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(InstantValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.instantType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.instantType().arrayType()); + } + + @Test + void iso8601() throws IOException { + parse(InstantValue.standard(), "\"" + XYZ_STR + "Z\"", LongChunk.chunkWrap(new long[] {XYZ_NANOS})); + } + + @Test + void iso8601WithOffset() throws IOException { + parse(InstantValue.standard(), "\"" + XYZ_STR + "+00:00\"", LongChunk.chunkWrap(new long[] {XYZ_NANOS})); + } + + @Test + void standardNull() throws IOException { + parse(InstantValue.standard(), "null", LongChunk.chunkWrap(new long[] {QueryConstants.NULL_LONG})); + } + + @Test + void standardMissing() throws IOException { + parse(InstantValue.standard(), "", LongChunk.chunkWrap(new long[] {QueryConstants.NULL_LONG})); + } + + @Test + void strictNull() throws IOException { + try { + process(InstantValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void strictMissing() throws IOException { + try { + process(InstantValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void customNull() throws IOException { + parse(InstantValue.builder().onNull(Instant.ofEpochMilli(0)).build(), "null", + LongChunk.chunkWrap(new long[] {0})); + } + + @Test + void customMissing() throws IOException { + parse(InstantValue.builder().onMissing(Instant.ofEpochMilli(0)).build(), "", + LongChunk.chunkWrap(new long[] {0})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/IntArrayTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/IntArrayTest.java new file mode 100644 index 00000000000..58c4b957a32 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/IntArrayTest.java @@ -0,0 +1,213 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class IntArrayTest { + + @Test + void standard() throws IOException { + parse(IntValue.standard().array(), "[42, 43]", ObjectChunk.chunkWrap(new Object[] {new int[] {42, 43}})); + } + + @Test + void standardMissing() throws IOException { + parse(IntValue.standard().array(), "", ObjectChunk.chunkWrap(new Object[] {null})); + } + + @Test + void standardNull() throws IOException { + parse(IntValue.standard().array(), "null", ObjectChunk.chunkWrap(new Object[] {null})); + } + + @Test + void strictMissing() { + try { + process(ArrayValue.strict(IntValue.standard()), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(ArrayValue.strict(IntValue.standard()), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void standardInt() { + try { + process(IntValue.standard().array(), "42"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Number int not expected"); + } + } + + @Test + void standardDecimal() { + try { + process(IntValue.standard().array(), "42.42"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Decimal not expected"); + } + } + + @Test + void standardString() { + try { + process(IntValue.standard().array(), "\"hello\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not expected"); + } + } + + @Test + void standardTrue() { + try { + process(IntValue.standard().array(), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(IntValue.standard().array(), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardObject() { + try { + process(IntValue.standard().array(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void doubleNestedArray() throws IOException { + parse(IntValue.standard().array().array(), "[null, [], [42, 43]]", + ObjectChunk.chunkWrap(new Object[] {new int[][] { + null, + new int[] {}, + new int[] {42, 43}}})); + } + + @Test + void tripleNestedArray() throws IOException { + parse(IntValue.standard().array().array().array(), "[null, [], [null, [], [42, 43]]]", + ObjectChunk.chunkWrap(new Object[] {new int[][][] { + null, + new int[][] {}, + new int[][] { + null, + new int[] {}, + new int[] {42, 43}} + }})); + } + + @Test + void quadNestedArray() throws IOException { + parse(IntValue.standard().array().array().array().array(), "[null, [], [null, [], [null, [], [42, 43]]]]", + ObjectChunk.chunkWrap(new Object[] {new int[][][][] { + null, + new int[][][] {}, + new int[][][] { + null, + new int[][] {}, + new int[][] { + null, + new int[] {}, + new int[] {42, 43}} + }}})); + } + + @Test + void innerStrict() throws IOException { + parse(ArrayValue.standard(IntValue.strict()), List.of("", "null", "[42, 43]"), + ObjectChunk.chunkWrap(new Object[] {null, null, new int[] {42, 43}})); + try { + process(ArrayValue.standard(IntValue.strict()), "[42, null]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed for IntValue"); + } + } + + @Test + void arrayStrict() throws IOException { + parse(ArrayValue.strict(IntValue.standard()), "[42, null]", + ObjectChunk.chunkWrap(new Object[] {new int[] {42, QueryConstants.NULL_INT}})); + try { + process(ArrayValue.strict(IntValue.standard()), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed for ArrayValue"); + } + try { + process(ArrayValue.strict(IntValue.standard()), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed for ArrayValue"); + } + } + + @Test + void doubleNestedArrayStrict() throws IOException { + parse(ArrayValue.strict(ArrayValue.standard(IntValue.standard())), "[null, [], [42, null]]", ObjectChunk + .chunkWrap(new Object[] {new int[][] {null, new int[] {}, new int[] {42, QueryConstants.NULL_INT}}})); + try { + process(ArrayValue.strict(ArrayValue.standard(IntValue.standard())), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed for ArrayValue"); + } + try { + process(ArrayValue.strict(ArrayValue.standard(IntValue.standard())), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed for ArrayValue"); + } + } + + @Test + void doubleNestedInnerArrayStrict() throws IOException { + parse(ArrayValue.standard(ArrayValue.strict(IntValue.standard())), List.of("", "null", "[[], [42, null]]"), + ObjectChunk.chunkWrap(new Object[] {null, null, + new int[][] {new int[] {}, new int[] {42, QueryConstants.NULL_INT}}})); + try { + process(ArrayValue.standard(ArrayValue.strict(IntValue.standard())), "[[], [42, null], null]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed for ArrayValue"); + } + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/IntValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/IntValueTest.java new file mode 100644 index 00000000000..ea915d13ddc --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/IntValueTest.java @@ -0,0 +1,211 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.IntChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class IntValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(IntValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.intType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.intType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(IntValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.intType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.intType().arrayType()); + } + + @Test + void standard() throws IOException { + parse(IntValue.standard(), "42", IntChunk.chunkWrap(new int[] {42})); + } + + @Test + void standardMissing() throws IOException { + parse(IntValue.standard(), "", IntChunk.chunkWrap(new int[] {QueryConstants.NULL_INT})); + } + + @Test + void standardNull() throws IOException { + parse(IntValue.standard(), "null", IntChunk.chunkWrap(new int[] {QueryConstants.NULL_INT})); + } + + @Test + void customMissing() throws IOException { + parse(IntValue.builder().onMissing(-1).build(), "", IntChunk.chunkWrap(new int[] {-1})); + } + + @Test + void customNull() throws IOException { + parse(IntValue.builder().onNull(-2).build(), "null", IntChunk.chunkWrap(new int[] {-2})); + } + + @Test + void strict() throws IOException { + parse(IntValue.strict(), "42", IntChunk.chunkWrap(new int[] {42})); + } + + @Test + void strictMissing() { + try { + process(IntValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(IntValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void standardUnderflow() { + try { + process(IntValue.standard(), Long.toString(Integer.MIN_VALUE - 1L)); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process current value for IntValue"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining( + "Numeric value (-2147483649) out of range of int (-2147483648 - 2147483647)"); + } + } + + @Test + void standardOverflow() { + try { + process(IntValue.standard(), Long.toString(Integer.MAX_VALUE + 1L)); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process current value for IntValue"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining( + "Numeric value (2147483648) out of range of int (-2147483648 - 2147483647)"); + } + } + + @Test + void standardString() { + try { + process(IntValue.standard(), "\"42\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not allowed"); + } + } + + @Test + void standardTrue() { + try { + process(IntValue.standard(), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(IntValue.standard(), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardDecimal() { + try { + process(IntValue.standard(), "42.0"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Decimal not allowed"); + } + } + + @Test + void standardObject() { + try { + process(IntValue.standard(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(IntValue.standard(), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void lenientString() throws IOException { + parse(IntValue.lenient(), List.of("\"42\"", "\"43\""), IntChunk.chunkWrap(new int[] {42, 43})); + } + + @Test + void allowDecimal() throws IOException { + parse(IntValue.builder() + .allowedTypes(JsonValueTypes.INT, JsonValueTypes.DECIMAL) + .build(), List.of("42.42", "43.999"), IntChunk.chunkWrap(new int[] {42, 43})); + } + + @Test + void allowDecimalString() throws IOException { + parse(IntValue.builder() + .allowedTypes(JsonValueTypes.STRING, JsonValueTypes.INT, JsonValueTypes.DECIMAL) + .build(), + List.of("\"42.42\"", "\"43.999\""), IntChunk.chunkWrap(new int[] {42, 43})); + } + + @Test + void decimalStringLimitsNearMinValue() throws IOException { + for (int i = 0; i < 100; ++i) { + parse(IntValue.builder().allowedTypes(JsonValueTypes.STRING, JsonValueTypes.DECIMAL, JsonValueTypes.INT) + .build(), + List.of(String.format("\"%d.0\"", Integer.MIN_VALUE + i)), + IntChunk.chunkWrap(new int[] {Integer.MIN_VALUE + i})); + } + } + + @Test + void decimalStringLimitsNearMaxValue() throws IOException { + for (int i = 0; i < 100; ++i) { + parse(IntValue.builder().allowedTypes(JsonValueTypes.STRING, JsonValueTypes.DECIMAL, JsonValueTypes.INT) + .build(), + List.of(String.format("\"%d.0\"", Integer.MAX_VALUE - i)), + IntChunk.chunkWrap(new int[] {Integer.MAX_VALUE - i})); + } + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/LocalDateValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/LocalDateValueTest.java new file mode 100644 index 00000000000..76a401e039b --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/LocalDateValueTest.java @@ -0,0 +1,85 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.LocalDate; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class LocalDateValueTest { + + private static final String XYZ_STR = "2009-02-13"; + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(LocalDateValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.ofCustom(LocalDate.class)); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.ofCustom(LocalDate.class)); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(LocalDateValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.ofCustom(LocalDate.class).arrayType()); + assertThat(provider.stringProcessor().outputTypes()) + .containsExactly(Type.ofCustom(LocalDate.class).arrayType()); + } + + @Test + void iso8601() throws IOException { + parse(LocalDateValue.standard(), "\"" + XYZ_STR + "\"", + ObjectChunk.chunkWrap(new LocalDate[] {LocalDate.of(2009, 2, 13)})); + } + + @Test + void standardNull() throws IOException { + parse(LocalDateValue.standard(), "null", ObjectChunk.chunkWrap(new LocalDate[] {null})); + } + + @Test + void standardMissing() throws IOException { + parse(LocalDateValue.standard(), "", ObjectChunk.chunkWrap(new LocalDate[] {null})); + } + + @Test + void strictNull() { + try { + process(LocalDateValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void strictMissing() { + try { + process(LocalDateValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void customNull() throws IOException { + parse(LocalDateValue.builder().onNull(LocalDate.ofEpochDay(0)).build(), "null", + ObjectChunk.chunkWrap(new LocalDate[] {LocalDate.ofEpochDay(0)})); + } + + @Test + void customMissing() throws IOException { + parse(LocalDateValue.builder().onMissing(LocalDate.ofEpochDay(0)).build(), "", + ObjectChunk.chunkWrap(new LocalDate[] {LocalDate.ofEpochDay(0)})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/LongArrayTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/LongArrayTest.java new file mode 100644 index 00000000000..6a6bf0075b2 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/LongArrayTest.java @@ -0,0 +1,29 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static io.deephaven.json.TestHelper.parse; + +public class LongArrayTest { + + @Test + void standard() throws IOException { + parse(LongValue.standard().array(), "[42, 43]", ObjectChunk.chunkWrap(new Object[] {new long[] {42, 43}})); + } + + @Test + void standardMissing() throws IOException { + parse(LongValue.standard().array(), "", ObjectChunk.chunkWrap(new Object[] {null})); + } + + @Test + void standardNull() throws IOException { + parse(LongValue.standard().array(), "null", ObjectChunk.chunkWrap(new Object[] {null})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/LongValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/LongValueTest.java new file mode 100644 index 00000000000..b59456d913f --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/LongValueTest.java @@ -0,0 +1,203 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.LongChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class LongValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(LongValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.longType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.longType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(LongValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.longType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.longType().arrayType()); + } + + @Test + void standard() throws IOException { + parse(LongValue.standard(), List.of("42", "43"), LongChunk.chunkWrap(new long[] {42, 43})); + } + + @Test + void standardMissing() throws IOException { + parse(LongValue.standard(), "", LongChunk.chunkWrap(new long[] {QueryConstants.NULL_LONG})); + } + + @Test + void standardNull() throws IOException { + parse(LongValue.standard(), "null", LongChunk.chunkWrap(new long[] {QueryConstants.NULL_LONG})); + } + + @Test + void customMissing() throws IOException { + parse(LongValue.builder().onMissing(-1L).build(), "", LongChunk.chunkWrap(new long[] {-1})); + } + + @Test + void strict() throws IOException { + parse(LongValue.strict(), "42", LongChunk.chunkWrap(new long[] {42})); + } + + @Test + void strictMissing() { + try { + process(LongValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(LongValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void standardUnderflow() { + try { + process(LongValue.standard(), BigInteger.valueOf(Long.MIN_VALUE).subtract(BigInteger.ONE).toString()); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process current value for LongValue"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining( + "Numeric value (-9223372036854775809) out of range of long (-9223372036854775808 - 9223372036854775807)"); + } + } + + @Test + void standardOverflow() { + try { + process(LongValue.standard(), BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE).toString()); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process current value for LongValue"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining( + "Numeric value (9223372036854775808) out of range of long (-9223372036854775808 - 9223372036854775807)"); + } + } + + @Test + void standardString() { + try { + process(LongValue.standard(), "\"42\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not allowed"); + } + } + + @Test + void standardTrue() { + try { + process(LongValue.standard(), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(LongValue.standard(), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardDecimal() { + try { + process(LongValue.standard(), "42.0"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Decimal not allowed"); + } + } + + @Test + void standardObject() { + try { + process(LongValue.standard(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(LongValue.standard(), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void lenientString() throws IOException { + parse(LongValue.lenient(), List.of("\"42\"", "\"43\""), LongChunk.chunkWrap(new long[] {42, 43})); + } + + @Test + void allowDecimal() throws IOException { + parse(LongValue.builder().allowedTypes(JsonValueTypes.INT, JsonValueTypes.DECIMAL).build(), + List.of("42.42", "43.999"), + LongChunk.chunkWrap(new long[] {42, 43})); + } + + @Test + void allowDecimalString() throws IOException { + parse(LongValue.builder().allowedTypes(JsonValueTypes.numberLike()).build(), + List.of("\"42.42\"", "\"43.999\""), LongChunk.chunkWrap(new long[] {42, 43})); + } + + @Test + void decimalStringLimitsNearMinValue() throws IOException { + for (int i = 0; i < 100; ++i) { + parse(LongValue.builder().allowedTypes(JsonValueTypes.numberLike()).build(), + List.of(String.format("\"%d.0\"", Long.MIN_VALUE + i)), + LongChunk.chunkWrap(new long[] {Long.MIN_VALUE + i})); + } + } + + @Test + void decimalStringLimitsNearMaxValue() throws IOException { + for (int i = 0; i < 100; ++i) { + parse(LongValue.builder().allowedTypes(JsonValueTypes.numberLike()).build(), + List.of(String.format("\"%d.0\"", Long.MAX_VALUE - i)), + LongChunk.chunkWrap(new long[] {Long.MAX_VALUE - i})); + } + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/ObjectFieldTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/ObjectFieldTest.java new file mode 100644 index 00000000000..b5b1e93fcfe --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/ObjectFieldTest.java @@ -0,0 +1,94 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.json.ObjectField.RepeatedBehavior; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class ObjectFieldTest { + @Test + void basic() { + final ObjectField field = ObjectField.of("Foo", IntValue.standard()); + assertThat(field.name()).isEqualTo("Foo"); + assertThat(field.options()).isEqualTo(IntValue.standard()); + assertThat(field.aliases()).isEmpty(); + assertThat(field.caseSensitive()).isTrue(); + assertThat(field.repeatedBehavior()).isEqualTo(RepeatedBehavior.ERROR); + } + + @Test + void caseInsensitiveMatch() { + final ObjectField field = ObjectField.builder() + .name("Foo") + .options(IntValue.standard()) + .caseSensitive(false) + .build(); + assertThat(field.name()).isEqualTo("Foo"); + assertThat(field.options()).isEqualTo(IntValue.standard()); + assertThat(field.aliases()).isEmpty(); + assertThat(field.caseSensitive()).isFalse(); + assertThat(field.repeatedBehavior()).isEqualTo(RepeatedBehavior.ERROR); + } + + @Test + void repeatedBehavior() { + final ObjectField field = ObjectField.builder() + .name("Foo") + .options(IntValue.standard()) + .repeatedBehavior(RepeatedBehavior.ERROR) + .build(); + assertThat(field.name()).isEqualTo("Foo"); + assertThat(field.options()).isEqualTo(IntValue.standard()); + assertThat(field.aliases()).isEmpty(); + assertThat(field.caseSensitive()).isTrue(); + assertThat(field.repeatedBehavior()).isEqualTo(RepeatedBehavior.ERROR); + } + + @Test + void alias() { + final ObjectField field = ObjectField.builder() + .name("SomeName") + .options(IntValue.standard()) + .addAliases("someName") + .build(); + assertThat(field.name()).isEqualTo("SomeName"); + assertThat(field.options()).isEqualTo(IntValue.standard()); + assertThat(field.aliases()).containsExactly("someName"); + assertThat(field.caseSensitive()).isTrue(); + assertThat(field.repeatedBehavior()).isEqualTo(RepeatedBehavior.ERROR); + } + + @Test + void badAliasRepeated() { + try { + ObjectField.builder() + .name("SomeName") + .options(IntValue.standard()) + .addAliases("SomeName") + .build(); + failBecauseExceptionWasNotThrown(IllegalArgumentException.class); + } catch (IllegalArgumentException e) { + assertThat(e).hasMessageContaining("name and aliases must be non-overlapping"); + } + } + + @Test + void badAliasCaseInsensitive() { + try { + // this is similar to the alias() test, but we are explicitly marking it as case-insensitive + ObjectField.builder() + .name("SomeName") + .options(IntValue.standard()) + .addAliases("someName") + .caseSensitive(false) + .build(); + failBecauseExceptionWasNotThrown(IllegalArgumentException.class); + } catch (IllegalArgumentException e) { + assertThat(e).hasMessageContaining("name and aliases must be non-overlapping"); + } + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/ObjectKvValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/ObjectKvValueTest.java new file mode 100644 index 00000000000..0a7574b66a8 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/ObjectKvValueTest.java @@ -0,0 +1,75 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static org.assertj.core.api.Assertions.assertThat; + +public class ObjectKvValueTest { + + private static final ObjectEntriesValue STRING_INT_KV = + ObjectEntriesValue.standard(IntValue.standard()); + + private static final ObjectValue NAME_AGE_OBJ = ObjectValue.builder() + .putFields("name", StringValue.standard()) + .putFields("age", IntValue.standard()) + .build(); + + private static final ObjectEntriesValue STRING_OBJ_KV = ObjectEntriesValue.standard(NAME_AGE_OBJ); + + @Test + void kvPrimitiveValue() throws IOException { + parse(STRING_INT_KV, List.of( + "{\"A\": 42, \"B\": null}"), + ObjectChunk.chunkWrap(new Object[] {new String[] {"A", "B"}}), + ObjectChunk.chunkWrap(new Object[] {new int[] {42, QueryConstants.NULL_INT}})); + } + + @Test + void kvObjectValue() throws IOException { + parse(STRING_OBJ_KV, List.of( + "{\"A\": {\"name\": \"Foo\", \"age\": 42}, \"B\": {}, \"C\": null}"), + ObjectChunk.chunkWrap(new Object[] {new String[] {"A", "B", "C"}}), + ObjectChunk.chunkWrap(new Object[] {new String[] {"Foo", null, null}}), + ObjectChunk.chunkWrap(new Object[] {new int[] {42, QueryConstants.NULL_INT, QueryConstants.NULL_INT}})); + } + + @Test + void kvPrimitiveKey() throws IOException { + parse(ObjectEntriesValue.builder().key(IntValue.lenient()).value(SkipValue.lenient()).build(), List.of( + "{\"42\": null, \"43\": null}"), + ObjectChunk.chunkWrap(new Object[] {new int[] {42, 43}})); + } + + @Test + void kvObjectKey() throws IOException { + parse(ObjectEntriesValue.builder().key(InstantValue.standard()).value(SkipValue.lenient()).build(), List.of( + "{\"2009-02-13T23:31:30.123456788Z\": null, \"2009-02-13T23:31:30.123456789Z\": null}"), + ObjectChunk.chunkWrap(new Object[] {new Instant[] { + Instant.parse("2009-02-13T23:31:30.123456788Z"), + Instant.parse("2009-02-13T23:31:30.123456789Z")}})); + } + + @Test + void columnNames() { + assertThat(JacksonProvider.of(STRING_INT_KV).named(Type.stringType()).names()).containsExactly("Key", + "Value"); + } + + @Test + void columnNamesValueIsObject() { + assertThat(JacksonProvider.of(STRING_OBJ_KV).named(Type.stringType()).names()).containsExactly("Key", + "name", "age"); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/ObjectValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/ObjectValueTest.java new file mode 100644 index 00000000000..0cdb6a3ddd5 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/ObjectValueTest.java @@ -0,0 +1,234 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.IntChunk; +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class ObjectValueTest { + + public static final ObjectValue OBJECT_AGE_FIELD = ObjectValue.builder() + .putFields("age", IntValue.standard()) + .build(); + + private static final ObjectValue OBJECT_NAME_AGE_FIELD = ObjectValue.builder() + .putFields("name", StringValue.standard()) + .putFields("age", IntValue.standard()) + .build(); + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(OBJECT_NAME_AGE_FIELD); + assertThat(provider.outputTypes()).containsExactly(Type.stringType(), Type.intType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.stringType(), Type.intType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(OBJECT_NAME_AGE_FIELD.array()); + assertThat(provider.outputTypes()).containsExactly(Type.stringType().arrayType(), Type.intType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.stringType().arrayType(), + Type.intType().arrayType()); + } + + @Test + void ofAge() throws IOException { + parse(OBJECT_AGE_FIELD, List.of( + "", + "null", + "{}", + "{\"age\": 42}", + "{\"name\": \"Devin\", \"age\": 43}"), + IntChunk.chunkWrap( + new int[] {QueryConstants.NULL_INT, QueryConstants.NULL_INT, QueryConstants.NULL_INT, 42, 43})); + } + + @Test + void ofNameAge() throws IOException { + parse(OBJECT_NAME_AGE_FIELD, List.of( + // "", + // "null", + // "{}", + // "{\"age\": 42}", + "{\"name\": \"Devin\", \"age\": 43}"), + ObjectChunk.chunkWrap(new String[] {"Devin"}), + IntChunk.chunkWrap( + new int[] {43})); + } + + @Test + void caseInsensitive() throws IOException { + final ObjectValue options = ObjectValue.builder() + .addFields(ObjectField.builder() + .name("Foo") + .options(IntValue.standard()) + .caseSensitive(false) + .build()) + .build(); + parse(options, List.of("{\"Foo\": 42}", "{\"fOO\": 43}"), + IntChunk.chunkWrap(new int[] {42, 43})); + } + + @Test + void caseSensitive() throws IOException { + final ObjectField f1 = ObjectField.of("Foo", IntValue.standard()); + final ObjectField f2 = ObjectField.of("foo", IntValue.standard()); + final ObjectValue options = ObjectValue.builder() + .addFields(f1) + .addFields(f2) + .build(); + parse(options, List.of("{\"Foo\": 42, \"foo\": 43}"), + IntChunk.chunkWrap(new int[] {42}), + IntChunk.chunkWrap(new int[] {43})); + } + + @Test + void alias() throws IOException { + final ObjectValue options = ObjectValue.builder() + .addFields(ObjectField.builder() + .name("FooBar") + .options(IntValue.standard()) + .addAliases("Foo_Bar") + .build()) + .build(); + parse(options, List.of("{\"FooBar\": 42}", "{\"Foo_Bar\": 43}"), + IntChunk.chunkWrap(new int[] {42, 43})); + } + + @Test + void caseInsensitiveAlias() throws IOException { + final ObjectValue options = ObjectValue.builder() + .addFields(ObjectField.builder() + .name("FooBar") + .options(IntValue.standard()) + .addAliases("Foo_Bar") + .caseSensitive(false) + .build()) + .build(); + parse(options, List.of("{\"fooBar\": 42}", "{\"fOO_BAR\": 43}"), + IntChunk.chunkWrap(new int[] {42, 43})); + } + + @Test + void caseSensitiveFields() { + final ObjectField f1 = ObjectField.of("Foo", IntValue.standard()); + final ObjectField f2 = ObjectField.of("foo", IntValue.standard()); + final ObjectValue options = ObjectValue.builder() + .addFields(f1) + .addFields(f2) + .build(); + assertThat(options.fields()).containsExactly(f1, f2); + } + + @Test + void caseInsensitiveOverlap() { + final ObjectField f1 = ObjectField.of("Foo", IntValue.standard()); + final ObjectField f2 = ObjectField.builder() + .name("foo") + .options(IntValue.standard()) + .caseSensitive(false) + .build(); + try { + ObjectValue.builder() + .addFields(f1) + .addFields(f2) + .build(); + failBecauseExceptionWasNotThrown(IllegalArgumentException.class); + } catch (IllegalArgumentException e) { + assertThat(e).hasMessageContaining("Found overlapping field name 'foo'"); + } + } + + @Test + void objectFields() throws IOException { + // { "prices": [1.1, 2.2, 3.3], "other": [2, 4, 8, 16] } + parse(ObjectValue.builder() + .putFields("prices", DoubleValue.standard().array()) + .putFields("other", LongValue.standard().array()) + .build(), + "{ \"prices\": [1.1, 2.2, 3.3], \"other\": [2, 4, 8, 16] }", + ObjectChunk + .chunkWrap(new Object[] {new double[] {1.1, 2.2, 3.3}}), + ObjectChunk.chunkWrap(new Object[] { + new long[] {2, 4, 8, 16}})); + } + + @Test + void objectFieldsArrayGroup() throws IOException { + // Note: array groups don't cause any difference wrt ObjectProcessor based destructuring + // { "prices": [1.1, 2.2, 3.3], "sizes": [2, 4, 8] } + parse(ObjectValue.builder() + .addFields(ObjectField.builder() + .name("prices") + .options(DoubleValue.standard().array()) + .arrayGroup("prices_and_sizes") + .build()) + .addFields(ObjectField.builder() + .name("sizes") + .options(LongValue.standard().array()) + .arrayGroup("prices_and_sizes") + .build()) + .build(), + "{ \"prices\": [1.1, 2.2, 3.3], \"sizes\": [2, 4, 8] }", + ObjectChunk + .chunkWrap(new Object[] {new double[] {1.1, 2.2, 3.3}}), + ObjectChunk.chunkWrap(new Object[] { + new long[] {2, 4, 8}})); + } + + @Test + void columnNames() { + assertThat(JacksonProvider.of(OBJECT_NAME_AGE_FIELD).named(Type.stringType()).names()) + .containsExactly("name", "age"); + } + + @Test + void columnNamesAlternateName() { + final ObjectValue obj = ObjectValue.builder() + .addFields(ObjectField.builder() + .name("MyName") + .addAliases("name") + .options(StringValue.standard()) + .build()) + .addFields(ObjectField.of("age", IntValue.standard())) + .build(); + assertThat(JacksonProvider.of(obj).named(Type.stringType()).names()).containsExactly("MyName", "age"); + } + + @Test + void columnNamesWithFieldThatIsNotColumnNameCompatible() { + final ObjectValue objPlusOneMinusOneCount = ObjectValue.builder() + .putFields("+1", IntValue.standard()) + .putFields("-1", IntValue.standard()) + .build(); + assertThat(JacksonProvider.of(objPlusOneMinusOneCount).named(Type.stringType()).names()) + .containsExactly("column_1", "column_12"); + } + + @Test + void fieldException() { + try { + process(OBJECT_NAME_AGE_FIELD, "{\"name\": \"Devin\", \"age\": 43.42}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process field 'age'"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining("Decimal not allowed"); + assertThat(e.getCause()).hasNoCause(); + } + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/RepeatedProcessorTests.java b/extensions/json-jackson/src/test/java/io/deephaven/json/RepeatedProcessorTests.java new file mode 100644 index 00000000000..42e640144c9 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/RepeatedProcessorTests.java @@ -0,0 +1,91 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static io.deephaven.json.TestHelper.parse; + +public class RepeatedProcessorTests { + + @Test + void arrayArrayPrimitive() throws IOException { + // [[1.1], null, [], [2.2, 3.3]] + parse(DoubleValue.standard().array().array(), + "[[1.1], null, [], [2.2, 3.3]]", + ObjectChunk.chunkWrap(new Object[] { + new double[][] {new double[] {1.1}, null, new double[0], new double[] {2.2, 3.3}}})); + } + + @Test + void arrayKvPrimitive() throws IOException { + // [{"a": 1.1}, null, {}, {"b": 2.2, "c": 3.3}] + parse(ObjectEntriesValue.builder().key(SkipValue.lenient()).value(DoubleValue.standard()).build().array(), + "[{\"a\": 1.1}, null, {}, {\"b\": 2.2, \"c\": 3.3}]", + ObjectChunk.chunkWrap(new Object[] { + new double[][] {new double[] {1.1}, null, new double[0], new double[] {2.2, 3.3}}})); + } + + @Test + void kvArrayPrimitive() throws IOException { + // {"a": [1.1], "b": null, "c": [], "d": [2.2, 3.3]} + parse(ObjectEntriesValue.standard(DoubleValue.standard().array()), + "{\"a\": [1.1], \"b\": null, \"c\": [], \"d\": [2.2, 3.3]}", + ObjectChunk.chunkWrap(new Object[] { + new String[] {"a", "b", "c", "d"}}), + ObjectChunk.chunkWrap(new Object[] { + new double[][] {new double[] {1.1}, null, new double[0], new double[] {2.2, 3.3}}})); + } + + @Test + void tuple() throws IOException { + // [[[1, 1.1]], null, [], [[2, 2.2], [3, 3.3]]] + parse(TupleValue.of(IntValue.standard(), DoubleValue.standard()).array().array(), + "[[[1, 1.1]], null, [], [[2, 2.2], [3, 3.3]]]", + ObjectChunk.chunkWrap(new Object[] {new int[][] {new int[] {1}, null, new int[0], new int[] {2, 3}}}), + ObjectChunk.chunkWrap(new Object[] { + new double[][] {new double[] {1.1}, null, new double[0], new double[] {2.2, 3.3}}})); + } + + @Test + void object() throws IOException { + // [[{"int": 1, "double": 1.1}], null, [], [{"int": 2}, {"double": 3.3}], [{"int": 4, "double": 4.4}, {"int": 5, + // "double": 5.5}]] + parse(ObjectValue.builder() + .putFields("int", IntValue.standard()) + .putFields("double", DoubleValue.standard()) + .build() + .array() + .array(), + "[[{\"int\": 1, \"double\": 1.1}], null, [], [{\"int\": 2}, {\"double\": 3.3}], [{\"int\": 4, \"double\": 4.4}, {\"int\": 5, \"double\": 5.5}]]", + ObjectChunk.chunkWrap(new Object[] {new int[][] {new int[] {1}, null, new int[0], + new int[] {2, QueryConstants.NULL_INT}, new int[] {4, 5}}}), + ObjectChunk.chunkWrap(new Object[] {new double[][] {new double[] {1.1}, null, new double[0], + new double[] {QueryConstants.NULL_DOUBLE, 3.3}, new double[] {4.4, 5.5}}})); + } + + + @Test + void differentNesting() throws IOException { + // [ { "foo": [ { "bar": 41 }, {} ], "baz": 1.1 }, null, {}, { "foo": [] }, { "foo": [ { "bar": 43 } ], "baz": + // 3.3 }] + parse(ObjectValue.builder() + .putFields("foo", ObjectValue.builder() + .putFields("bar", IntValue.standard()) + .build() + .array()) + .putFields("baz", DoubleValue.standard()) + .build() + .array(), + "[ { \"foo\": [ { \"bar\": 41 }, {} ], \"baz\": 1.1 }, null, {}, { \"foo\": [] }, { \"foo\": [ { \"bar\": 43 } ], \"baz\": 3.3 }]", + ObjectChunk.chunkWrap(new Object[] { + new int[][] {new int[] {41, QueryConstants.NULL_INT}, null, null, new int[0], new int[] {43}}}), + ObjectChunk.chunkWrap(new Object[] {new double[] {1.1, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE, QueryConstants.NULL_DOUBLE, 3.3}})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/ShortValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/ShortValueTest.java new file mode 100644 index 00000000000..85c21576905 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/ShortValueTest.java @@ -0,0 +1,211 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ShortChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class ShortValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(ShortValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.shortType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.shortType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(ShortValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.shortType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.shortType().arrayType()); + } + + @Test + void standard() throws IOException { + parse(ShortValue.standard(), "42", ShortChunk.chunkWrap(new short[] {42})); + } + + @Test + void standardMissing() throws IOException { + parse(ShortValue.standard(), "", ShortChunk.chunkWrap(new short[] {QueryConstants.NULL_SHORT})); + } + + @Test + void standardNull() throws IOException { + parse(ShortValue.standard(), "null", ShortChunk.chunkWrap(new short[] {QueryConstants.NULL_SHORT})); + } + + @Test + void customMissing() throws IOException { + parse(ShortValue.builder().onMissing((short) -1).build(), "", ShortChunk.chunkWrap(new short[] {-1})); + } + + @Test + void customNull() throws IOException { + parse(ShortValue.builder().onNull((short) -2).build(), "null", ShortChunk.chunkWrap(new short[] {-2})); + } + + @Test + void strict() throws IOException { + parse(ShortValue.strict(), "42", ShortChunk.chunkWrap(new short[] {42})); + } + + @Test + void strictMissing() { + try { + process(ShortValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(ShortValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + @Test + void standardUnderflow() { + try { + process(ShortValue.standard(), Integer.toString(Short.MIN_VALUE - 1)); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process current value for ShortValue"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining( + "Numeric value (-32769) out of range of Java short"); + } + } + + @Test + void standardOverflow() { + try { + process(ShortValue.standard(), Integer.toString(Short.MAX_VALUE + 1)); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process current value for ShortValue"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining( + "Numeric value (32768) out of range of Java short"); + } + } + + @Test + void standardString() { + try { + process(ShortValue.standard(), "\"42\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not allowed"); + } + } + + @Test + void standardTrue() { + try { + process(ShortValue.standard(), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(ShortValue.standard(), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardDecimal() { + try { + process(ShortValue.standard(), "42.0"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Decimal not allowed"); + } + } + + @Test + void standardObject() { + try { + process(ShortValue.standard(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(ShortValue.standard(), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void lenientString() throws IOException { + parse(ShortValue.lenient(), List.of("\"42\"", "\"43\""), ShortChunk.chunkWrap(new short[] {42, 43})); + } + + @Test + void allowDecimal() throws IOException { + parse(ShortValue.builder() + .allowedTypes(JsonValueTypes.INT, JsonValueTypes.DECIMAL) + .build(), List.of("42.42", "43.999"), ShortChunk.chunkWrap(new short[] {42, 43})); + } + + @Test + void allowDecimalString() throws IOException { + parse(ShortValue.builder() + .allowedTypes(JsonValueTypes.STRING, JsonValueTypes.INT, JsonValueTypes.DECIMAL) + .build(), + List.of("\"42.42\"", "\"43.999\""), ShortChunk.chunkWrap(new short[] {42, 43})); + } + + @Test + void decimalStringLimitsNearMinValue() throws IOException { + for (int i = 0; i < 100; ++i) { + parse(ShortValue.builder().allowedTypes(JsonValueTypes.STRING, JsonValueTypes.DECIMAL, JsonValueTypes.INT) + .build(), + List.of(String.format("\"%d.0\"", Short.MIN_VALUE + i)), + ShortChunk.chunkWrap(new short[] {(short) (Short.MIN_VALUE + i)})); + } + } + + @Test + void decimalStringLimitsNearMaxValue() throws IOException { + for (int i = 0; i < 100; ++i) { + parse(ShortValue.builder().allowedTypes(JsonValueTypes.STRING, JsonValueTypes.DECIMAL, JsonValueTypes.INT) + .build(), + List.of(String.format("\"%d.0\"", Short.MAX_VALUE - i)), + ShortChunk.chunkWrap(new short[] {(short) (Short.MAX_VALUE - i)})); + } + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/StringValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/StringValueTest.java new file mode 100644 index 00000000000..5344622fce2 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/StringValueTest.java @@ -0,0 +1,154 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class StringValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(StringValue.standard()); + assertThat(provider.outputTypes()).containsExactly(Type.stringType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.stringType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(StringValue.standard().array()); + assertThat(provider.outputTypes()).containsExactly(Type.stringType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.stringType().arrayType()); + } + + @Test + void standard() throws IOException { + parse(StringValue.standard(), "\"foo\"", ObjectChunk.chunkWrap(new String[] {"foo"})); + } + + @Test + void standardMissing() throws IOException { + parse(StringValue.standard(), "", ObjectChunk.chunkWrap(new String[] {null})); + } + + @Test + void standardNull() throws IOException { + parse(StringValue.standard(), "null", ObjectChunk.chunkWrap(new String[] {null})); + } + + + @Test + void customMissing() throws IOException { + parse(StringValue.builder().onMissing("").build(), "", + ObjectChunk.chunkWrap(new String[] {""})); + } + + @Test + void customNull() throws IOException { + parse(StringValue.builder().onNull("").build(), "null", ObjectChunk.chunkWrap(new String[] {""})); + } + + @Test + void strict() throws IOException { + parse(StringValue.strict(), "\"foo\"", ObjectChunk.chunkWrap(new String[] {"foo"})); + } + + @Test + void strictMissing() { + try { + process(StringValue.strict(), ""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Missing not allowed"); + } + } + + @Test + void strictNull() { + try { + process(StringValue.strict(), "null"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Null not allowed"); + } + } + + + @Test + void standardInt() { + try { + process(StringValue.standard(), "42"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Number int not allowed"); + } + } + + @Test + void standardDecimal() { + try { + process(StringValue.standard(), "42.42"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Decimal not allowed"); + } + } + + + @Test + void standardTrue() { + try { + process(StringValue.standard(), "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not allowed"); + } + } + + @Test + void standardFalse() { + try { + process(StringValue.standard(), "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not allowed"); + } + } + + @Test + void standardObject() { + try { + process(StringValue.standard(), "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Object not expected"); + } + } + + @Test + void standardArray() { + try { + process(StringValue.standard(), "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + @Test + void lenientString() throws IOException { + parse(StringValue.lenient(), List.of("42", "42.42", "true", "false"), + ObjectChunk.chunkWrap(new String[] {"42", "42.42", "true", "false"})); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/TestHelper.java b/extensions/json-jackson/src/test/java/io/deephaven/json/TestHelper.java new file mode 100644 index 00000000000..8c750018fef --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/TestHelper.java @@ -0,0 +1,99 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.ChunkType; +import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.WritableObjectChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.util.hashing.ChunkEquals; +import io.deephaven.chunk.util.hashing.ObjectChunkDeepEquals; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.processor.ObjectProcessor; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHelper { + public static void parse(Value options, String json, Chunk... expected) throws IOException { + parse(options, List.of(json), expected); + } + + public static void parse(Value options, List jsonRows, Chunk... expectedCols) throws IOException { + parse(JacksonProvider.of(options).stringProcessor(), jsonRows, expectedCols); + } + + public static void parse(ObjectProcessor processor, List rows, Chunk... expectedCols) + throws IOException { + final List> out = process(processor, rows); + try { + assertThat(out.size()).isEqualTo(expectedCols.length); + assertThat(out.stream().map(Chunk::getChunkType).collect(Collectors.toList())) + .isEqualTo(Stream.of(expectedCols).map(Chunk::getChunkType).collect(Collectors.toList())); + for (int i = 0; i < expectedCols.length; ++i) { + check(out.get(i), expectedCols[i]); + } + } finally { + for (WritableChunk wc : out) { + wc.close(); + } + } + } + + public static List> process(Value options, String jsonRows) throws IOException { + return process(JacksonProvider.of(options).stringProcessor(), List.of(jsonRows)); + } + + + public static List> process(ObjectProcessor processor, List rows) + throws IOException { + final List> out = processor + .outputTypes() + .stream() + .map(ObjectProcessor::chunkType) + .map(x -> x.makeWritableChunk(rows.size())) + .collect(Collectors.toList()); + try { + for (WritableChunk wc : out) { + wc.setSize(0); + } + try (final WritableObjectChunk in = WritableObjectChunk.makeWritableChunk(rows.size())) { + int i = 0; + for (T input : rows) { + in.set(i, input); + ++i; + } + try { + processor.processAll(in, out); + } catch (UncheckedIOException e) { + throw e.getCause(); + } + return out; + } + } catch (IOException | RuntimeException e) { + for (WritableChunk wc : out) { + wc.close(); + } + throw e; + } + } + + static void check(Chunk actual, Chunk expected) { + assertThat(actual.getChunkType()).isEqualTo(expected.getChunkType()); + assertThat(actual.size()).isEqualTo(expected.size()); + assertThat(getChunkEquals(actual).equalReduce(actual, expected)).isTrue(); + } + + private static ChunkEquals getChunkEquals(Chunk actual) { + return actual.getChunkType() == ChunkType.Object + ? ObjectChunkDeepEquals.INSTANCE + : ChunkEquals.makeEqual(actual.getChunkType()); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/TupleValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/TupleValueTest.java new file mode 100644 index 00000000000..36bb667a63c --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/TupleValueTest.java @@ -0,0 +1,73 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.IntChunk; +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class TupleValueTest { + + private static final TupleValue STRING_INT_TUPLE = + TupleValue.of(StringValue.standard(), IntValue.standard()); + + private static final TupleValue STRING_SKIPINT_TUPLE = + TupleValue.of(StringValue.standard(), IntValue.standard().skip()); + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(STRING_INT_TUPLE); + assertThat(provider.outputTypes()).containsExactly(Type.stringType(), Type.intType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.stringType(), Type.intType()); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(STRING_INT_TUPLE.array()); + assertThat(provider.outputTypes()).containsExactly(Type.stringType().arrayType(), Type.intType().arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.stringType().arrayType(), + Type.intType().arrayType()); + } + + @Test + void stringIntTuple() throws IOException { + parse(STRING_INT_TUPLE, List.of( + "[\"foo\", 42]", + "[\"bar\", 43]"), + ObjectChunk.chunkWrap(new String[] {"foo", "bar"}), + IntChunk.chunkWrap(new int[] {42, 43})); + + } + + @Test + void stringSkipIntTuple() throws IOException { + parse(STRING_SKIPINT_TUPLE, List.of( + "[\"foo\", 42]", + "[\"bar\", 43]"), + ObjectChunk.chunkWrap(new String[] {"foo", "bar"})); + } + + @Test + void indexException() { + try { + process(STRING_INT_TUPLE, "[\"foo\", 43.43]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Unable to process tuple ix 1"); + assertThat(e).hasCauseInstanceOf(IOException.class); + assertThat(e.getCause()).hasMessageContaining("Decimal not allowed"); + assertThat(e.getCause()).hasNoCause(); + } + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/TypedObjectValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/TypedObjectValueTest.java new file mode 100644 index 00000000000..c0fc482f452 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/TypedObjectValueTest.java @@ -0,0 +1,287 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.chunk.DoubleChunk; +import io.deephaven.chunk.IntChunk; +import io.deephaven.chunk.LongChunk; +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.jackson.JacksonProvider; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; + +import static io.deephaven.json.TestHelper.parse; +import static io.deephaven.json.TestHelper.process; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.failBecauseExceptionWasNotThrown; + +public class TypedObjectValueTest { + + private static final ObjectValue SYMBOL_INFO = ObjectValue.builder() + .putFields("name", StringValue.strict()) + .putFields("id", LongValue.strict()) + .build(); + + private static final ObjectValue QUOTE_OBJECT = ObjectValue.builder() + .putFields("symbol", SYMBOL_INFO) + .putFields("quote", ObjectValue.builder() + .putFields("bid", DoubleValue.standard()) + .putFields("ask", DoubleValue.standard()) + .build()) + .build(); + + private static final ObjectValue TRADE_OBJECT = ObjectValue.builder() + .putFields("symbol", SYMBOL_INFO) + .putFields("price", DoubleValue.standard()) + .putFields("size", DoubleValue.standard()) + .build(); + + private static final TypedObjectValue QUOTE_OR_TRADE_OBJECT = + TypedObjectValue.builder(new LinkedHashMap<>() { + { + put("quote", QUOTE_OBJECT); + put("trade", TRADE_OBJECT); + } + }) + .typeFieldName("type") + .onNull("") + .onMissing("") + .build(); + + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(QUOTE_OR_TRADE_OBJECT); + assertThat(provider.outputTypes()).containsExactly(Type.stringType(), Type.stringType(), Type.longType(), + Type.doubleType(), Type.doubleType(), Type.doubleType(), Type.doubleType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.stringType(), Type.stringType(), + Type.longType(), Type.doubleType(), Type.doubleType(), Type.doubleType(), Type.doubleType()); + } + + @Test + void typeDiscriminationQuoteTrade() throws IOException { + parse(QUOTE_OR_TRADE_OBJECT, List.of( + "{\"type\": \"quote\", \"symbol\": {\"name\": \"foo\", \"id\": 42}, \"quote\":{\"bid\": 1.01, \"ask\": 1.05}}", + "{\"type\": \"trade\", \"symbol\": {\"name\": \"bar\", \"id\": 43}, \"price\": 42.42, \"size\": 123}", + "{\"type\": \"other\"}", + "{\"type\": \"other_mimic_trade\", \"symbol\": {\"name\": \"bar\", \"id\": 43}, \"price\": 42.42, \"size\": 123}"), + ObjectChunk.chunkWrap(new String[] {"quote", "trade", "other", "other_mimic_trade"}), // type + ObjectChunk.chunkWrap(new String[] {"foo", "bar", null, null}), // symbol/symbol + LongChunk.chunkWrap(new long[] {42, 43, QueryConstants.NULL_LONG, QueryConstants.NULL_LONG}), // symbol/symbol_id + DoubleChunk.chunkWrap(new double[] {1.01, QueryConstants.NULL_DOUBLE, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // quote: quote/bid + DoubleChunk.chunkWrap(new double[] {1.05, QueryConstants.NULL_DOUBLE, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // quote: quote/ask + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE, 42.42, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // trade: price + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE, 123, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE})); // trade: size + } + + @Test + void standardNull() throws IOException { + parse(QUOTE_OR_TRADE_OBJECT, "null", + ObjectChunk.chunkWrap(new String[] {""}), // type + ObjectChunk.chunkWrap(new String[] {null}), // symbol/symbol + LongChunk.chunkWrap(new long[] {QueryConstants.NULL_LONG}), // symbol/symbol_id + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE}), // quote: quote/bid + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE}), // quote: quote/ask + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE}), // trade: price + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE})); // trade: size + } + + @Test + void standardMissing() throws IOException { + parse(QUOTE_OR_TRADE_OBJECT, "", + ObjectChunk.chunkWrap(new String[] {""}), // type + ObjectChunk.chunkWrap(new String[] {null}), // symbol/symbol + LongChunk.chunkWrap(new long[] {QueryConstants.NULL_LONG}), // symbol/symbol_id + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE}), // quote: quote/bid + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE}), // quote: quote/ask + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE}), // trade: price + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE})); // trade: size + } + + @Test + void unexpectedFirstField() { + try { + process(QUOTE_OR_TRADE_OBJECT, "{\"foo\": 42}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Expected the first field to be 'type', is 'foo'"); + } + } + + @Test + void standardEmptyObject() { + try { + process(QUOTE_OR_TRADE_OBJECT, "{}"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Expected a non-empty object"); + } + } + + @Test + void standardInt() { + try { + process(QUOTE_OR_TRADE_OBJECT, "42"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Number int not expected"); + } + } + + @Test + void standardDecimal() { + try { + process(QUOTE_OR_TRADE_OBJECT, "42.42"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Decimal not expected"); + } + } + + @Test + void standardString() { + try { + process(QUOTE_OR_TRADE_OBJECT, "\"hello\""); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("String not expected"); + } + } + + @Test + void standardTrue() { + try { + process(QUOTE_OR_TRADE_OBJECT, "true"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardFalse() { + try { + process(QUOTE_OR_TRADE_OBJECT, "false"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Bool not expected"); + } + } + + @Test + void standardArray() { + try { + process(QUOTE_OR_TRADE_OBJECT, "[]"); + failBecauseExceptionWasNotThrown(IOException.class); + } catch (IOException e) { + assertThat(e).hasMessageContaining("Array not expected"); + } + } + + // Disabled; this may be a feature we want to spec out in the future + @Test + @Disabled + void intAsDiscriminator() throws IOException { + // Note: need to pass + final TypedObjectValue tov = TypedObjectValue.builder(new LinkedHashMap<>() { + { + put(1, QUOTE_OBJECT); + put(2, TRADE_OBJECT); + } + }).typeField(ObjectField.of("id", IntValue.standard())).build(); + + parse(tov, List.of( + "{\"id\": 1, \"symbol\": {\"name\": \"foo\", \"id\": 42}, \"quote\":{\"bid\": 1.01, \"ask\": 1.05}}", + "{\"id\": 2, \"symbol\": {\"name\": \"bar\", \"id\": 43}, \"price\": 42.42, \"size\": 123}", + "{\"id\": 3}", + "{\"id\": 4, \"symbol\": {\"name\": \"bar\", \"id\": 43}, \"price\": 42.42, \"size\": 123}"), + IntChunk.chunkWrap(new int[] {1, 2, 3, 4}), // id + ObjectChunk.chunkWrap(new String[] {"foo", "bar", null, null}), // symbol/symbol + LongChunk.chunkWrap(new long[] {42, 43, QueryConstants.NULL_LONG, QueryConstants.NULL_LONG}), // symbol/symbol_id + DoubleChunk.chunkWrap(new double[] {1.01, QueryConstants.NULL_DOUBLE, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // quote: quote/bid + DoubleChunk.chunkWrap(new double[] {1.05, QueryConstants.NULL_DOUBLE, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // quote: quote/ask + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE, 42.42, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // trade: price + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE, 123, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE})); // trade: size + + } + + @Test + void intStringAsDiscriminator() throws IOException { + final TypedObjectValue tov = TypedObjectValue.builder(new LinkedHashMap<>() { + { + put("1", QUOTE_OBJECT); + put("2", TRADE_OBJECT); + } + }).typeFieldName("id").build(); + + parse(tov, List.of( + "{\"id\": 1, \"symbol\": {\"name\": \"foo\", \"id\": 42}, \"quote\":{\"bid\": 1.01, \"ask\": 1.05}}", + "{\"id\": 2, \"symbol\": {\"name\": \"bar\", \"id\": 43}, \"price\": 42.42, \"size\": 123}", + "{\"id\": 3}", + "{\"id\": 4, \"symbol\": {\"name\": \"bar\", \"id\": 43}, \"price\": 42.42, \"size\": 123}"), + ObjectChunk.chunkWrap(new String[] {"1", "2", "3", "4"}), // id + ObjectChunk.chunkWrap(new String[] {"foo", "bar", null, null}), // symbol/symbol + LongChunk.chunkWrap(new long[] {42, 43, QueryConstants.NULL_LONG, QueryConstants.NULL_LONG}), // symbol/symbol_id + DoubleChunk.chunkWrap(new double[] {1.01, QueryConstants.NULL_DOUBLE, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // quote: quote/bid + DoubleChunk.chunkWrap(new double[] {1.05, QueryConstants.NULL_DOUBLE, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // quote: quote/ask + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE, 42.42, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // trade: price + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE, 123, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE})); // trade: size + } + + @Test + void caseSensitiveDiscriminator() throws IOException { + final TypedObjectValue tov = TypedObjectValue.builder(new LinkedHashMap<>() { + { + put("q", QUOTE_OBJECT); + put("Q", TRADE_OBJECT); + } + }).typeFieldName("type").build(); + parse(tov, List.of( + "{\"type\": \"q\", \"symbol\": {\"name\": \"foo\", \"id\": 42}, \"quote\":{\"bid\": 1.01, \"ask\": 1.05}}", + "{\"type\": \"Q\", \"symbol\": {\"name\": \"bar\", \"id\": 43}, \"price\": 42.42, \"size\": 123}", + "{\"type\": \"other\"}", + "{\"type\": \"other_mimic_trade\", \"symbol\": {\"name\": \"bar\", \"id\": 43}, \"price\": 42.42, \"size\": 123}"), + ObjectChunk.chunkWrap(new String[] {"q", "Q", "other", "other_mimic_trade"}), // type + ObjectChunk.chunkWrap(new String[] {"foo", "bar", null, null}), // symbol/symbol + LongChunk.chunkWrap(new long[] {42, 43, QueryConstants.NULL_LONG, QueryConstants.NULL_LONG}), // symbol/symbol_id + DoubleChunk.chunkWrap(new double[] {1.01, QueryConstants.NULL_DOUBLE, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // quote: quote/bid + DoubleChunk.chunkWrap(new double[] {1.05, QueryConstants.NULL_DOUBLE, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // quote: quote/ask + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE, 42.42, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE}), // trade: price + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE, 123, QueryConstants.NULL_DOUBLE, + QueryConstants.NULL_DOUBLE})); // trade: size + } + + @Test + void columnNames() { + assertThat(JacksonProvider.of(QUOTE_OR_TRADE_OBJECT).named(Type.stringType()).names()).containsExactly( + "type", + "symbol_name", + "symbol_id", + "quote_quote_bid", + "quote_quote_ask", + "trade_price", + "trade_size"); + } +} diff --git a/extensions/json-jackson/src/test/java/io/deephaven/json/jackson/JacksonAnyValueTest.java b/extensions/json-jackson/src/test/java/io/deephaven/json/jackson/JacksonAnyValueTest.java new file mode 100644 index 00000000000..27aa2859fa3 --- /dev/null +++ b/extensions/json-jackson/src/test/java/io/deephaven/json/jackson/JacksonAnyValueTest.java @@ -0,0 +1,125 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json.jackson; + +import com.fasterxml.jackson.core.TreeNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DoubleNode; +import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.MissingNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import io.deephaven.chunk.DoubleChunk; +import io.deephaven.chunk.IntChunk; +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.json.AnyValue; +import io.deephaven.json.CharValue; +import io.deephaven.json.DoubleValue; +import io.deephaven.json.IntValue; +import io.deephaven.json.ObjectValue; +import io.deephaven.json.TestHelper; +import io.deephaven.json.TupleValue; +import io.deephaven.qst.type.Type; +import io.deephaven.util.QueryConstants; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JacksonAnyValueTest { + + @Test + void provider() { + final JacksonProvider provider = JacksonProvider.of(AnyValue.of()); + assertThat(provider.outputTypes()).containsExactly(Type.ofCustom(TreeNode.class)); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.ofCustom(TreeNode.class)); + } + + @Test + void arrayProvider() { + final JacksonProvider provider = JacksonProvider.of(AnyValue.of().array()); + assertThat(provider.outputTypes()).containsExactly(Type.ofCustom(TreeNode.class).arrayType()); + assertThat(provider.stringProcessor().outputTypes()).containsExactly(Type.ofCustom(TreeNode.class).arrayType()); + } + + @Test + void anyMissing() throws IOException { + checkAny("", MissingNode.getInstance()); + } + + @Test + void anyNull() throws IOException { + checkAny("null", NullNode.getInstance()); + } + + @Test + void anyTrue() throws IOException { + checkAny("true", BooleanNode.getTrue()); + } + + @Test + void anyFalse() throws IOException { + checkAny("false", BooleanNode.getFalse()); + } + + @Test + void anyNumberInt() throws IOException { + checkAny("42", IntNode.valueOf(42)); + } + + @Test + void anyNumberFloat() throws IOException { + checkAny("42.42", DoubleNode.valueOf(42.42)); + } + + @Test + void anyNumberString() throws IOException { + checkAny("\"my string\"", TextNode.valueOf("my string")); + } + + @Test + void anyObject() throws IOException { + checkAny("{}", new ObjectNode(null, Map.of())); + checkAny("{\"foo\": 42}", new ObjectNode(null, Map.of("foo", IntNode.valueOf(42)))); + } + + @Test + void anyArray() throws IOException { + checkAny("[]", new ArrayNode(null, List.of())); + checkAny("[42]", new ArrayNode(null, List.of(IntNode.valueOf(42)))); + } + + @Test + void anyInTuple() throws IOException { + final TupleValue options = TupleValue.of(IntValue.standard(), AnyValue.of(), DoubleValue.standard()); + TestHelper.parse(options, List.of("", "[42, {\"zip\": 43}, 44.44]"), + IntChunk.chunkWrap(new int[] {QueryConstants.NULL_INT, 42}), + ObjectChunk.chunkWrap(new TreeNode[] {MissingNode.getInstance(), + new ObjectNode(null, Map.of("zip", IntNode.valueOf(43)))}), + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE, 44.44})); + } + + @Test + void anyInObject() throws IOException { + final ObjectValue options = ObjectValue.builder() + .putFields("foo", IntValue.standard()) + .putFields("bar", AnyValue.of()) + .putFields("baz", DoubleValue.standard()) + .build(); + TestHelper.parse(options, List.of("", "{\"foo\": 42, \"bar\": {\"zip\": 43}, \"baz\": 44.44}"), + IntChunk.chunkWrap(new int[] {QueryConstants.NULL_INT, 42}), + ObjectChunk.chunkWrap(new TreeNode[] {MissingNode.getInstance(), + new ObjectNode(null, Map.of("zip", IntNode.valueOf(43)))}), + DoubleChunk.chunkWrap(new double[] {QueryConstants.NULL_DOUBLE, 44.44})); + } + + private static void checkAny(String json, TreeNode expected) throws IOException { + TestHelper.parse(AnyValue.of(), json, ObjectChunk.chunkWrap(new TreeNode[] {expected})); + } +} diff --git a/extensions/json-jackson/src/test/resources/io/deephaven/json/test-array-objects.json b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-array-objects.json new file mode 100644 index 00000000000..7ae79bc9505 --- /dev/null +++ b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-array-objects.json @@ -0,0 +1,10 @@ +[ + { + "name": "foo", + "age": 42 + }, + { + "name": "bar", + "age": 43 + } +] diff --git a/extensions/json-jackson/src/test/resources/io/deephaven/json/test-compact-objects.json.txt b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-compact-objects.json.txt new file mode 100644 index 00000000000..2033322908e --- /dev/null +++ b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-compact-objects.json.txt @@ -0,0 +1 @@ +{"name":"foo","age":42}{"name":"bar","age":43} \ No newline at end of file diff --git a/extensions/json-jackson/src/test/resources/io/deephaven/json/test-double-nested-array-objects.json b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-double-nested-array-objects.json new file mode 100644 index 00000000000..1b5138db604 --- /dev/null +++ b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-double-nested-array-objects.json @@ -0,0 +1,14 @@ +{ + "outer": { + "inner": [ + { + "name": "foo", + "age": 42 + }, + { + "name": "bar", + "age": 43 + } + ] + } +} \ No newline at end of file diff --git a/extensions/json-jackson/src/test/resources/io/deephaven/json/test-nested-array-objects.json b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-nested-array-objects.json new file mode 100644 index 00000000000..8b2b3745ba9 --- /dev/null +++ b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-nested-array-objects.json @@ -0,0 +1,12 @@ +{ + "data": [ + { + "name": "foo", + "age": 42 + }, + { + "name": "bar", + "age": 43 + } + ] +} \ No newline at end of file diff --git a/extensions/json-jackson/src/test/resources/io/deephaven/json/test-newline-objects.json.txt b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-newline-objects.json.txt new file mode 100644 index 00000000000..6ea285be438 --- /dev/null +++ b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-newline-objects.json.txt @@ -0,0 +1,8 @@ +{ + "name": "foo", + "age": 42 +} +{ + "name": "bar", + "age": 43 +} diff --git a/extensions/json-jackson/src/test/resources/io/deephaven/json/test-single-object.json b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-single-object.json new file mode 100644 index 00000000000..cabe16dfbc3 --- /dev/null +++ b/extensions/json-jackson/src/test/resources/io/deephaven/json/test-single-object.json @@ -0,0 +1,4 @@ +{ + "name": "foo", + "age": 42 +} diff --git a/extensions/json/build.gradle b/extensions/json/build.gradle new file mode 100644 index 00000000000..2eb199e160e --- /dev/null +++ b/extensions/json/build.gradle @@ -0,0 +1,17 @@ +plugins { + id 'java-library' + id 'io.deephaven.project.register' +} + +dependencies { + Classpaths.inheritImmutables(project) + compileOnly 'com.google.code.findbugs:jsr305:3.0.2' + + Classpaths.inheritJUnitPlatform(project) + Classpaths.inheritAssertJ(project) + testImplementation 'org.junit.jupiter:junit-jupiter' +} + +test { + useJUnitPlatform() +} diff --git a/extensions/json/gradle.properties b/extensions/json/gradle.properties new file mode 100644 index 00000000000..c186bbfdde1 --- /dev/null +++ b/extensions/json/gradle.properties @@ -0,0 +1 @@ +io.deephaven.project.ProjectType=JAVA_PUBLIC diff --git a/extensions/json/src/main/java/io/deephaven/json/AnyValue.java b/extensions/json/src/main/java/io/deephaven/json/AnyValue.java new file mode 100644 index 00000000000..b1a0440cc1d --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/AnyValue.java @@ -0,0 +1,47 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.SingletonStyle; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value as an implementation-specific object. + */ +@Immutable +@SingletonStyle +public abstract class AnyValue extends Value { + + /** + * Allows missing and accepts {@link JsonValueTypes#all()}. + * + * @return the any options + */ + public static AnyValue of() { + return ImmutableAnyValue.of(); + } + + /** + * Always {@link JsonValueTypes#all()}. + */ + @Override + public final Set allowedTypes() { + return JsonValueTypes.all(); + } + + /** + * Always {@code true}. + */ + @Override + public final boolean allowMissing() { + return true; + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/ArrayValue.java b/extensions/json/src/main/java/io/deephaven/json/ArrayValue.java new file mode 100644 index 00000000000..096c519e111 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/ArrayValue.java @@ -0,0 +1,80 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * A "typed array", where all the elements in the {@link JsonValueTypes#ARRAY} have the same {@link #element()} type. + * + *

+ * For example, the JSON value {@code [1, 42, 43, 13]} might be modelled as + * {@code ArrayOptions.standard(IntOptions.standard())}. + */ +@Immutable +@BuildableStyle +public abstract class ArrayValue extends ValueRestrictedUniverseBase { + + public static Builder builder() { + return ImmutableArrayValue.builder(); + } + + /** + * The standard array options. Allows missing and accepts {@link JsonValueTypes#arrayOrNull()}. + * + * @param element the element type + * @return the standard array options + */ + public static ArrayValue standard(Value element) { + return builder().element(element).build(); + } + + /** + * The strict array options. Disallows missing and accepts {@link JsonValueTypes#array()}. + * + * @param element the element type + * @return the strict array options + */ + public static ArrayValue strict(Value element) { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.array()) + .element(element) + .build(); + } + + /** + * The type for the elements of the array. + */ + public abstract Value element(); + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#arrayOrNull()}. By default is + * {@link JsonValueTypes#arrayOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.arrayOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.arrayOrNull(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends Value.Builder { + + Builder element(Value options); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/BigDecimalValue.java b/extensions/json/src/main/java/io/deephaven/json/BigDecimalValue.java new file mode 100644 index 00000000000..3454df81169 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/BigDecimalValue.java @@ -0,0 +1,77 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.math.BigDecimal; +import java.util.Set; + +/** + * Processes a JSON value as a {@link BigDecimal}. + */ +@Immutable +@BuildableStyle +public abstract class BigDecimalValue extends ValueSingleValueBase { + + public static Builder builder() { + return ImmutableBigDecimalValue.builder(); + } + + /** + * The lenient {@link BigDecimal} options. Allows missing and accepts {@link JsonValueTypes#numberLike()}. + * + * @return the lenient BigDecimal options + */ + public static BigDecimalValue lenient() { + return builder().allowedTypes(JsonValueTypes.numberLike()).build(); + } + + /** + * The standard {@link BigDecimal} options. Allows missing and accepts {@link JsonValueTypes#numberOrNull()}. + * + * @return the standard BigDecimal options + */ + public static BigDecimalValue standard() { + return builder().build(); + } + + /** + * The strict {@link BigDecimal} options. Disallows missing and accepts {@link JsonValueTypes#number()}. + * + * @return the strict BigDecimal options + */ + public static BigDecimalValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.number()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#numberLike()}. By default is + * {@link JsonValueTypes#numberOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.numberOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.numberLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends ValueSingleValueBase.Builder { + + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/BigIntegerValue.java b/extensions/json/src/main/java/io/deephaven/json/BigIntegerValue.java new file mode 100644 index 00000000000..5b87ec26a27 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/BigIntegerValue.java @@ -0,0 +1,83 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.math.BigInteger; +import java.util.Set; + +/** + * Processes a JSON value as a {@link BigInteger}. + */ +@Immutable +@BuildableStyle +public abstract class BigIntegerValue extends ValueSingleValueBase { + public static Builder builder() { + return ImmutableBigIntegerValue.builder(); + } + + /** + * The lenient {@link BigInteger} options. Allows missing. If {@code allowDecimal}, accepts + * {@link JsonValueTypes#numberLike()}, otherwise accepts {@link JsonValueTypes#intLike()}. + * + * @return the lenient BigInteger options + */ + public static BigIntegerValue lenient(boolean allowDecimal) { + return builder() + .allowedTypes(allowDecimal ? JsonValueTypes.numberLike() : JsonValueTypes.intLike()) + .build(); + } + + /** + * The standard {@link BigInteger} options. Allows missing. If {@code allowDecimal}, accepts + * {@link JsonValueTypes#numberOrNull()}, otherwise accepts {@link JsonValueTypes#intOrNull()}. + * + * @return the standard BigInteger options + */ + public static BigIntegerValue standard(boolean allowDecimal) { + return builder() + .allowedTypes(allowDecimal ? JsonValueTypes.numberOrNull() : JsonValueTypes.intOrNull()) + .build(); + } + + /** + * The strict {@link BigInteger} options. Allows missing. If {@code allowDecimal}, accepts + * {@link JsonValueTypes#number()}, otherwise accepts {@link JsonValueTypes#int_()}. + * + * @return the strict BigInteger options + */ + public static BigIntegerValue strict(boolean allowDecimal) { + return builder() + .allowMissing(false) + .allowedTypes(allowDecimal ? JsonValueTypes.number() : JsonValueTypes.int_()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#numberLike()}. By default is + * {@link JsonValueTypes#intOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.intOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.numberLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends ValueSingleValueBase.Builder { + + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/BoolValue.java b/extensions/json/src/main/java/io/deephaven/json/BoolValue.java new file mode 100644 index 00000000000..2b263349616 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/BoolValue.java @@ -0,0 +1,81 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value as a {@code boolean}. + */ +@Immutable +@BuildableStyle +public abstract class BoolValue extends ValueSingleValueBase { + + public static Builder builder() { + return ImmutableBoolValue.builder(); + } + + /** + * The lenient bool options. Allows missing and accepts {@link JsonValueTypes#boolLike()}. + * + * @return the lenient bool options + */ + public static BoolValue lenient() { + return builder() + .allowedTypes(JsonValueTypes.boolLike()) + .build(); + } + + /** + * The standard bool options. Allows missing and accepts {@link JsonValueTypes#boolOrNull()}. + * + * @return the standard bool options + */ + public static BoolValue standard() { + return builder().build(); + } + + /** + * The strict bool options. Disallows missing and accepts {@link JsonValueTypes#bool()}. + * + * @return the strict bool options + */ + public static BoolValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.bool()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#boolLike()}. By default is + * {@link JsonValueTypes#boolOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.boolOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.boolLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends BuilderSpecial { + + Builder onNull(boolean onNull); + + Builder onMissing(boolean onMissing); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/ByteValue.java b/extensions/json/src/main/java/io/deephaven/json/ByteValue.java new file mode 100644 index 00000000000..75b7be59e78 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/ByteValue.java @@ -0,0 +1,81 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value as a {@code byte}. + */ +@Immutable +@BuildableStyle +public abstract class ByteValue extends ValueSingleValueBase { + + public static Builder builder() { + return ImmutableByteValue.builder(); + } + + /** + * The lenient byte options. Allows missing and accepts {@link JsonValueTypes#intLike()}. + * + * @return the lenient byte options + */ + public static ByteValue lenient() { + return builder() + .allowedTypes(JsonValueTypes.intLike()) + .build(); + } + + /** + * The standard byte options. Allows missing and accepts {@link JsonValueTypes#intOrNull()}. + * + * @return the standard byte options + */ + public static ByteValue standard() { + return builder().build(); + } + + /** + * The strict byte options. Disallows missing and accepts {@link JsonValueTypes#int_()}. + * + * @return the strict byte options + */ + public static ByteValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.int_()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#numberLike()}. By default is + * {@link JsonValueTypes#intOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.intOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.numberLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends BuilderSpecial { + + Builder onNull(byte onNull); + + Builder onMissing(byte onMissing); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/CharValue.java b/extensions/json/src/main/java/io/deephaven/json/CharValue.java new file mode 100644 index 00000000000..85c7a60fbda --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/CharValue.java @@ -0,0 +1,71 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value as a {@code char}. + */ +@Immutable +@BuildableStyle +public abstract class CharValue extends ValueSingleValueBase { + + public static Builder builder() { + return ImmutableCharValue.builder(); + } + + + /** + * The standard char options. Allows missing and accepts {@link JsonValueTypes#stringOrNull()}. + * + * @return the standard char options + */ + public static CharValue standard() { + return builder().build(); + } + + /** + * The strict char options. Disallows missing and accepts {@link JsonValueTypes#string()}. + * + * @return the strict char options + */ + public static CharValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.string()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#stringOrNull()}. By default is + * {@link JsonValueTypes#stringOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.stringOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.stringOrNull(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends BuilderSpecial { + + Builder onNull(char onNull); + + Builder onMissing(char onMissing); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/DoubleValue.java b/extensions/json/src/main/java/io/deephaven/json/DoubleValue.java new file mode 100644 index 00000000000..9c3e0a0f635 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/DoubleValue.java @@ -0,0 +1,82 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value as a {@code double}. + */ +@Immutable +@BuildableStyle +public abstract class DoubleValue extends ValueSingleValueBase { + + + public static Builder builder() { + return ImmutableDoubleValue.builder(); + } + + /** + * The lenient double options. Allows missing and accepts {@link JsonValueTypes#numberLike()}. + * + * @return the lenient double options + */ + public static DoubleValue lenient() { + return builder() + .allowedTypes(JsonValueTypes.numberLike()) + .build(); + } + + /** + * The standard double options. Allows missing and accepts {@link JsonValueTypes#numberOrNull()}. + * + * @return the standard double options + */ + public static DoubleValue standard() { + return builder().build(); + } + + /** + * The strict double options. Disallows missing and accepts {@link JsonValueTypes#number()}. + * + * @return the strict double options + */ + public static DoubleValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.number()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#numberLike()}. By default is + * {@link JsonValueTypes#numberOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.numberOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.numberLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends BuilderSpecial { + + Builder onNull(double onNull); + + Builder onMissing(double onMissing); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/FloatValue.java b/extensions/json/src/main/java/io/deephaven/json/FloatValue.java new file mode 100644 index 00000000000..49b145fca30 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/FloatValue.java @@ -0,0 +1,80 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value as a {@code float}. + */ +@Immutable +@BuildableStyle +public abstract class FloatValue extends ValueSingleValueBase { + + public static Builder builder() { + return ImmutableFloatValue.builder(); + } + + /** + * The lenient float options. Allows missing and accepts {@link JsonValueTypes#numberLike()}. + * + * @return the lenient float options + */ + public static FloatValue lenient() { + return builder() + .allowedTypes(JsonValueTypes.numberLike()) + .build(); + } + + /** + * The standard float options. Allows missing and accepts {@link JsonValueTypes#numberOrNull()}. + * + * @return the standard float options + */ + public static FloatValue standard() { + return builder().build(); + } + + /** + * The strict float options. Disallows missing and accepts {@link JsonValueTypes#number()}. + * + * @return the strict float options + */ + public static FloatValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.number()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#numberLike()}. By default is + * {@link JsonValueTypes#numberOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.numberOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.numberLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends BuilderSpecial { + Builder onNull(float onNull); + + Builder onMissing(float onMissing); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/InstantNumberValue.java b/extensions/json/src/main/java/io/deephaven/json/InstantNumberValue.java new file mode 100644 index 00000000000..6e56d3806a1 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/InstantNumberValue.java @@ -0,0 +1,117 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.time.Instant; +import java.util.Set; + +/** + * Processes a JSON number as an {@link Instant}. + */ +@Immutable +@BuildableStyle +public abstract class InstantNumberValue extends ValueSingleValueBase { + + public enum Format { + /** + * Seconds from the epoch of 1970-01-01T00:00:00Z. + */ + EPOCH_SECONDS, + + /** + * Milliseconds from the epoch of 1970-01-01T00:00:00Z. + */ + EPOCH_MILLIS, + + /** + * Microseconds from the epoch of 1970-01-01T00:00:00Z. + */ + EPOCH_MICROS, + + /** + * Nanoseconds from the epoch of 1970-01-01T00:00:00Z. + */ + EPOCH_NANOS; + + /** + * The lenient {@link Instant} number options. Allows missing. If {@code allowDecimal}, accepts + * {@link JsonValueTypes#numberLike()}, otherwise accepts {@link JsonValueTypes#intLike()}. + * + * @param allowDecimal if decimals should be allowed + * @return the lenient Instant number options + */ + public InstantNumberValue lenient(boolean allowDecimal) { + return builder() + .format(this) + .allowedTypes(allowDecimal ? JsonValueTypes.numberLike() : JsonValueTypes.intLike()) + .build(); + } + + /** + * The standard {@link Instant} number options. Allows missing. If {@code allowDecimal}, accepts + * {@link JsonValueTypes#numberOrNull()}, otherwise accepts {@link JsonValueTypes#intOrNull()}. + * + * @param allowDecimal if decimals should be allowed + * @return the standard Instant number options + */ + public InstantNumberValue standard(boolean allowDecimal) { + return builder() + .format(this) + .allowedTypes(allowDecimal ? JsonValueTypes.numberOrNull() : JsonValueTypes.intOrNull()) + .build(); + } + + /** + * The strict {@link Instant} number options. Disallows missing. If {@code allowDecimal}, accepts + * {@link JsonValueTypes#number()}, otherwise accepts {@link JsonValueTypes#int_()}. + * + * @param allowDecimal if decimals should be allowed + * @return the lenient Instant number options + */ + public InstantNumberValue strict(boolean allowDecimal) { + return builder() + .format(this) + .allowMissing(false) + .allowedTypes(allowDecimal ? JsonValueTypes.number() : JsonValueTypes.int_()) + .build(); + } + } + + public static Builder builder() { + return ImmutableInstantNumberValue.builder(); + } + + /** + * The format to use. + */ + public abstract Format format(); + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#numberLike()}. By default is + * {@link JsonValueTypes#intOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.intOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.numberLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends ValueSingleValueBase.Builder { + Builder format(Format format); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/InstantValue.java b/extensions/json/src/main/java/io/deephaven/json/InstantValue.java new file mode 100644 index 00000000000..19a303a93d2 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/InstantValue.java @@ -0,0 +1,91 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.lang.Runtime.Version; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.Set; + +/** + * Processes a JSON string as an {@link Instant}. + */ +@Immutable +@BuildableStyle +public abstract class InstantValue extends ValueSingleValueBase { + + private static final Version VERSION_12 = Version.parse("12"); + + public static Builder builder() { + return ImmutableInstantValue.builder(); + } + + /** + * The standard {@link Instant} options. Allows missing and accepts {@link JsonValueTypes#stringOrNull()}. + * + * @return the standard Instant options + */ + public static InstantValue standard() { + return builder().build(); + } + + /** + * The strict {@link Instant} options. Disallows missing and accepts {@link JsonValueTypes#string()}. + * + * @return the strict Instant options + */ + public static InstantValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.string()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#stringOrNull()}. By default is + * {@link JsonValueTypes#stringOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.stringOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.stringOrNull(); + } + + /** + * The date-time formatter to use for {@link DateTimeFormatter#parse(CharSequence) parsing}. The parsed result must + * support extracting {@link java.time.temporal.ChronoField#INSTANT_SECONDS INSTANT_SECONDS} and + * {@link java.time.temporal.ChronoField#NANO_OF_SECOND NANO_OF_SECOND} fields. Defaults to + * {@link DateTimeFormatter#ISO_INSTANT} for java versions 12+, and {@link DateTimeFormatter#ISO_OFFSET_DATE_TIME} + * otherwise. These defaults will parse offsets, converting to UTC as necessary. + * + * @return the date-time formatter + */ + @Default + public DateTimeFormatter dateTimeFormatter() { + // ISO_INSTANT became more versatile in 12+ (handling the parsing of offsets), and is likely more efficient, so + // we should choose to use it when we can. + return Runtime.version().compareTo(VERSION_12) >= 0 + ? DateTimeFormatter.ISO_INSTANT + : DateTimeFormatter.ISO_OFFSET_DATE_TIME; + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends ValueSingleValueBase.Builder { + + Builder dateTimeFormatter(DateTimeFormatter formatter); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/IntValue.java b/extensions/json/src/main/java/io/deephaven/json/IntValue.java new file mode 100644 index 00000000000..67988a87b74 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/IntValue.java @@ -0,0 +1,81 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value as an {@code int}. + */ +@Immutable +@BuildableStyle +public abstract class IntValue extends ValueSingleValueBase { + + public static Builder builder() { + return ImmutableIntValue.builder(); + } + + /** + * The lenient int options. Allows missing and accepts {@link JsonValueTypes#intLike()}. + * + * @return the lenient int options + */ + public static IntValue lenient() { + return builder() + .allowedTypes(JsonValueTypes.intLike()) + .build(); + } + + /** + * The standard int options. Allows missing and accepts {@link JsonValueTypes#intOrNull()}. + * + * @return the standard int options + */ + public static IntValue standard() { + return builder().build(); + } + + /** + * The strict int options. Disallows missing and accepts {@link JsonValueTypes#int_()}. + * + * @return the strict int options + */ + public static IntValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.int_()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#numberLike()}. By default is + * {@link JsonValueTypes#intOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.intOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.numberLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends BuilderSpecial { + + Builder onNull(int onNull); + + Builder onMissing(int onMissing); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/JsonValueTypes.java b/extensions/json/src/main/java/io/deephaven/json/JsonValueTypes.java new file mode 100644 index 00000000000..1527910a627 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/JsonValueTypes.java @@ -0,0 +1,218 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +/** + * The JSON value types. + */ +public enum JsonValueTypes { + /** + * A JSON object type. + */ + OBJECT, + + /** + * A JSON array type. + */ + ARRAY, + + /** + * A JSON string type. + */ + STRING, + + /** + * A JSON number type without a decimal. + */ + INT, + + /** + * A JSON number type with a decimal. + */ + DECIMAL, + + /** + * The JSON literal 'true' or 'false' type. + */ + BOOL, + + /** + * The JSON literal 'null' type. + */ + NULL; + + /** + * An unmodifiable set of all {@link JsonValueTypes}. + */ + public static Set all() { + return ALL; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#INT}. + */ + public static Set int_() { + return INT_SET; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#INT}, {@link JsonValueTypes#NULL}. + */ + public static Set intOrNull() { + return INT_OR_NULL; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#INT}, {@link JsonValueTypes#STRING}, {@link JsonValueTypes#NULL} + */ + public static Set intLike() { + return INT_LIKE; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#INT}, {@link JsonValueTypes#DECIMAL}. + */ + public static Set number() { + return NUMBER; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#INT}, {@link JsonValueTypes#DECIMAL}, {@link JsonValueTypes#NULL}. + */ + public static Set numberOrNull() { + return NUMBER_OR_NULL; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#INT}, {@link JsonValueTypes#DECIMAL}, {@link JsonValueTypes#STRING}, + * {@link JsonValueTypes#NULL}. + */ + public static Set numberLike() { + return NUMBER_LIKE; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#STRING}. + */ + public static Set string() { + return STRING_SET; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#STRING}, {@link JsonValueTypes#NULL}. + */ + public static Set stringOrNull() { + return STRING_OR_NULL; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#STRING}, {@link JsonValueTypes#INT}, {@link JsonValueTypes#DECIMAL}, + * {@link JsonValueTypes#BOOL}, {@link JsonValueTypes#NULL}. + */ + public static Set stringLike() { + return STRING_LIKE; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#BOOL}. + */ + public static Set bool() { + return BOOL_SET; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#BOOL}, {@link JsonValueTypes#NULL}. + */ + public static Set boolOrNull() { + return BOOL_OR_NULL; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#BOOL}, {@link JsonValueTypes#STRING}, {@link JsonValueTypes#NULL}. + */ + public static Set boolLike() { + return BOOL_LIKE; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#OBJECT}. + */ + public static Set object() { + return OBJECT_SET; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#OBJECT}, {@link JsonValueTypes#NULL}. + */ + public static Set objectOrNull() { + return OBJECT_OR_NULL; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#ARRAY}. + */ + public static Set array() { + return ARRAY_SET; + } + + /** + * An unmodifiable set of {@link JsonValueTypes#ARRAY}, {@link JsonValueTypes#NULL}. + */ + public static Set arrayOrNull() { + return ARRAY_OR_NULL; + } + + static void checkAllowedTypeInvariants(Set allowedTypes) { + if (allowedTypes.isEmpty()) { + throw new IllegalArgumentException("allowedTypes is empty"); + } + if (allowedTypes.size() == 1 && allowedTypes.contains(JsonValueTypes.NULL)) { + throw new IllegalArgumentException("allowedTypes is only accepting NULL"); + } + if (allowedTypes.contains(JsonValueTypes.DECIMAL) && !allowedTypes.contains(JsonValueTypes.INT)) { + throw new IllegalArgumentException("allowedTypes is accepting DECIMAL but not INT"); + } + } + + private static final Set ALL = Collections.unmodifiableSet(EnumSet.allOf(JsonValueTypes.class)); + + private static final Set INT_SET = Collections.unmodifiableSet(EnumSet.of(INT)); + + private static final Set INT_OR_NULL = Collections.unmodifiableSet(EnumSet.of(INT, NULL)); + + private static final Set INT_LIKE = Collections.unmodifiableSet(EnumSet.of(INT, STRING, NULL)); + + private static final Set NUMBER = Collections.unmodifiableSet(EnumSet.of(INT, DECIMAL)); + + private static final Set NUMBER_OR_NULL = + Collections.unmodifiableSet(EnumSet.of(INT, DECIMAL, NULL)); + + private static final Set NUMBER_LIKE = + Collections.unmodifiableSet(EnumSet.of(INT, DECIMAL, STRING, NULL)); + + private static final Set STRING_SET = Collections.unmodifiableSet(EnumSet.of(STRING)); + + private static final Set STRING_OR_NULL = Collections.unmodifiableSet(EnumSet.of(STRING, NULL)); + + private static final Set STRING_LIKE = + Collections.unmodifiableSet(EnumSet.of(STRING, INT, DECIMAL, BOOL, NULL)); + + private static final Set BOOL_SET = Collections.unmodifiableSet(EnumSet.of(BOOL)); + + private static final Set BOOL_OR_NULL = Collections.unmodifiableSet(EnumSet.of(BOOL, NULL)); + + private static final Set BOOL_LIKE = Collections.unmodifiableSet(EnumSet.of(BOOL, STRING, NULL)); + + private static final Set OBJECT_SET = Collections.unmodifiableSet(EnumSet.of(OBJECT)); + + private static final Set OBJECT_OR_NULL = Collections.unmodifiableSet(EnumSet.of(OBJECT, NULL)); + + private static final Set ARRAY_SET = Collections.unmodifiableSet(EnumSet.of(ARRAY)); + + private static final Set ARRAY_OR_NULL = Collections.unmodifiableSet(EnumSet.of(ARRAY, NULL)); +} diff --git a/extensions/json/src/main/java/io/deephaven/json/LocalDateValue.java b/extensions/json/src/main/java/io/deephaven/json/LocalDateValue.java new file mode 100644 index 00000000000..d30d7fb58d6 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/LocalDateValue.java @@ -0,0 +1,72 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoField; +import java.util.Set; + +/** + * Processes a JSON string as an {@link LocalDate}. + */ +@Immutable +@BuildableStyle +public abstract class LocalDateValue extends ValueSingleValueBase { + public static Builder builder() { + return ImmutableLocalDateValue.builder(); + } + + public static LocalDateValue standard() { + return builder().build(); + } + + public static LocalDateValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.string()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#stringOrNull()}. By default is + * {@link JsonValueTypes#stringOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.stringOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.stringOrNull(); + } + + /** + * The date-time formatter to use for {@link DateTimeFormatter#parse(CharSequence) parsing}. The parsed result must + * support extracting an {@link ChronoField#EPOCH_DAY EPOCH_DAY} field. Defaults to + * {@link DateTimeFormatter#ISO_LOCAL_DATE}. + * + * @return the date-time formatter + */ + @Default + public DateTimeFormatter dateTimeFormatter() { + return DateTimeFormatter.ISO_LOCAL_DATE; + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends ValueSingleValueBase.Builder { + + Builder dateTimeFormatter(DateTimeFormatter formatter); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/LongValue.java b/extensions/json/src/main/java/io/deephaven/json/LongValue.java new file mode 100644 index 00000000000..132903d6c4b --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/LongValue.java @@ -0,0 +1,81 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value as a {@code long}. + */ +@Immutable +@BuildableStyle +public abstract class LongValue extends ValueSingleValueBase { + + public static Builder builder() { + return ImmutableLongValue.builder(); + } + + /** + * The lenient long options. Allows missing and accepts {@link JsonValueTypes#intLike()}. + * + * @return the lenient long options + */ + public static LongValue lenient() { + return builder() + .allowedTypes(JsonValueTypes.intLike()) + .build(); + } + + /** + * The standard long options. Allows missing and accepts {@link JsonValueTypes#intOrNull()}. + * + * @return the standard long options + */ + public static LongValue standard() { + return builder().build(); + } + + /** + * The strict long options. Disallows missing and accepts {@link JsonValueTypes#int_()}. + * + * @return the strict long options + */ + public static LongValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.int_()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#numberLike()}. By default is + * {@link JsonValueTypes#intOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.intOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.numberLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends BuilderSpecial { + + Builder onNull(long onNull); + + Builder onMissing(long onMissing); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/ObjectEntriesValue.java b/extensions/json/src/main/java/io/deephaven/json/ObjectEntriesValue.java new file mode 100644 index 00000000000..34ac3687d03 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/ObjectEntriesValue.java @@ -0,0 +1,94 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Check; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Represents a JSON object of variable size with a given key and value type. For example, when a JSON object structure + * represents a list of key-value entries (as opposed to a set of separately typed fields): + * + * 

+ * {
+ *   "foo": 1,
+ *   "bar": 42,
+ *   "baz": 3,
+ *   ...
+ *   "xyz": 100
+ * }
+ *
+ */ +@Immutable +@BuildableStyle +public abstract class ObjectEntriesValue extends ValueRestrictedUniverseBase { + + public static Builder builder() { + return ImmutableObjectEntriesValue.builder(); + } + + public static ObjectEntriesValue standard(Value value) { + return builder().value(value).build(); + } + + public static ObjectEntriesValue strict(Value value) { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.object()) + .value(value) + .build(); + } + + /** + * The key options which must minimally support {@link JsonValueTypes#STRING}. By default is + * {@link StringValue#standard()}. + */ + @Default + public Value key() { + return StringValue.standard(); + } + + /** + * The value options. + */ + public abstract Value value(); + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#objectOrNull()}. By default is + * {@link JsonValueTypes#objectOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.objectOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.objectOrNull(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends Value.Builder { + + Builder key(Value key); + + Builder value(Value value); + } + + @Check + final void checkKey() { + if (!key().allowedTypes().contains(JsonValueTypes.STRING)) { + throw new IllegalArgumentException("key argument must support STRING"); + } + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/ObjectField.java b/extensions/json/src/main/java/io/deephaven/json/ObjectField.java new file mode 100644 index 00000000000..538f6e731b0 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/ObjectField.java @@ -0,0 +1,154 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Check; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; + +@Immutable +@BuildableStyle +public abstract class ObjectField { + + public static Builder builder() { + return ImmutableObjectField.builder(); + } + + /** + * Creates a field with a single {@code name}. Equivalent to {@code builder().name(name).options(options).build()}. + * + * @param name the name + * @param options the options + * @return the field options + */ + public static ObjectField of(String name, Value options) { + return builder().name(name).options(options).build(); + } + + /** + * The canonical field name. + */ + public abstract String name(); + + /** + * The value options. + */ + public abstract Value options(); + + /** + * The field name aliases. + */ + public abstract Set aliases(); + + /** + * If the field name and aliases should be compared using case-sensitive equality. By default is {@code true}. + */ + @Default + public boolean caseSensitive() { + return true; + } + + /** + * The behavior when a repeated field is encountered. By default is {@link RepeatedBehavior#ERROR}. + */ + @Default + public RepeatedBehavior repeatedBehavior() { + return RepeatedBehavior.ERROR; + } + + /** + * The array group for {@code this} field. This is useful in scenarios where {@code this} field's array is + * guaranteed to have the same cardinality as one or more other array fields. For example, in the following snippet, + * we might model "prices" and "quantities" as having the same array group: + * + * 
+     * {
+     *   "prices": [1.1, 2.2, 3.3],
+     *   "quantities": [9, 5, 42]
+     * }
+     *
+ */ + public abstract Optional arrayGroup(); + + /** + * The behavior when a repeated field is encountered in a JSON object. For example, as in: + * + * 
+     * {
+     *   "foo": 1,
+     *   "foo": 2
+     * }
+     *
+ */ + public enum RepeatedBehavior { + /** + * Throws an error if a repeated field is encountered + */ + ERROR, + + /** + * Uses the first field of a given name, ignores the rest + */ + USE_FIRST, + + // /** + // * Uses the last field of a given name, ignores the rest. Not currently supported. + // */ + // USE_LAST + } + + public interface Builder { + Builder name(String name); + + Builder options(Value options); + + Builder addAliases(String element); + + Builder addAliases(String... elements); + + Builder addAllAliases(Iterable elements); + + Builder repeatedBehavior(RepeatedBehavior repeatedBehavior); + + Builder caseSensitive(boolean caseSensitive); + + Builder arrayGroup(Object arrayGroup); + + ObjectField build(); + } + + @Check + final void checkNonOverlapping() { + if (caseSensitive()) { + if (aliases().contains(name())) { + throw new IllegalArgumentException( + String.format("name and aliases must be non-overlapping, found '%s' overlaps", name())); + } + } else { + final Set set = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + set.add(name()); + for (String alias : aliases()) { + if (!set.add(alias)) { + throw new IllegalArgumentException( + String.format("name and aliases must be non-overlapping, found '%s' overlaps", alias)); + } + } + } + } + + @Check + final void checkArrayGroup() { + if (arrayGroup().isEmpty()) { + return; + } + if (!(options() instanceof ArrayValue)) { + throw new IllegalArgumentException("arrayGroup is only valid with ArrayOptions"); + } + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/ObjectValue.java b/extensions/json/src/main/java/io/deephaven/json/ObjectValue.java new file mode 100644 index 00000000000..5c0eb3ae670 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/ObjectValue.java @@ -0,0 +1,164 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import io.deephaven.json.ObjectField.RepeatedBehavior; +import org.immutables.value.Value.Check; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; + +/** + * Processes a JSON object as set of named fields. + * + *

+ * For example, the following might be modelled as an object with a String "name" field, an int "age" field, and a + * double "height" field: + * + * 

+ * {
+ *     "name": "Foo",
+ *     "age": 42,
+ *     "height": 6.5
+ * }
+ *
+ */ +@Immutable +@BuildableStyle +public abstract class ObjectValue extends ValueRestrictedUniverseBase { + + public static Builder builder() { + return ImmutableObjectValue.builder(); + } + + /** + * The lenient object options. Allows missing, accepts {@link JsonValueTypes#objectOrNull()}, and allows unknown + * fields. The object fields are constructed with {@link ObjectField#caseSensitive()} as {@code false} and + * {@link ObjectField#repeatedBehavior()} as {@link ObjectField.RepeatedBehavior#USE_FIRST}. + * + * @param fields the fields + * @return the lenient object options + */ + public static ObjectValue lenient(Map fields) { + final Builder builder = builder(); + for (Entry e : fields.entrySet()) { + builder.addFields(ObjectField.builder() + .name(e.getKey()) + .options(e.getValue()) + .caseSensitive(false) + .repeatedBehavior(RepeatedBehavior.USE_FIRST) + .build()); + } + return builder.build(); + } + + /** + * The standard object options. Allows missing, accepts {@link JsonValueTypes#objectOrNull()}, and allows unknown + * fields. The object fields are constructed with {@link ObjectField#of(String, Value)}. + * + * @param fields the fields + * @return the standard object options + */ + public static ObjectValue standard(Map fields) { + final Builder builder = builder(); + for (Entry e : fields.entrySet()) { + builder.putFields(e.getKey(), e.getValue()); + } + return builder.build(); + } + + /** + * The strict object options. Disallows missing, accepts {@link JsonValueTypes#object()}, and disallows unknown + * fields. The object fields are constructed with {@link ObjectField#of(String, Value)}. + * + * @param fields the fields + * @return the strict object options + */ + public static ObjectValue strict(Map fields) { + final Builder builder = builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.object()); + for (Entry e : fields.entrySet()) { + builder.putFields(e.getKey(), e.getValue()); + } + return builder.build(); + } + + /** + * The fields. + */ + public abstract Set fields(); + + /** + * If unknown fields are allowed. By default is {@code true}. + */ + @Default + public boolean allowUnknownFields() { + return true; + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#objectOrNull()}. By default is + * {@link JsonValueTypes#objectOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.objectOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.objectOrNull(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + // not extending value options + public interface Builder extends Value.Builder { + + Builder allowUnknownFields(boolean allowUnknownFields); + + /** + * A convenience method, equivalent to {@code addFields(ObjectFieldOptions.of(key, value))}. + */ + default Builder putFields(String key, Value value) { + return addFields(ObjectField.of(key, value)); + } + + Builder addFields(ObjectField element); + + Builder addFields(ObjectField... elements); + + Builder addAllFields(Iterable elements); + } + + @Check + final void checkNonOverlapping() { + // We need to make sure there is no inter-field overlapping. We will be stricter if _any_ field is + // case-insensitive. + final Set keys = fields().stream().allMatch(ObjectField::caseSensitive) + ? new HashSet<>() + : new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (ObjectField field : fields()) { + if (!keys.add(field.name())) { + throw new IllegalArgumentException(String.format("Found overlapping field name '%s'", field.name())); + } + for (String alias : field.aliases()) { + if (!keys.add(alias)) { + throw new IllegalArgumentException(String.format("Found overlapping field alias '%s'", alias)); + } + } + } + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/ShortValue.java b/extensions/json/src/main/java/io/deephaven/json/ShortValue.java new file mode 100644 index 00000000000..a3e110051dd --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/ShortValue.java @@ -0,0 +1,81 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value as an {@code short}. + */ +@Immutable +@BuildableStyle +public abstract class ShortValue extends ValueSingleValueBase { + + public static Builder builder() { + return ImmutableShortValue.builder(); + } + + /** + * The lenient short options. Allows missing and accepts {@link JsonValueTypes#intLike()}. + * + * @return the lenient short options + */ + public static ShortValue lenient() { + return builder() + .allowedTypes(JsonValueTypes.intLike()) + .build(); + } + + /** + * The standard short options. Allows missing and accepts {@link JsonValueTypes#intOrNull()}. + * + * @return the standard short options + */ + public static ShortValue standard() { + return builder().build(); + } + + /** + * The strict short options. Disallows missing and accepts {@link JsonValueTypes#int_()}. + * + * @return the strict short options + */ + public static ShortValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.int_()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#numberLike()}. By default is + * {@link JsonValueTypes#intOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.intOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.numberLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends BuilderSpecial { + + Builder onNull(short onNull); + + Builder onMissing(short onMissing); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/SkipValue.java b/extensions/json/src/main/java/io/deephaven/json/SkipValue.java new file mode 100644 index 00000000000..be988287712 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/SkipValue.java @@ -0,0 +1,49 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value by skipping it. + */ +@Immutable +@BuildableStyle +public abstract class SkipValue extends Value { + + public static Builder builder() { + return ImmutableSkipValue.builder(); + } + + /** + * The lenient skip options. Allows missing and accepts {@link JsonValueTypes#all()}. + * + * @return the lenient skip options + */ + public static SkipValue lenient() { + return builder().build(); + } + + /** + * {@inheritDoc} By default is {@link JsonValueTypes#all()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.all(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends Value.Builder { + + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/StringValue.java b/extensions/json/src/main/java/io/deephaven/json/StringValue.java new file mode 100644 index 00000000000..80dffdd63af --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/StringValue.java @@ -0,0 +1,81 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Set; + +/** + * Processes a JSON value as a {@link String}. + */ +@Immutable +@BuildableStyle +public abstract class StringValue extends ValueSingleValueBase { + + public static Builder builder() { + return ImmutableStringValue.builder(); + } + + /** + * The lenient {@link String} options. Allows missing and accepts {@link JsonValueTypes#stringLike()}. + * + * @return the lenient String options + */ + public static StringValue lenient() { + return builder() + .allowedTypes(JsonValueTypes.stringLike()) + .build(); + } + + /** + * The standard {@link String} options. Allows missing and accepts {@link JsonValueTypes#stringOrNull()}. + * + * @return the standard String options + */ + public static StringValue standard() { + return builder().build(); + } + + /** + * The strict {@link String} options. Disallows missing and accepts {@link JsonValueTypes#string()}. + * + * @return the strict String options + */ + public static StringValue strict() { + return builder() + .allowMissing(false) + .allowedTypes(JsonValueTypes.string()) + .build(); + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#stringLike()}. By default is + * {@link JsonValueTypes#stringOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.stringOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.stringLike(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends BuilderSpecial { + + Builder onNull(String onNull); + + Builder onMissing(String onMissing); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/TupleValue.java b/extensions/json/src/main/java/io/deephaven/json/TupleValue.java new file mode 100644 index 00000000000..2b394268a72 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/TupleValue.java @@ -0,0 +1,90 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * A "tuple", where an {@link JsonValueTypes#ARRAY} is a known size and each element has a defined type. + * + *

+ * For example, the JSON value {@code ["foo", 42, 5.72]} might be modelled as + * {@code TupleOptions.of(StringOptions.standard(), IntOptions.standard(), DoubleOptions.standard())}. + */ +@Immutable +@BuildableStyle +public abstract class TupleValue extends ValueRestrictedUniverseBase { + + public static Builder builder() { + return ImmutableTupleValue.builder(); + } + + /** + * Creates a tuple of the given {@code values}, with name incrementing, starting from "0". + * + * @param values the values + * @return the tuple options + */ + public static TupleValue of(Value... values) { + return of(Arrays.asList(values)); + } + + /** + * Creates a tuple of the given {@code values}, with name incrementing, starting from "0". + * + * @param values the values + * @return the tuple options + */ + public static TupleValue of(Iterable values) { + final Builder builder = builder(); + final Iterator it = values.iterator(); + for (int i = 0; it.hasNext(); ++i) { + builder.putNamedValues(Integer.toString(i), it.next()); + } + return builder.build(); + } + + /** + * The named, ordered values of the tuple. + */ + public abstract Map namedValues(); + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#arrayOrNull()}. By default is + * {@link JsonValueTypes#arrayOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.arrayOrNull(); + } + + @Override + final Set universe() { + return JsonValueTypes.arrayOrNull(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends Value.Builder { + + Builder putNamedValues(String key, Value value); + + Builder putNamedValues(Map.Entry entry); + + Builder putAllNamedValues(Map entries); + + TupleValue build(); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/TypedObjectValue.java b/extensions/json/src/main/java/io/deephaven/json/TypedObjectValue.java new file mode 100644 index 00000000000..e975b1636c8 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/TypedObjectValue.java @@ -0,0 +1,209 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Check; +import org.immutables.value.Value.Default; +import org.immutables.value.Value.Immutable; + +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; + +/** + * A type-discriminated object is a JSON object whose fields depend on a specific type field. + * + *

+ * For example, the following might be modelled as a type-discriminated object with "type" as the type field, "symbol" + * as a shared field, with a "quote" object containing a "bid" and an "ask" field, and with a "trade" object containing + * a "price" and a "size" field: + * + * 

+ * {
+ *   "type": "quote",
+ *   "symbol": "BAR",
+ *   "bid": 10.01,
+ *   "ask": 10.05
+ * }
+ * {
+ *   "type": "trade",
+ *   "symbol": "FOO",
+ *   "price": 70.03,
+ *   "size": 42
+ * }
+ *
+ */ +@Immutable +@BuildableStyle +public abstract class TypedObjectValue extends ValueRestrictedUniverseBase { + + public static Builder builder() { + return ImmutableTypedObjectValue.builder(); + } + + /** + * Creates a new builder with {@link #sharedFields()} inferred from {@code objects} based on {@link ObjectField} + * equality, and {@link #objects()} set to {@code objects} with the shared fields removed. + * + * @param objects the objects + * @return the builder + */ + public static Builder builder(Map objects) { + final Builder builder = builder(); + final Set sharedFields = new LinkedHashSet<>(); + final ObjectValue first = objects.values().iterator().next(); + for (ObjectField field : first.fields()) { + boolean isShared = true; + for (ObjectValue obj : objects.values()) { + if (!obj.fields().contains(field)) { + isShared = false; + break; + } + } + if (isShared) { + sharedFields.add(field); + } + } + for (Entry e : objects.entrySet()) { + builder.putObjects(e.getKey(), without(e.getValue(), sharedFields)); + } + return builder.addAllSharedFields(sharedFields); + } + + /** + * Creates a typed object by inferring the shared fields. Equivalent to + * {@code builder(typeFieldName, objects).build()}. + * + * @param typeFieldName the type field name + * @param objects the objects + * @return the typed object + */ + public static TypedObjectValue standard(String typeFieldName, Map objects) { + return builder(objects).typeFieldName(typeFieldName).build(); + } + + /** + * Creates a typed object by inferring the shared fields. Equivalent to + * {@code builder(typeFieldName, objects).allowUnknownTypes(false).allowMissing(false).desiredTypes(JsonValueTypes.OBJECT).build()}. + * + * @param typeFieldName the type field name + * @param objects the objects + * @return the typed object + */ + public static TypedObjectValue strict(String typeFieldName, Map objects) { + return builder(objects) + .typeFieldName(typeFieldName) + .allowUnknownTypes(false) + .allowMissing(false) + .allowedTypes(JsonValueTypes.object()) + .build(); + } + + /** + * The type field. + */ + public abstract ObjectField typeField(); + + /** + * The shared fields. + */ + public abstract Set sharedFields(); + + /** + * The discriminated objects. + */ + public abstract Map objects(); + + /** + * If unknown fields are allowed. By default is {@code true}. + */ + @Default + public boolean allowUnknownTypes() { + return true; + } + + /** + * {@inheritDoc} Must be a subset of {@link JsonValueTypes#objectOrNull()}. By default is + * {@link JsonValueTypes#objectOrNull()}. + */ + @Override + @Default + public Set allowedTypes() { + return JsonValueTypes.objectOrNull(); + } + + /** + * The output type value to use when {@link JsonValueTypes#NULL} is encountered. {@link #allowedTypes()} must + * contain {@link JsonValueTypes#NULL}. + */ + public abstract Optional onNull(); + + /** + * The output type value to use when a value is missing. {@link #allowMissing()} must be {@code true}. + */ + public abstract Optional onMissing(); + + @Override + final Set universe() { + return JsonValueTypes.objectOrNull(); + } + + @Override + public final T walk(Visitor visitor) { + return visitor.visit(this); + } + + public interface Builder extends Value.Builder { + + default Builder typeFieldName(String typeFieldName) { + return typeField(ObjectField.of(typeFieldName, StringValue.standard())); + } + + Builder typeField(ObjectField typeField); + + Builder addSharedFields(ObjectField element); + + Builder addSharedFields(ObjectField... elements); + + Builder addAllSharedFields(Iterable elements); + + Builder putObjects(Object key, ObjectValue value); + + Builder allowUnknownTypes(boolean allowUnknownTypes); + + Builder onNull(Object onNull); + + Builder onMissing(Object onMissing); + } + + private static ObjectValue without(ObjectValue options, Set excludedFields) { + final ObjectValue.Builder builder = ObjectValue.builder() + .allowUnknownFields(options.allowUnknownFields()) + .allowMissing(options.allowMissing()) + .allowedTypes(options.allowedTypes()); + for (ObjectField field : options.fields()) { + if (!excludedFields.contains(field)) { + builder.addFields(field); + } + } + return builder.build(); + } + + @Check + final void checkOnNull() { + if (!allowedTypes().contains(JsonValueTypes.NULL) && onNull().isPresent()) { + throw new IllegalArgumentException("onNull set, but NULL is not allowed"); + } + } + + @Check + final void checkOnMissing() { + if (!allowMissing() && onMissing().isPresent()) { + throw new IllegalArgumentException("onMissing set, but allowMissing is false"); + } + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/Value.java b/extensions/json/src/main/java/io/deephaven/json/Value.java new file mode 100644 index 00000000000..cbb1451c770 --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/Value.java @@ -0,0 +1,132 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import org.immutables.value.Value.Check; +import org.immutables.value.Value.Default; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Map; +import java.util.Set; + +/** + * The base configuration for JSON values. + */ +public abstract class Value { + + /** + * The allowed types. + */ + public abstract Set allowedTypes(); + + /** + * If the processor should allow a missing JSON value. By default is {@code true}. + */ + @Default + public boolean allowMissing() { + return true; + } + + /** + * Wraps the allowed values of {@code this} as {@link SkipValue}. Equivalent to + * {@code SkipValue.builder().allowMissing(allowMissing()).allowedTypes(allowedTypes()).build()}. + * + * @return this allowed values of this as skip options + */ + public final SkipValue skip() { + return SkipValue.builder() + .allowMissing(allowMissing()) + .allowedTypes(allowedTypes()) + .build(); + } + + /** + * Wraps {@code this} as the value of an {@link ArrayValue}. Equivalent to {@code ArrayOptions.standard(this)}. + * + * @return this as the value of an array options + * @see ArrayValue#standard(Value) + */ + public final ArrayValue array() { + return ArrayValue.standard(this); + } + + /** + * Wraps {@code this} as a singular field of an {@link ObjectValue}. Equivalent to + * {@code ObjectOptions.standard(Map.of(name, this))}. + * + * @param name the field name + * @return this as the singular field of an object options + * @see ObjectValue#standard(Map) + */ + public final ObjectValue field(String name) { + return ObjectValue.standard(Map.of(name, this)); + } + + public abstract T walk(Visitor visitor); + + public interface Visitor { + + T visit(StringValue _string); + + T visit(BoolValue _bool); + + T visit(CharValue _char); + + T visit(ByteValue _byte); + + T visit(ShortValue _short); + + T visit(IntValue _int); + + T visit(LongValue _long); + + T visit(FloatValue _float); + + T visit(DoubleValue _double); + + T visit(ObjectValue object); + + T visit(ObjectEntriesValue objectKv); + + T visit(InstantValue instant); + + T visit(InstantNumberValue instantNumber); + + T visit(BigIntegerValue bigInteger); + + T visit(BigDecimalValue bigDecimal); + + T visit(SkipValue skip); + + T visit(TupleValue tuple); + + T visit(TypedObjectValue typedObject); + + T visit(LocalDateValue localDate); + + T visit(ArrayValue array); + + T visit(AnyValue any); + } + + public interface Builder> { + + B allowMissing(boolean allowMissing); + + B allowedTypes(Set allowedTypes); + + default B allowedTypes(JsonValueTypes... allowedTypes) { + return allowedTypes(Collections.unmodifiableSet(EnumSet.copyOf(Arrays.asList(allowedTypes)))); + } + + V build(); + } + + @Check + final void checkAllowedTypeInvariants() { + JsonValueTypes.checkAllowedTypeInvariants(allowedTypes()); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/ValueRestrictedUniverseBase.java b/extensions/json/src/main/java/io/deephaven/json/ValueRestrictedUniverseBase.java new file mode 100644 index 00000000000..07336654ffb --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/ValueRestrictedUniverseBase.java @@ -0,0 +1,30 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import org.immutables.value.Value.Check; + +import java.util.Collection; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A base {@link Value} where the implementation has a clearly defined universe. + */ +public abstract class ValueRestrictedUniverseBase extends Value { + + abstract Set universe(); + + @Check + void checkAllowedTypes() { + if (!universe().containsAll(allowedTypes())) { + throw new IllegalArgumentException(String.format("Unexpected allowedTypes=%s, universe=%s", + toString(allowedTypes()), toString(universe()))); + } + } + + private static String toString(Collection> s) { + return s.stream().map(Enum::name).collect(Collectors.joining(",", "[", "]")); + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/ValueSingleValueBase.java b/extensions/json/src/main/java/io/deephaven/json/ValueSingleValueBase.java new file mode 100644 index 00000000000..ed3cb0b57cd --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/ValueSingleValueBase.java @@ -0,0 +1,63 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import org.immutables.value.Value.Check; + +import java.util.Optional; + +/** + * A base {@link Value} where the JSON value represents a single value. + * + * @param the value type + */ +public abstract class ValueSingleValueBase extends ValueRestrictedUniverseBase { + + /** + * The value to use when {@link JsonValueTypes#NULL} is encountered. {@link #allowedTypes()} must contain + * {@link JsonValueTypes#NULL}. + */ + public abstract Optional onNull(); + + /** + * The value to use when a value is missing. {@link #allowMissing()} must be {@code true}. + */ + public abstract Optional onMissing(); + + public interface Builder, B extends Builder> + extends Value.Builder { + B onNull(T onNull); + + B onNull(Optional onNull); + + B onMissing(T onMissing); + + B onMissing(Optional onMissing); + } + + public interface BuilderSpecial, B extends BuilderSpecial> + extends Value.Builder { + + // Immutables has special handling for primitive types and some "special" types like String. + // This differs from the above Builder where the Optional generic is "? extends T". + + B onNull(Optional onNull); + + B onMissing(Optional onMissing); + } + + @Check + final void checkOnNull() { + if (!allowedTypes().contains(JsonValueTypes.NULL) && onNull().isPresent()) { + throw new IllegalArgumentException("onNull set, but NULL is not allowed"); + } + } + + @Check + final void checkOnMissing() { + if (!allowMissing() && onMissing().isPresent()) { + throw new IllegalArgumentException("onMissing set, but allowMissing is false"); + } + } +} diff --git a/extensions/json/src/main/java/io/deephaven/json/package-info.java b/extensions/json/src/main/java/io/deephaven/json/package-info.java new file mode 100644 index 00000000000..8f5e6431c3b --- /dev/null +++ b/extensions/json/src/main/java/io/deephaven/json/package-info.java @@ -0,0 +1,23 @@ +/** + * The deephaven JSON package presents a declarative and composable configuration layer for describing the structure of + * a JSON value. It is meant to have sane defaults while also providing finer-grained configuration options for typical + * scenarios. The primary purpose of this package is to provide a common layer that various consumers can use to parse + * JSON values into appropriate Deephaven structures. As such (and by the very nature of JSON), these types represent a + * superset of JSON. This package can also service other use cases where the JSON structuring is necessary (for example, + * producing a JSON value from a Deephaven structure). + * + *

+ * Most of the configuration layers allow the user the choice of a "standard" option, a "strict" option, a "lenient" + * option, and a "builder" option. The "standard" option is typically configured to allow missing values and to accept + * the typically expected {@link io.deephaven.json.JsonValueTypes}, including + * {@link io.deephaven.json.JsonValueTypes#NULL}. The "strict" option is typically configured to disallow missing values + * and to accept the typically expected {@link io.deephaven.json.JsonValueTypes}, excluding + * {@link io.deephaven.json.JsonValueTypes#NULL}. The "lenient" option is typically configured to allow missing values, + * and to accept values a wide range of {@link io.deephaven.json.JsonValueTypes} by coercing atypical types into the + * requested type (for example, parsing a {@link io.deephaven.json.JsonValueTypes#STRING} into an {@code int}). The + * "builder" option allows the user fine-grained control over the behavior, and otherwise uses the "standard" options + * when the user does not override. + * + * @see io.deephaven.json.Value + */ +package io.deephaven.json; diff --git a/extensions/json/src/test/java/io/deephaven/json/JsonValueTypesTest.java b/extensions/json/src/test/java/io/deephaven/json/JsonValueTypesTest.java new file mode 100644 index 00000000000..87e7306df97 --- /dev/null +++ b/extensions/json/src/test/java/io/deephaven/json/JsonValueTypesTest.java @@ -0,0 +1,28 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JsonValueTypesTest { + + @Test + void checkAllowedTypeInvariants() throws InvocationTargetException, IllegalAccessException { + int count = 0; + for (Method declaredMethod : JsonValueTypes.class.getDeclaredMethods()) { + if (declaredMethod.getReturnType().equals(Set.class)) { + final Set set = (Set) declaredMethod.invoke(null); + JsonValueTypes.checkAllowedTypeInvariants(set); + ++count; + } + } + assertThat(count).isEqualTo(17); + } +} diff --git a/extensions/json/src/test/java/io/deephaven/json/TypedObjectValueBuilderTest.java b/extensions/json/src/test/java/io/deephaven/json/TypedObjectValueBuilderTest.java new file mode 100644 index 00000000000..6abe077d3cc --- /dev/null +++ b/extensions/json/src/test/java/io/deephaven/json/TypedObjectValueBuilderTest.java @@ -0,0 +1,47 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.json; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TypedObjectValueBuilderTest { + + public static final ObjectValue TRADE = ObjectValue.builder() + .putFields("symbol", StringValue.strict()) + .putFields("price", DoubleValue.strict()) + .putFields("size", LongValue.strict()) + .build(); + + public static final ObjectValue QUOTE = ObjectValue.builder() + .putFields("symbol", StringValue.strict()) + .putFields("bid", DoubleValue.strict()) + .putFields("ask", DoubleValue.strict()) + .build(); + + public static final TypedObjectValue COMBINED = TypedObjectValue.builder() + .typeFieldName("type") + .addSharedFields(ObjectField.of("symbol", StringValue.strict())) + .putObjects("trade", ObjectValue.builder() + .putFields("price", DoubleValue.strict()) + .putFields("size", LongValue.strict()) + .build()) + .putObjects("quote", ObjectValue.builder() + .putFields("bid", DoubleValue.strict()) + .putFields("ask", DoubleValue.strict()) + .build()) + .build(); + + @Test + void builderHelper() { + final TypedObjectValue combined = TypedObjectValue + .builder(Map.of("quote", QUOTE, "trade", TRADE)) + .typeFieldName("type") + .build(); + assertThat(combined).isEqualTo(COMBINED); + } +} diff --git a/extensions/kafka/src/main/java/io/deephaven/kafka/KafkaTools.java b/extensions/kafka/src/main/java/io/deephaven/kafka/KafkaTools.java index f4c906f165a..e51f44251d6 100644 --- a/extensions/kafka/src/main/java/io/deephaven/kafka/KafkaTools.java +++ b/extensions/kafka/src/main/java/io/deephaven/kafka/KafkaTools.java @@ -70,9 +70,11 @@ import io.deephaven.kafka.publish.KafkaPublisherException; import io.deephaven.kafka.publish.KeyOrValueSerializer; import io.deephaven.kafka.publish.PublishToKafka; +import io.deephaven.processor.NamedObjectProcessor; import io.deephaven.processor.ObjectProcessor; import io.deephaven.protobuf.ProtobufDescriptorParserOptions; import io.deephaven.qst.column.header.ColumnHeader; +import io.deephaven.qst.type.Type; import io.deephaven.stream.StreamChunkUtils; import io.deephaven.stream.StreamConsumer; import io.deephaven.stream.StreamPublisher; @@ -580,8 +582,7 @@ public static KeyOrValueSpec rawSpec(ColumnHeader header, Class - * The respective column definitions are derived from the combination of {@code columnNames} and - * {@link ObjectProcessor#outputTypes()}. + * Equivalent to {@code objectProcessorSpec(deserializer, NamedObjectProcessor.of(processor, columnNames))}. * * @param deserializer the deserializer * @param processor the object processor @@ -593,14 +594,14 @@ public static KeyOrValueSpec objectProcessorSpec( Deserializer deserializer, ObjectProcessor processor, List columnNames) { - return new KeyOrValueSpecObjectProcessorImpl<>(deserializer, processor, columnNames); + return objectProcessorSpec(deserializer, NamedObjectProcessor.of(processor, columnNames)); } /** * Creates a kafka key or value spec implementation from a byte-array {@link ObjectProcessor}. * *

- * Equivalent to {@code objectProcessorSpec(new ByteArrayDeserializer(), processor, columnNames)}. + * Equivalent to {@code objectProcessorSpec(NamedObjectProcessor.of(processor, columnNames))}. * * @param processor the byte-array object processor * @param columnNames the column names @@ -609,7 +610,51 @@ public static KeyOrValueSpec objectProcessorSpec( @SuppressWarnings("unused") public static KeyOrValueSpec objectProcessorSpec(ObjectProcessor processor, List columnNames) { - return objectProcessorSpec(new ByteArrayDeserializer(), processor, columnNames); + return objectProcessorSpec(NamedObjectProcessor.of(processor, columnNames)); + } + + /** + * Creates a kafka key or value spec implementation from a {@link NamedObjectProcessor}. + * + * @param deserializer the deserializer + * @param processor the named object processor + * @return the Kafka key or value spec + * @param the object type + */ + public static KeyOrValueSpec objectProcessorSpec( + Deserializer deserializer, + NamedObjectProcessor processor) { + return new KeyOrValueSpecObjectProcessorImpl<>(deserializer, processor); + } + + /** + * Creates a kafka key or value spec implementation from the named object processor. + * + *

+ * Equivalent to {@code objectProcessorSpec(new ByteArrayDeserializer(), processor)}. + * + * @param processor the named object processor + * @return the Kafka key or value spec + * @see #objectProcessorSpec(Deserializer, NamedObjectProcessor) + * @see ByteArrayDeserializer + */ + public static KeyOrValueSpec objectProcessorSpec(NamedObjectProcessor processor) { + return objectProcessorSpec(new ByteArrayDeserializer(), processor); + } + + /** + * Creates a kafka key or value spec implementation from a named object processor provider. It must be capable + * of supporting {@code byte[]}. + * + *

+ * Equivalent to {@code objectProcessorSpec(provider.named(Type.byteType().arrayType()))}. + * + * @param provider the named object processor provider + * @return the Kafka key or value spec + * @see #objectProcessorSpec(NamedObjectProcessor) + */ + public static KeyOrValueSpec objectProcessorSpec(NamedObjectProcessor.Provider provider) { + return objectProcessorSpec(provider.named(Type.byteType().arrayType())); } } diff --git a/extensions/kafka/src/main/java/io/deephaven/kafka/KeyOrValueSpecObjectProcessorImpl.java b/extensions/kafka/src/main/java/io/deephaven/kafka/KeyOrValueSpecObjectProcessorImpl.java index 9b0043823d6..4fa4d63dcf0 100644 --- a/extensions/kafka/src/main/java/io/deephaven/kafka/KeyOrValueSpecObjectProcessorImpl.java +++ b/extensions/kafka/src/main/java/io/deephaven/kafka/KeyOrValueSpecObjectProcessorImpl.java @@ -16,6 +16,7 @@ import io.deephaven.kafka.ingest.KafkaStreamPublisher; import io.deephaven.kafka.ingest.KeyOrValueProcessor; import io.deephaven.kafka.ingest.MultiFieldChunkAdapter; +import io.deephaven.processor.NamedObjectProcessor; import io.deephaven.processor.ObjectProcessor; import io.deephaven.qst.type.Type; import io.deephaven.util.mutable.MutableInt; @@ -33,27 +34,19 @@ /** * This implementation is useful for presenting an easier onboarding ramp and better (and public) interface - * {@link KafkaTools.Consume#objectProcessorSpec(Deserializer, ObjectProcessor, List)} for end-users. The + * {@link KafkaTools.Consume#objectProcessorSpec(Deserializer, NamedObjectProcessor)} for end-users. The * {@link ObjectProcessor} is a user-visible replacement for {@link KeyOrValueProcessor}. In the meantime though, we are * adapting into a {@link KeyOrValueProcessor} until such a time when {@link KafkaStreamPublisher} can be re-written to * take advantage of these better interfaces. */ class KeyOrValueSpecObjectProcessorImpl extends KeyOrValueSpec { private final Deserializer deserializer; - private final ObjectProcessor processor; - private final List columnNames; + private final NamedObjectProcessor processor; - KeyOrValueSpecObjectProcessorImpl( - Deserializer deserializer, ObjectProcessor processor, List columnNames) { - if (columnNames.size() != processor.outputTypes().size()) { - throw new IllegalArgumentException("Expected columnNames and processor.outputTypes() to be the same size"); - } - if (columnNames.stream().distinct().count() != columnNames.size()) { - throw new IllegalArgumentException("Expected columnNames to have distinct values"); - } + KeyOrValueSpecObjectProcessorImpl(Deserializer deserializer, + NamedObjectProcessor processor) { this.deserializer = Objects.requireNonNull(deserializer); this.processor = Objects.requireNonNull(processor); - this.columnNames = List.copyOf(columnNames); } @Override @@ -73,10 +66,12 @@ protected KeyOrValueIngestData getIngestData(KeyOrValue keyOrValue, SchemaRegist Map configs, MutableInt nextColumnIndexMut, List> columnDefinitionsOut) { final KeyOrValueIngestData data = new KeyOrValueIngestData(); data.fieldPathToColumnName = new LinkedHashMap<>(); - final int L = columnNames.size(); + final List names = processor.names(); + final List> types = processor.processor().outputTypes(); + final int L = names.size(); for (int i = 0; i < L; ++i) { - final String columnName = columnNames.get(i); - final Type type = processor.outputTypes().get(i); + final String columnName = names.get(i); + final Type type = types.get(i); data.fieldPathToColumnName.put(columnName, columnName); columnDefinitionsOut.add(ColumnDefinition.of(columnName, type)); } @@ -101,7 +96,7 @@ public void handleChunk(ObjectChunk inputChunk, WritableChunk in = (ObjectChunk) inputChunk; // we except isInOrder to be true, so apply should be an O(1) op no matter how many columns there are. - processor.processAll(in, offsetsAdapter.apply(publisherChunks)); + processor.processor().processAll(in, offsetsAdapter.apply(publisherChunks)); } } diff --git a/py/server/deephaven/dtypes.py b/py/server/deephaven/dtypes.py index e0308072acd..1d83343881a 100644 --- a/py/server/deephaven/dtypes.py +++ b/py/server/deephaven/dtypes.py @@ -97,6 +97,8 @@ def __call__(self, *args, **kwargs): """String type""" Character = DType(j_name="java.lang.Character") """Character type""" +BigInteger = DType(j_name="java.math.BigInteger") +"""Java BigInteger type""" BigDecimal = DType(j_name="java.math.BigDecimal") """Java BigDecimal type""" StringSet = DType(j_name="io.deephaven.stringset.StringSet") diff --git a/py/server/deephaven/json/__init__.py b/py/server/deephaven/json/__init__.py new file mode 100644 index 00000000000..77957733df8 --- /dev/null +++ b/py/server/deephaven/json/__init__.py @@ -0,0 +1,1283 @@ +# +# Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +# + +"""The deephaven JSON module presents a declarative and composable configuration layer for describing the structure of a +JSON (https://www.json.org) value. Most commonly, this will be used to model the structure for a JSON object. For +example, the JSON object + +.. code-block:: json + { "name": "Foo", "age": 42, "location": { "lat": 45.018269, "lon": -93.473892 } } + +can be modelled with the dictionary + +.. code-block:: python + { "name": str, "age": int, "location": { "lat": float, "lon": float } } + +Notice that this allows for the nested modelling of JSON values. Other common constructions involve the modelling of +JSON arrays. For example, a variable-length JSON array where the elements are the same type + +.. code-block:: json + [42, 31, ..., 12345] + +can be modelled with a single-element list containing the element type + +.. code-block:: python + [ int ] + +If the JSON array is a fixed size and each elements' type is known, for example + +.. code-block:: json + ["Foo", 42, [45.018269, -93.473892]] + +can be modelled with a tuple containing each type + +.. code-block:: python + (str, int, (float, float)) + +Notice again that this allows for the nested modelling of JSON values. Of course, these constructions can be all be used +together. For example, the JSON object + +.. code-block:: json + { + "name": "Foo", + "locations": [ + [45.018269, -93.473892], + ..., + [40.730610, -73.935242] + ] + } + +can be modelled as + +.. code-block:: python + {"name": str, "locations": [(float, float)]} + +See the methods in this module more more details on modelling JSON values. +""" + +import jpy +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import Dict, List, Union, Tuple, Optional, Literal, Any + +from deephaven import dtypes +from deephaven._wrapper import JObjectWrapper +from deephaven.time import to_j_instant +from deephaven._jpy import strict_cast + + +__all__ = [ + "string_val", + "bool_val", + "char_val", + "byte_val", + "short_val", + "int_val", + "long_val", + "float_val", + "double_val", + "instant_val", + "big_integer_val", + "big_decimal_val", + "array_val", + "object_val", + "typed_object_val", + "object_entries_val", + "tuple_val", + "any_val", + "skip_val", + "json_val", + "JsonValue", + "JsonValueType", + "RepeatedFieldBehavior", + "ObjectField", +] + +_JValue = jpy.get_type("io.deephaven.json.Value") +_JObjectValue = jpy.get_type("io.deephaven.json.ObjectValue") +_JTypedObjectValue = jpy.get_type("io.deephaven.json.TypedObjectValue") +_JArrayValue = jpy.get_type("io.deephaven.json.ArrayValue") +_JObjectEntriesValue = jpy.get_type("io.deephaven.json.ObjectEntriesValue") +_JTupleValue = jpy.get_type("io.deephaven.json.TupleValue") +_JObjectField = jpy.get_type("io.deephaven.json.ObjectField") +_JRepeatedFieldBehavior = jpy.get_type("io.deephaven.json.ObjectField$RepeatedBehavior") +_JJsonValueTypes = jpy.get_type("io.deephaven.json.JsonValueTypes") +_JBoolValue = jpy.get_type("io.deephaven.json.BoolValue") +_JCharValue = jpy.get_type("io.deephaven.json.CharValue") +_JByteValue = jpy.get_type("io.deephaven.json.ByteValue") +_JShortValue = jpy.get_type("io.deephaven.json.ShortValue") +_JIntValue = jpy.get_type("io.deephaven.json.IntValue") +_JLongValue = jpy.get_type("io.deephaven.json.LongValue") +_JFloatValue = jpy.get_type("io.deephaven.json.FloatValue") +_JDoubleValue = jpy.get_type("io.deephaven.json.DoubleValue") +_JStringValue = jpy.get_type("io.deephaven.json.StringValue") +_JSkipValue = jpy.get_type("io.deephaven.json.SkipValue") +_JInstantValue = jpy.get_type("io.deephaven.json.InstantValue") +_JInstantNumberValue = jpy.get_type("io.deephaven.json.InstantNumberValue") +_JInstantNumberValueFormat = jpy.get_type("io.deephaven.json.InstantNumberValue$Format") +_JBigIntegerValue = jpy.get_type("io.deephaven.json.BigIntegerValue") +_JBigDecimalValue = jpy.get_type("io.deephaven.json.BigDecimalValue") +_JAnyValue = jpy.get_type("io.deephaven.json.AnyValue") + + +_VALUE_STRING = _JJsonValueTypes.STRING +_VALUE_NULL = _JJsonValueTypes.NULL +_VALUE_INT = _JJsonValueTypes.INT +_VALUE_DECIMAL = _JJsonValueTypes.DECIMAL +_VALUE_BOOL = _JJsonValueTypes.BOOL +_VALUE_OBJECT = _JJsonValueTypes.OBJECT +_VALUE_ARRAY = _JJsonValueTypes.ARRAY + +_EPOCH_SECONDS = _JInstantNumberValueFormat.EPOCH_SECONDS +_EPOCH_MILLIS = _JInstantNumberValueFormat.EPOCH_MILLIS +_EPOCH_MICROS = _JInstantNumberValueFormat.EPOCH_MICROS +_EPOCH_NANOS = _JInstantNumberValueFormat.EPOCH_NANOS + + +class JsonValue(JObjectWrapper): + """The JSON Value type.""" + + j_object_type = _JValue + + def __init__(self, j_value: jpy.JType): + self.j_value = j_value + + @property + def j_object(self) -> jpy.JType: + return self.j_value + + +JsonValueType = Union[ + JsonValue, + dtypes.DType, + type, + Dict[str, Union["JsonValueType", "ObjectField"]], + List["JsonValueType"], + Tuple["JsonValueType", ...], +] +"""The JSON value alias""" + + +def json_val(json_value_type: JsonValueType) -> JsonValue: + """Creates a JsonValue from a JsonValueType. + + - JsonValue is returned unchanged + - bool returns bool_val() + - int returns long_val() + - float returns double_val() + - str returns string_val() + - datetime.datetime returns instant_val() + - object returns any_val() + - Dictionaries returns object_val(json_value_type) + - Lists of length 1 returns array_val(json_value_type[0]) (Lists of other sizes are not supported) + - Tuples returns tuple_val(json_value_type) + + Args: + json_value_type (JsonValueType): the JSON value type + + Returns: + the JSON value + """ + if isinstance(json_value_type, JsonValue): + return json_value_type + if isinstance(json_value_type, dtypes.DType): + return _dtype_dict[json_value_type] + if isinstance(json_value_type, type): + return _type_dict[json_value_type] + if isinstance(json_value_type, Dict): + return object_val(json_value_type) + if isinstance(json_value_type, List): + if len(json_value_type) is not 1: + raise TypeError("Expected List as json type to have exactly one element") + return array_val(json_value_type[0]) + if isinstance(json_value_type, Tuple): + return tuple_val(json_value_type) + raise TypeError(f"Unsupported JSON value type {type(json_value_type)}") + + +class RepeatedFieldBehavior(Enum): + """ + The behavior to use when a repeated field is encountered in a JSON object. For example, + .. code-block:: json + { + "foo": 42, + "foo": 43 + } + """ + + USE_FIRST = _JRepeatedFieldBehavior.USE_FIRST + """Use the first field""" + + ERROR = _JRepeatedFieldBehavior.ERROR + """Raise an error""" + + +@dataclass +class ObjectField: + """The object field options. + + In contexts where the user needs to create an object field value and isn't changing any default values, the user can + simplify by just using the JsonValueType. For example, + + .. code-block:: python + { + "name": ObjectField(str), + "age": ObjectField(int), + } + + could be simplified to + + .. code-block:: python + { + "name": str, + "age": int, + } + """ + + value_type: JsonValueType + """The json value type""" + aliases: Union[str, List[str]] = field(default_factory=list) + """The field name aliases. By default, is an empty list.""" + repeated_behavior: RepeatedFieldBehavior = RepeatedFieldBehavior.ERROR + """The repeated field behavior. By default, is RepeatedFieldBehavior.ERROR.""" + case_sensitive: bool = True + """If the field name and aliases should be compared using case-sensitive equality. By default, is True.""" + + def _j_field_options(self, name: str) -> jpy.JType: + builder = ( + _JObjectField.builder() + .name(name) + .options(json_val(self.value_type).j_value) + .repeatedBehavior(self.repeated_behavior.value) + .caseSensitive(self.case_sensitive) + ) + if self.aliases: + builder.addAliases( + [self.aliases] if isinstance(self.aliases, str) else self.aliases + ) + return builder.build() + + +def _build( + builder, + allow_missing: bool, + allow_null: bool, + allow_int: bool = False, + allow_decimal: bool = False, + allow_string: bool = False, + allow_bool: bool = False, + allow_object: bool = False, + allow_array: bool = False, +): + builder.allowMissing(allow_missing) + builder.allowedTypes( + ([_VALUE_STRING] if allow_string else []) + + ([_VALUE_NULL] if allow_null else []) + + ([_VALUE_INT] if allow_int else []) + + ([_VALUE_DECIMAL] if allow_decimal else []) + + ([_VALUE_BOOL] if allow_bool else []) + + ([_VALUE_OBJECT] if allow_object else []) + + ([_VALUE_ARRAY] if allow_array else []) + ) + + +def object_val( + fields: Dict[str, Union[JsonValueType, ObjectField]], + allow_unknown_fields: bool = True, + allow_missing: bool = True, + allow_null: bool = True, + repeated_field_behavior: RepeatedFieldBehavior = RepeatedFieldBehavior.ERROR, + case_sensitive: bool = True, +) -> JsonValue: + """Creates an object value. For example, the JSON object + + .. code-block:: json + { "name": "foo", "age": 42 } + + might be modelled as the object type + + .. code-block:: python + object_val({ "name": str, "age": int }) + + In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can + simplify by using a Dict[str, Union[JsonValueType, ObjectField]]. For example, + + .. code-block:: python + some_method(object_val({ "name": str, "age": int })) + + could be simplified to + + .. code-block:: python + some_method({ "name": str, "age": int }) + + Args: + fields (Dict[str, Union[JsonValueType, ObjectField]]): the fields + allow_unknown_fields (bool): if unknown fields are allow, by default is True + allow_missing (bool): if the object is allowed to be missing, by default is True + allow_null (bool): if the object is allowed to be a JSON null type, by default is True + repeated_field_behavior (RepeatedFieldBehavior): the default repeated field behavior, only used for fields that + are specified using JsonValueType, by default is RepeatedFieldBehavior.ERROR + case_sensitive (bool): if the field name and aliases should be compared using case-sensitive equality, only used + for fields that are specified using JsonValueType, by default is True + + Returns: + the object value + """ + builder = _JObjectValue.builder() + _build(builder, allow_missing, allow_null, allow_object=True) + builder.allowUnknownFields(allow_unknown_fields) + for field_name, field_opts in fields.items(): + field_opts = ( + field_opts + if isinstance(field_opts, ObjectField) + else ObjectField( + field_opts, + repeated_behavior=repeated_field_behavior, + case_sensitive=case_sensitive, + ) + ) + # noinspection PyProtectedMember + builder.addFields(field_opts._j_field_options(field_name)) + return JsonValue(builder.build()) + + +def typed_object_val( + type_field: str, + shared_fields: Dict[str, Union[JsonValueType, ObjectField]], + objects: Dict[str, JsonValueType], + allow_unknown_types: bool = True, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[str] = None, + on_null: Optional[str] = None, +) -> JsonValue: + """Creates a type-discriminated object value. For example, the JSON objects + + .. code-block:: json + { "type": "trade", "symbol": "FOO", "price": 70.03, "size": 42 } + + .. code-block:: json + { "type": "quote", "symbol": "BAR", "bid": 10.01, "ask": 10.05 } + + might be modelled as a type-discriminated object with "type" as the type field, "symbol" as a shared field, with a + "trade" object containing a "bid" and an "ask" field, and with a "quote" object containing a "price" and a "size" + field: + + .. code-block:: python + typed_object_val( + "type", + {"symbol": str}, + { + "quote": { + "price": float, + "size": int + }, + "trade": { + "bid": float, + "ask": float + } + } + ) + + Args: + type_field (str): the type-discriminating field + shared_fields (Dict[str, Union[JsonValueType, ObjectField]]): the shared fields + objects (Dict[str, Union[JsonValueType, ObjectField]]): the individual objects, keyed by their + type-discriminated value. The values must be object options. + allow_unknown_types (bool): if unknown types are allow, by default is True + allow_missing (bool): if the object is allowed to be missing, by default is True + allow_null (bool): if the object is allowed to be a JSON null type, by default is True + on_missing (Optional[str]): the type value to use when the JSON value is missing and allow_missing is True, + default is None + on_null (Optional[str]): the type value to use when the JSON value is null and allow_null is True, default is + None + + Returns: + the typed object value + """ + builder = _JTypedObjectValue.builder() + _build(builder, allow_missing, allow_null, allow_object=True) + builder.typeFieldName(type_field) + builder.allowUnknownTypes(allow_unknown_types) + if on_missing: + builder.onMissing(on_missing) + if on_null: + builder.onNull(on_null) + for shared_field_name, shared_field_opts in shared_fields.items(): + shared_field_opts = ( + shared_field_opts + if isinstance(shared_field_opts, ObjectField) + else ObjectField( + shared_field_opts, + repeated_behavior=RepeatedFieldBehavior.ERROR, + case_sensitive=True, + ) + ) + # noinspection PyProtectedMember + builder.addSharedFields(shared_field_opts._j_field_options(shared_field_name)) + for object_name, object_type in objects.items(): + builder.putObjects( + object_name, strict_cast(json_val(object_type).j_value, _JObjectValue) + ) + return JsonValue(builder.build()) + + +def array_val( + element: JsonValueType, + allow_missing: bool = True, + allow_null: bool = True, +) -> JsonValue: + """Creates a "typed array", where all elements of the array have the same element type. For example, the JSON array + + .. code-block:: json + [1, 42, 43, 13] + + might be modelled as an array of ints + + .. code-block:: python + array_val(int) + + In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can + simplify by using a list with a single element type. For example, + + .. code-block:: python + some_method(array_val(element)) + + could be simplified to + + .. code-block:: python + some_method([element]) + + Args: + element (JsonValueType): the array element type + allow_missing (bool): if the array is allowed to be missing, by default is True + allow_null (bool): if the array is allowed to be a JSON null type, by default is True + + Returns: + the array value + """ + builder = _JArrayValue.builder() + builder.element(json_val(element).j_value) + _build(builder, allow_missing, allow_null, allow_array=True) + return JsonValue(builder.build()) + + +def object_entries_val( + value_type: JsonValueType, + key_type: JsonValueType = str, + allow_missing: bool = True, + allow_null: bool = True, +) -> JsonValue: + """Creates an object entries value. This is used in situations where the number of fields in an object is + variable and all the values types are the same. For example, the JSON object + + .. code-block:: json + { + "foo": 1, + "bar": 42, + "baz": 3, + ... + "xyz": 100 + } + + might be modelled as the object kv type + + .. code-block:: python + object_entries_val(int) + + Args: + value_type (JsonValueType): the value type element, required + key_type (JsonValueType): the key type element, by default is type str + allow_missing (bool): if the object is allowed to be missing, by default is True + allow_null (bool): if the object is allowed to be a JSON null type, by default is True + + Returns: + the object entries value + """ + builder = _JObjectEntriesValue.builder() + builder.key(json_val(key_type).j_value) + builder.value(json_val(value_type).j_value) + _build(builder, allow_missing, allow_null, allow_object=True) + return JsonValue(builder.build()) + + +def tuple_val( + values: Union[Tuple[JsonValueType, ...], Dict[str, JsonValueType]], + allow_missing: bool = True, + allow_null: bool = True, +) -> JsonValue: + """Creates a tuple value. For example, the JSON array + + .. code-block:: json + ["foo", 42, 5.72] + + might be modelled as the tuple type + + .. code-block:: python + tuple_val((str, int, float)) + + To provide meaningful names, a dictionary can be used: + + .. code-block:: python + tuple_val({"name": str, "age": int, "height": float}) + + otherwise, default names based on the indexes of the values will be used. + + In contexts where the user needs to create a JsonValueType and isn't changing any default values nor is setting + names, the user can simplify passing through a python tuple type. For example, + + .. code-block:: python + some_method(tuple_val((tuple_type_1, tuple_type_2))) + + could be simplified to + + .. code-block:: python + some_method((tuple_type_1, tuple_type_2)) + + Args: + values (Union[Tuple[JsonValueType, ...], Dict[str, JsonValueType]]): the tuple value types + allow_missing (bool): if the array is allowed to be missing, by default is True + allow_null (bool): if the array is allowed to be a JSON null type, by default is True + Returns: + the tuple value + """ + if isinstance(values, Tuple): + kvs = enumerate(values) + elif isinstance(values, Dict): + kvs = values.items() + else: + raise TypeError(f"Invalid tuple type: {type(values)}") + builder = _JTupleValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_array=True, + ) + for name, json_value_type in kvs: + builder.putNamedValues(str(name), json_val(json_value_type).j_value) + return JsonValue(builder.build()) + + +def bool_val( + allow_string: bool = False, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[bool] = None, + on_null: Optional[bool] = None, +) -> JsonValue: + """Creates a bool value. For example, the JSON boolean + + .. code-block:: json + True + + might be modelled as the bool type + + .. code-block:: python + bool_val() + + In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can + simplify by using the python built-in bool type. For example, + + .. code-block:: python + some_method(bool_val()) + + could be simplified to + + .. code-block:: python + some_method(bool) + + Args: + allow_string (bool): if the bool value is allowed to be a JSON string type, default is False + allow_missing (bool): if the bool value is allowed to be missing, default is True + allow_null (bool): if the bool value is allowed to be a JSON null type, default is True + on_missing (Optional[bool]): the value to use when the JSON value is missing and allow_missing is True, default is None + on_null (Optional[bool]): the value to use when the JSON value is null and allow_null is True, default is None + + Returns: + the bool value + """ + builder = _JBoolValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_bool=True, + allow_string=allow_string, + ) + if on_null: + builder.onNull(on_null) + if on_missing: + builder.onMissing(on_missing) + return JsonValue(builder.build()) + + +def char_val( + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[str] = None, + on_null: Optional[str] = None, +) -> JsonValue: + """Creates a char value. For example, the JSON string + + .. code-block:: json + "F" + + might be modelled as the char type + + .. code-block:: python + char_val() + + Args: + allow_missing (bool): if the char value is allowed to be missing, default is True + allow_null (bool): if the char value is allowed to be a JSON null type, default is True + on_missing (Optional[str]): the value to use when the JSON value is missing and allow_missing is True, default is None. If specified, must be a single character. + on_null (Optional[str]): the value to use when the JSON value is null and allow_null is True, default is None. If specified, must be a single character. + + Returns: + the char value + """ + builder = _JCharValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_string=True, + ) + if on_null: + builder.onNull(ord(on_null)) + if on_missing: + builder.onMissing(ord(on_missing)) + return JsonValue(builder.build()) + + +def byte_val( + allow_decimal: bool = False, + allow_string: bool = False, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[int] = None, + on_null: Optional[int] = None, +) -> JsonValue: + """Creates a byte (signed 8-bit) value. For example, the JSON integer + + .. code-block:: json + 42 + + might be modelled as the byte type + + .. code-block:: python + byte_val() + + Args: + allow_decimal (bool): if the byte value is allowed to be a JSON decimal type, default is False + allow_string (bool): if the byte value is allowed to be a JSON string type, default is False + allow_missing (bool): if the byte value is allowed to be missing, default is True + allow_null (bool): if the byte value is allowed to be a JSON null type, default is True + on_missing (Optional[int]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[int]): the value to use when the JSON value is null and allow_null is True, default is None. + + Returns: + the byte value + """ + builder = _JByteValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_int=True, + allow_decimal=allow_decimal, + allow_string=allow_string, + ) + if on_null: + builder.onNull(on_null) + if on_missing: + builder.onMissing(on_missing) + return JsonValue(builder.build()) + + +def short_val( + allow_decimal: bool = False, + allow_string: bool = False, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[int] = None, + on_null: Optional[int] = None, +) -> JsonValue: + """Creates a short (signed 16-bit) value. For example, the JSON integer + + .. code-block:: json + 30000 + + might be modelled as the short type + + .. code-block:: python + short_val() + + Args: + allow_decimal (bool): if the short value is allowed to be a JSON decimal type, default is False + allow_string (bool): if the short value is allowed to be a JSON string type, default is False + allow_missing (bool): if the short value is allowed to be missing, default is True + allow_null (bool): if the short value is allowed to be a JSON null type, default is True + on_missing (Optional[int]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[int]): the value to use when the JSON value is null and allow_null is True, default is None. + + Returns: + the short value + """ + builder = _JShortValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_int=True, + allow_decimal=allow_decimal, + allow_string=allow_string, + ) + if on_null: + builder.onNull(on_null) + if on_missing: + builder.onMissing(on_missing) + return JsonValue(builder.build()) + + +def int_val( + allow_decimal: bool = False, + allow_string: bool = False, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[int] = None, + on_null: Optional[int] = None, +) -> JsonValue: + """Creates an int (signed 32-bit) value. For example, the JSON integer + + .. code-block:: json + 100000 + + might be modelled as the int type + + .. code-block:: python + int_val() + + Args: + allow_decimal (bool): if the int value is allowed to be a JSON decimal type, default is False + allow_string (bool): if the int value is allowed to be a JSON string type, default is False + allow_missing (bool): if the int value is allowed to be missing, default is True + allow_null (bool): if the int value is allowed to be a JSON null type, default is True + on_missing (Optional[int]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[int]): the value to use when the JSON value is null and allow_null is True, default is None. + + Returns: + the int value + """ + builder = _JIntValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_int=True, + allow_decimal=allow_decimal, + allow_string=allow_string, + ) + if on_null: + builder.onNull(on_null) + if on_missing: + builder.onMissing(on_missing) + return JsonValue(builder.build()) + + +def long_val( + allow_decimal: bool = False, + allow_string: bool = False, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[int] = None, + on_null: Optional[int] = None, +) -> JsonValue: + """Creates a long (signed 64-bit) value. For example, the JSON integer + + .. code-block:: json + 8000000000 + + might be modelled as the long type + + .. code-block:: python + long_val() + + In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can + simplify by using the python built-in long type. For example, + + .. code-block:: python + some_method(long_val()) + + could be simplified to + + .. code-block:: python + some_method(int) + + Args: + allow_decimal (bool): if the long value is allowed to be a JSON decimal type, default is False + allow_string (bool): if the long value is allowed to be a JSON string type, default is False + allow_missing (bool): if the long value is allowed to be missing, default is True + allow_null (bool): if the long value is allowed to be a JSON null type, default is True + on_missing (Optional[int]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[int]): the value to use when the JSON value is null and allow_null is True, default is None. + + Returns: + the long value + """ + builder = _JLongValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_int=True, + allow_decimal=allow_decimal, + allow_string=allow_string, + ) + if on_null: + builder.onNull(on_null) + if on_missing: + builder.onMissing(on_missing) + return JsonValue(builder.build()) + + +def float_val( + allow_string: bool = False, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[float] = None, + on_null: Optional[float] = None, +) -> JsonValue: + """Creates a float (signed 32-bit) value. For example, the JSON decimal + + .. code-block:: json + 42.42 + + might be modelled as the float type + + .. code-block:: python + float_val() + + Args: + allow_string (bool): if the float value is allowed to be a JSON string type, default is False + allow_missing (bool): if the float value is allowed to be missing, default is True + allow_null (bool): if the float value is allowed to be a JSON null type, default is True + on_missing (Optional[float]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[float]): the value to use when the JSON value is null and allow_null is True, default is None. + + Returns: + the float value + """ + builder = _JFloatValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_decimal=True, + allow_int=True, + allow_string=allow_string, + ) + if on_null: + builder.onNull(on_null) + if on_missing: + builder.onMissing(on_missing) + return JsonValue(builder.build()) + + +def double_val( + allow_string: bool = False, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[float] = None, + on_null: Optional[float] = None, +) -> JsonValue: + """Creates a double (signed 64-bit) value. For example, the JSON decimal + + .. code-block:: json + 42.42424242 + + might be modelled as the double type + + .. code-block:: python + double_val() + + In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can + simplify by using the python built-in float type. For example, + + .. code-block:: python + some_method(double_val()) + + could be simplified to + + .. code-block:: python + some_method(float) + + Args: + allow_string (bool): if the double value is allowed to be a JSON string type, default is False + allow_missing (bool): if the double value is allowed to be missing, default is True + allow_null (bool): if the double value is allowed to be a JSON null type, default is True + on_missing (Optional[int]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[int]): the value to use when the JSON value is null and allow_null is True, default is None. + + Returns: + the double value + """ + builder = _JDoubleValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_decimal=True, + allow_int=True, + allow_string=allow_string, + ) + if on_null: + builder.onNull(on_null) + if on_missing: + builder.onMissing(on_missing) + return JsonValue(builder.build()) + + +def string_val( + allow_int: bool = False, + allow_decimal: bool = False, + allow_bool: bool = False, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[str] = None, + on_null: Optional[str] = None, +) -> JsonValue: + """Creates a String value. For example, the JSON string + + .. code-block:: json + "Hello, world!" + + might be modelled as the string type + + .. code-block:: python + string_val() + + In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can + simplify by using the python built-in str type. For example, + + .. code-block:: python + some_method(string_val()) + + could be simplified to + + .. code-block:: python + some_method(str) + + Args: + allow_int (bool): if the string value is allowed to be a JSON integer type, default is False + allow_decimal (bool): if the string value is allowed to be a JSON decimal type, default is False + allow_bool (bool): if the string value is allowed to be a JSON boolean type, default is False + allow_missing (bool): if the double value is allowed to be missing, default is True + allow_null (bool): if the double value is allowed to be a JSON null type, default is True + on_missing (Optional[str]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[str]): the value to use when the JSON value is null and allow_null is True, default is None. + + Returns: + the String value + """ + builder = _JStringValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_string=True, + allow_int=allow_int, + allow_decimal=allow_decimal, + allow_bool=allow_bool, + ) + if on_null: + builder.onNull(on_null) + if on_missing: + builder.onMissing(on_missing) + return JsonValue(builder.build()) + + +# TODO(deephaven-core#5269): Create deephaven.time time-type aliases +def instant_val( + allow_missing: bool = True, + allow_null: bool = True, + number_format: Literal[None, "s", "ms", "us", "ns"] = None, + allow_decimal: bool = False, + on_missing: Optional[Any] = None, + on_null: Optional[Any] = None, +) -> JsonValue: + """Creates an Instant value. For example, the JSON string + + .. code-block:: json + "2009-02-13T23:31:30.123456789Z" + + might be modelled as the Instant type + + .. code-block:: python + instant_val() + + In another example, the JSON decimal + + .. code-block:: json + 1234567890.123456789 + + might be modelled as the Instant type + + .. code-block:: python + instant_val(number_format="s", allow_decimal=True) + + In contexts where the user needs to create a JsonValueType and isn't changing any default values, the user can + simplify by using the python datetime.datetime type. For example, + + .. code-block:: python + some_method(instant_val()) + + could be simplified to + + .. code-block:: python + some_method(datetime.datetime) + + Args: + allow_missing (bool): if the Instant value is allowed to be missing, default is True + allow_null (bool): if the Instant value is allowed to be a JSON null type, default is True + number_format (Literal[None, "s", "ms", "us", "ns"]): when set, signifies that a JSON numeric type is expected. + "s" is for seconds, "ms" is for milliseconds, "us" is for microseconds, and "ns" is for nanoseconds since + the epoch. When not set, a JSON string in the ISO-8601 format is expected. + allow_decimal (bool): if the Instant value is allowed to be a JSON decimal type, default is False. Only valid + when number_format is specified. + on_missing (Optional[Any]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[Any]): the value to use when the JSON value is null and allow_null is True, default is None. + + Returns: + the Instant value + """ + if number_format: + builder = _JInstantNumberValue.builder() + if on_missing: + builder.onMissing(to_j_instant(on_missing)) + if on_null: + builder.onNull(to_j_instant(on_null)) + _build( + builder, + allow_missing, + allow_null, + allow_int=True, + allow_decimal=allow_decimal, + ) + if number_format == "s": + builder.format(_EPOCH_SECONDS) + elif number_format == "ms": + builder.format(_EPOCH_MILLIS) + elif number_format == "us": + builder.format(_EPOCH_MICROS) + elif number_format == "ns": + builder.format(_EPOCH_NANOS) + else: + raise TypeError(f"Invalid number format: {number_format}") + return JsonValue(builder.build()) + else: + if allow_decimal: + raise TypeError("allow_decimal is only valid when using number_format") + builder = _JInstantValue.builder() + if on_missing: + builder.onMissing(to_j_instant(on_missing)) + if on_null: + builder.onNull(to_j_instant(on_null)) + _build( + builder, + allow_missing, + allow_null, + allow_string=True, + ) + return JsonValue(builder.build()) + + +def big_integer_val( + allow_string: bool = False, + allow_decimal: bool = False, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[Union[int, str]] = None, + on_null: Optional[Union[int, str]] = None, +) -> JsonValue: + """Creates a BigInteger value. For example, the JSON integer + + .. code-block:: json + 123456789012345678901 + + might be modelled as the BigInteger type + + .. code-block:: python + big_integer_val() + + Args: + allow_string (bool): if the BigInteger value is allowed to be a JSON string type, default is False. + allow_decimal (bool): if the BigInteger value is allowed to be a JSON decimal type, default is False. + allow_missing (bool): if the BigInteger value is allowed to be missing, default is True + allow_null (bool): if the BigInteger value is allowed to be a JSON null type, default is True + on_missing (Optional[Union[int, str]]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[Union[int, str]]): the value to use when the JSON value is null and allow_null is True, default is None. + + Returns: + the BigInteger value + """ + builder = _JBigIntegerValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_int=True, + allow_decimal=allow_decimal, + allow_string=allow_string, + ) + if on_missing: + builder.onMissing(dtypes.BigInteger(str(on_missing))) + if on_null: + builder.onNull(dtypes.BigInteger(str(on_null))) + return JsonValue(builder.build()) + + +def big_decimal_val( + allow_string: bool = False, + allow_missing: bool = True, + allow_null: bool = True, + on_missing: Optional[Union[float, str]] = None, + on_null: Optional[Union[float, str]] = None, +) -> JsonValue: + """Creates a BigDecimal value. For example, the JSON decimal + + .. code-block:: json + 123456789012345678901.42 + + might be modelled as the BigDecimal type + + .. code-block:: python + big_decimal_val() + + Args: + allow_string (bool): if the BigDecimal value is allowed to be a JSON string type, default is False. + allow_missing (bool): if the BigDecimal value is allowed to be missing, default is True + allow_null (bool): if the BigDecimal value is allowed to be a JSON null type, default is True + on_missing (Optional[Union[float, str]]): the value to use when the JSON value is missing and allow_missing is True, default is None. + on_null (Optional[Union[float, str]]): the value to use when the JSON value is null and allow_null is True, default is None. + + Returns: + the BigDecimal value + """ + builder = _JBigDecimalValue.builder() + _build( + builder, + allow_missing, + allow_null, + allow_int=True, + allow_decimal=True, + allow_string=allow_string, + ) + if on_missing: + builder.onMissing(dtypes.BigDecimal(str(on_missing))) + if on_null: + builder.onNull(dtypes.BigDecimal(str(on_null))) + return JsonValue(builder.build()) + + +def any_val() -> JsonValue: + """Creates an "any" value. The resulting type is implementation dependant. + + Returns: + the "any" value + """ + return JsonValue(_JAnyValue.of()) + + +def skip_val( + allow_missing: Optional[bool] = None, + allow_null: Optional[bool] = None, + allow_int: Optional[bool] = None, + allow_decimal: Optional[bool] = None, + allow_string: Optional[bool] = None, + allow_bool: Optional[bool] = None, + allow_object: Optional[bool] = None, + allow_array: Optional[bool] = None, + allow_by_default: bool = True, +) -> JsonValue: + """Creates a "skip" value. No resulting type will be returned, but the JSON types will be validated as configured. + This may be useful in combination with an object type where allow_unknown_fields=False. For example, the JSON object + + .. code-block:: json + { "name": "foo", "age": 42 } + + might be modelled as the object type + + .. code-block:: python + object_val({ "name": str, "age": skip_val() }, allow_unknown_fields=False) + + Args: + allow_missing (Optional[bool]): if a missing JSON value is allowed, by default is None + allow_null (Optional[bool]): if a JSON null type is allowed, by default is None + allow_int (Optional[bool]): if a JSON integer type is allowed, by default is None + allow_decimal (Optional[bool]): if a JSON decimal type is allowed, by default is None + allow_string (Optional[bool]): if a JSON string type is allowed, by default is None + allow_bool (Optional[bool]): if a JSON boolean type is allowed, by default is None + allow_object (Optional[bool]): if a JSON object type is allowed, by default is None + allow_array (Optional[bool]): if a JSON array type is allowed, by default is None + allow_by_default (bool): the default behavior for the other arguments when they are set to None, by default is True + + Returns: + the "skip" value + """ + + def _allow(x: Optional[bool]) -> bool: + return x if x is not None else allow_by_default + + builder = _JSkipValue.builder() + _build( + builder, + allow_missing=_allow(allow_missing), + allow_null=_allow(allow_null), + allow_int=_allow(allow_int), + allow_decimal=_allow(allow_decimal), + allow_string=_allow(allow_string), + allow_bool=_allow(allow_bool), + allow_object=_allow(allow_object), + allow_array=_allow(allow_array), + ) + return JsonValue(builder.build()) + + +_dtype_dict = { + dtypes.bool_: bool_val(), + dtypes.char: char_val(), + dtypes.int8: byte_val(), + dtypes.int16: short_val(), + dtypes.int32: int_val(), + dtypes.int64: long_val(), + dtypes.float32: float_val(), + dtypes.float64: double_val(), + dtypes.string: string_val(), + dtypes.Instant: instant_val(), + dtypes.BigInteger: big_integer_val(), + dtypes.BigDecimal: big_decimal_val(), + dtypes.JObject: any_val(), + dtypes.bool_array: array_val(bool_val()), + dtypes.char_array: array_val(char_val()), + dtypes.int8_array: array_val(byte_val()), + dtypes.int16_array: array_val(short_val()), + dtypes.int32_array: array_val(int_val()), + dtypes.int64_array: array_val(long_val()), + dtypes.float32_array: array_val(float_val()), + dtypes.float64_array: array_val(double_val()), + dtypes.string_array: array_val(string_val()), + dtypes.instant_array: array_val(instant_val()), +} + +_type_dict = { + bool: bool_val(), + int: long_val(), + float: double_val(), + str: string_val(), + datetime: instant_val(), + object: any_val(), +} diff --git a/py/server/deephaven/json/jackson.py b/py/server/deephaven/json/jackson.py new file mode 100644 index 00000000000..4b206df6900 --- /dev/null +++ b/py/server/deephaven/json/jackson.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +# + +"""A JSON processor provider implementation using Jackson (https://github.com/FasterXML/jackson).""" + +import jpy + +from typing import Optional + +from . import JsonValueType, json_val + + +def provider( + json_value_type: JsonValueType, factory: Optional[jpy.JType] = None +) -> jpy.JType: + """Creates a Jackson JSON named object processor provider. + + Args: + json_value_type (JsonValueType): the JSON value + factory (Optional[jpy.JType]): the factory (java type "com.fasterxml.jackson.core.JsonFactory"), by default is + None which will use a default factory + + Returns: + the jackson JSON named object processor provider + """ + _JProvider = jpy.get_type("io.deephaven.json.jackson.JacksonProvider") + return ( + _JProvider.of(json_val(json_value_type).j_value, factory) + if factory + else _JProvider.of(json_val(json_value_type).j_value) + ) diff --git a/py/server/deephaven/stream/kafka/consumer.py b/py/server/deephaven/stream/kafka/consumer.py index 35f8623648a..01d33e7463b 100644 --- a/py/server/deephaven/stream/kafka/consumer.py +++ b/py/server/deephaven/stream/kafka/consumer.py @@ -483,3 +483,19 @@ def simple_spec(col_name: str, data_type: DType = None) -> KeyValueSpec: ) except Exception as e: raise DHError(e, "failed to create a Kafka key/value spec") from e + + +def object_processor_spec(provider: jpy.JType) -> KeyValueSpec: + """Creates a kafka key-value spec implementation from a named object processor provider. It must be capable of + supporting a byte array. + + Args: + provider (jpy.JType): the named object processor provider + + Returns: + a KeyValueSpec + + Raises: + DHError + """ + return KeyValueSpec(j_spec=_JKafkaTools_Consume.objectProcessorSpec(provider)) diff --git a/py/server/tests/test_dtypes.py b/py/server/tests/test_dtypes.py index 5058dda1143..0dfbaf3bc2f 100644 --- a/py/server/tests/test_dtypes.py +++ b/py/server/tests/test_dtypes.py @@ -46,6 +46,7 @@ def test_j_type(self): self.assertEqual(dtypes.float64.j_type, jpy.get_type("double")) self.assertEqual(dtypes.double.j_type, jpy.get_type("double")) self.assertEqual(dtypes.string.j_type, jpy.get_type("java.lang.String")) + self.assertEqual(dtypes.BigInteger.j_type, jpy.get_type("java.math.BigInteger")) self.assertEqual(dtypes.BigDecimal.j_type, jpy.get_type("java.math.BigDecimal")) self.assertEqual(dtypes.StringSet.j_type, jpy.get_type("io.deephaven.stringset.StringSet")) self.assertEqual(dtypes.Instant.j_type, jpy.get_type("java.time.Instant")) @@ -70,6 +71,7 @@ def test_np_type(self): self.assertEqual(dtypes.float64.np_type, np.float64) self.assertEqual(dtypes.double.np_type, np.float64) self.assertEqual(dtypes.string.np_type, np.str_) + self.assertEqual(dtypes.BigInteger.np_type, np.object_) self.assertEqual(dtypes.BigDecimal.np_type, np.object_) self.assertEqual(dtypes.StringSet.np_type, np.object_) self.assertEqual(dtypes.Instant.np_type, np.dtype("datetime64[ns]")) diff --git a/py/server/tests/test_json.py b/py/server/tests/test_json.py new file mode 100644 index 00000000000..cc10c586a20 --- /dev/null +++ b/py/server/tests/test_json.py @@ -0,0 +1,172 @@ +# +# Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +# + +import unittest + +from datetime import datetime +from tests.testbase import BaseTestCase + +from deephaven import dtypes +from deephaven.json import * + + +def all_equals(items) -> bool: + return len(set(items)) <= 1 + + +class JsonTestCase(BaseTestCase): + def all_same_json_internal(self, items): + self.assertTrue(all_equals([json_val(x) for x in items])) + + def all_same_json(self, items): + self.all_same_json_internal(items) + # The following might be seen as redundant special cases, but worthwhile for a bit of extra coverage + self.all_same_json_internal( + [array_val(x) for x in items] + [[x] for x in items] + ) + self.all_same_json_internal([object_entries_val(x) for x in items]) + self.all_same_json_internal([object_val({"Foo": x}) for x in items]) + self.all_same_json_internal( + [tuple_val((x,)) for x in items] + [(x,) for x in items] + ) + self.all_same_json_internal([tuple_val({"Bar": x}) for x in items]) + self.all_same_json_internal( + [array_val(array_val(x)) for x in items] + [[[x]] for x in items] + ) + self.all_same_json_internal([object_entries_val(array_val(x)) for x in items]) + + def test_bool(self): + self.all_same_json([bool_val(), dtypes.bool_, bool]) + with self.subTest("on_missing"): + bool_val(on_missing=False) + with self.subTest("on_null"): + bool_val(on_null=False) + + def test_char(self): + self.all_same_json([char_val(), dtypes.char]) + with self.subTest("on_missing"): + char_val(on_missing="m") + with self.subTest("on_null"): + char_val(on_null="n") + + def test_byte(self): + self.all_same_json([byte_val(), dtypes.byte]) + with self.subTest("on_missing"): + byte_val(on_missing=-1) + with self.subTest("on_null"): + byte_val(on_null=-1) + + def test_short(self): + self.all_same_json([short_val(), dtypes.short]) + with self.subTest("on_missing"): + short_val(on_missing=-1) + with self.subTest("on_null"): + short_val(on_null=-1) + + def test_int(self): + self.all_same_json([int_val(), dtypes.int32]) + with self.subTest("on_missing"): + int_val(on_missing=-1) + with self.subTest("on_null"): + int_val(on_null=-1) + + def test_long(self): + self.all_same_json([long_val(), dtypes.long, int]) + with self.subTest("on_missing"): + long_val(on_missing=-1) + with self.subTest("on_null"): + long_val(on_null=-1) + + def test_float(self): + self.all_same_json([float_val(), dtypes.float32]) + with self.subTest("on_missing"): + float_val(on_missing=-1.0) + with self.subTest("on_null"): + float_val(on_null=-1.0) + + def test_double(self): + self.all_same_json([double_val(), dtypes.double, float]) + with self.subTest("on_missing"): + double_val(on_missing=-1.0) + with self.subTest("on_null"): + double_val(on_null=-1.0) + + def test_string(self): + self.all_same_json([string_val(), dtypes.string, str]) + with self.subTest("on_missing"): + string_val(on_missing="(missing)") + with self.subTest("on_null"): + string_val(on_null="(null)") + + def test_instant(self): + self.all_same_json([instant_val(), dtypes.Instant, datetime]) + with self.subTest("on_missing"): + instant_val(on_missing=datetime.fromtimestamp(0)) + with self.subTest("on_null"): + instant_val(on_null=datetime.fromtimestamp(0)) + + def test_any(self): + self.all_same_json([any_val(), dtypes.JObject, object]) + + def test_big_integer(self): + self.all_same_json([big_integer_val(), dtypes.BigInteger]) + with self.subTest("on_missing"): + big_integer_val(on_missing=123456789012345678901234567890) + with self.subTest("on_null"): + big_integer_val(on_null=123456789012345678901234567890) + + def test_big_decimal(self): + self.all_same_json([big_decimal_val(), dtypes.BigDecimal]) + with self.subTest("on_missing"): + big_decimal_val(on_missing="123456789012345678901234567890.999999999999") + with self.subTest("on_null"): + big_decimal_val(on_null="123456789012345678901234567890.999999999999") + + def test_object(self): + e1 = [ + {"name": str, "age": int}, + {"name": string_val(), "age": long_val()}, + {"name": ObjectField(str), "age": ObjectField(int)}, + {"name": ObjectField(string_val()), "age": ObjectField(long_val())}, + ] + e2 = [object_val(x) for x in e1] + self.all_same_json(e1 + e2) + + def test_array(self): + self.all_same_json( + [ + array_val(int), + array_val(long_val()), + [int], + [long_val()], + ] + ) + + def test_tuple(self): + e1 = [(str, int), (string_val(), long_val())] + e2 = [tuple_val(x) for x in e1] + self.all_same_json(e1 + e2) + + with self.subTest("named tuple"): + self.all_same_json( + [ + tuple_val({"name": str, "age": int}), + tuple_val({"name": string_val(), "age": long_val()}), + ] + ) + + def test_object_entries(self): + self.all_same_json( + [ + object_entries_val(int), + object_entries_val(long_val()), + ] + ) + + def test_skip(self): + self.all_same_json([skip_val()]) + + +if __name__ == "__main__": + unittest.main() diff --git a/py/server/tests/test_kafka_consumer.py b/py/server/tests/test_kafka_consumer.py index 91c2990f4e7..35a32fc1cc0 100644 --- a/py/server/tests/test_kafka_consumer.py +++ b/py/server/tests/test_kafka_consumer.py @@ -4,12 +4,13 @@ import os import unittest +from datetime import datetime from deephaven import kafka_consumer as ck from deephaven.stream.kafka.consumer import TableType, KeyValueSpec from tests.testbase import BaseTestCase from deephaven import dtypes - +from deephaven.json.jackson import provider as jackson_provider class KafkaConsumerTestCase(BaseTestCase): @@ -80,7 +81,13 @@ def test_json_spec(self): 'jqty': 'Qty', 'jts': 'Tstamp' } - )] + ), ck.object_processor_spec(jackson_provider({ + 'Symbol': str, + 'Side': str, + 'Price': float, + 'Qty': int, + 'Tstamp': datetime + }))] for value_spec in value_specs: t = ck.consume( diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateHashing.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateHashing.java index 0d5e0a50835..5268c3c6697 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateHashing.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateHashing.java @@ -48,6 +48,10 @@ public static void main(String[] args) throws IOException { charToObject(TASK, "engine/chunk/src/main/java/io/deephaven/chunk/util/hashing/CharChunkEquals.java"); fixupObjectChunkIdentityEquals(objectIdentityEquals); + final String objectDeepEquals = + charToObject(TASK, "engine/chunk/src/main/java/io/deephaven/chunk/util/hashing/CharChunkEquals.java"); + fixupObjectChunkDeepEquals(objectDeepEquals); + final String objectEquals = charToObject(TASK, "engine/chunk/src/main/java/io/deephaven/chunk/util/hashing/CharChunkEquals.java"); fixupObjectChunkEquals(objectEquals); @@ -146,6 +150,27 @@ private static void fixupObjectChunkIdentityEquals(String objectPath) throws IOE "name", "ObjectChunkEquals", "ObjectChunkIdentityEquals")); } + private static void fixupObjectChunkDeepEquals(String objectPath) throws IOException { + final File objectChunkEqualsFileName = new File(objectPath); + final File objectChunkIdentifyEqualsFileName = + new File(objectChunkEqualsFileName.getParent(), "ObjectChunkDeepEquals.java"); + Assert.eqTrue(objectChunkEqualsFileName.renameTo(objectChunkIdentifyEqualsFileName), + "objectChunkEqualsFileName.renameTo(objectChunkIdentifyEqualsFileName)"); + + { + List lines = FileUtils.readLines(objectChunkIdentifyEqualsFileName, Charset.defaultCharset()); + lines = addImport(lines, Objects.class); + FileUtils.writeLines(objectChunkIdentifyEqualsFileName, simpleFixup(fixupChunkAttributes(lines), + "name", "ObjectChunkEquals", "ObjectChunkDeepEquals")); + } + { + final List lines = FileUtils.readLines(objectChunkIdentifyEqualsFileName, Charset.defaultCharset()); + FileUtils.writeLines(objectChunkIdentifyEqualsFileName, simpleFixup(lines, + "eq", "lhs == rhs", "Objects.deepEquals(lhs, rhs)")); + } + + } + private static void fixupDoubleChunkEquals(String doublePath) throws IOException { final File objectFile = new File(doublePath); final List lines = FileUtils.readLines(objectFile, Charset.defaultCharset()); diff --git a/server/jetty-app/build.gradle b/server/jetty-app/build.gradle index 39bb137af16..670c7cfb982 100644 --- a/server/jetty-app/build.gradle +++ b/server/jetty-app/build.gradle @@ -58,6 +58,12 @@ if (!hasProperty('excludeS3')) { } } +if (!hasProperty('excludeJson')) { + dependencies { + runtimeOnly project(':extensions-json-jackson') + } +} + def authHandlers = [] def authConfigs = ['AuthHandlers'] if (hasProperty('anonymous')) { diff --git a/settings.gradle b/settings.gradle index 9a240a942db..d381956ca88 100644 --- a/settings.gradle +++ b/settings.gradle @@ -273,6 +273,15 @@ project(':extensions-iceberg').projectDir = file('extensions/iceberg') include(':extensions-iceberg:s3') project(':extensions-iceberg:s3').projectDir = file('extensions/iceberg/s3') +include(':extensions-json') +project(':extensions-json').projectDir = file('extensions/json') + +include(':extensions-json-jackson') +project(':extensions-json-jackson').projectDir = file('extensions/json-jackson') + +include(':extensions-bson-jackson') +project(':extensions-bson-jackson').projectDir = file('extensions/bson-jackson') + include(':plugin') include(':plugin-dagger')