From c9bb557a76ec4589de88790bd43a2d39a247157d Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Fri, 1 Nov 2024 10:13:15 -0700 Subject: [PATCH] feat: add multi-column support to `UpdateBy` `RollingFormula()` operator (#6143) --- .../ringbuffer/AggregatingByteRingBuffer.java | 9 +- .../ringbuffer/AggregatingCharRingBuffer.java | 9 +- .../AggregatingDoubleRingBuffer.java | 9 +- .../AggregatingFloatRingBuffer.java | 9 +- .../ringbuffer/AggregatingIntRingBuffer.java | 9 +- .../ringbuffer/AggregatingLongRingBuffer.java | 9 +- .../AggregatingObjectRingBuffer.java | 9 +- .../AggregatingShortRingBuffer.java | 9 +- .../base/ringbuffer/ByteRingBuffer.java | 50 ++- .../base/ringbuffer/CharRingBuffer.java | 50 ++- .../base/ringbuffer/DoubleRingBuffer.java | 50 ++- .../base/ringbuffer/FloatRingBuffer.java | 50 ++- .../base/ringbuffer/IntRingBuffer.java | 50 ++- .../base/ringbuffer/LongRingBuffer.java | 50 ++- .../base/ringbuffer/ObjectRingBuffer.java | 58 ++- .../deephaven/base/ringbuffer/RingBuffer.java | 79 ++++ .../base/ringbuffer/ShortRingBuffer.java | 50 ++- ...ava => TestAggregatingByteRingBuffer.java} | 4 +- ...ava => TestAggregatingCharRingBuffer.java} | 2 +- ...a => TestAggregatingDoubleRingBuffer.java} | 4 +- ...va => TestAggregatingFloatRingBuffer.java} | 4 +- ...java => TestAggregatingIntRingBuffer.java} | 4 +- ...ava => TestAggregatingLongRingBuffer.java} | 4 +- ...a => TestAggregatingObjectRingBuffer.java} | 2 +- ...va => TestAggregatingShortRingBuffer.java} | 4 +- ...ufferTest.java => TestByteRingBuffer.java} | 6 +- ...ufferTest.java => TestCharRingBuffer.java} | 4 +- ...ferTest.java => TestDoubleRingBuffer.java} | 6 +- ...fferTest.java => TestFloatRingBuffer.java} | 6 +- ...BufferTest.java => TestIntRingBuffer.java} | 6 +- ...ufferTest.java => TestLongRingBuffer.java} | 6 +- ...ferTest.java => TestObjectRingBuffer.java} | 9 +- ...fferTest.java => TestShortRingBuffer.java} | 6 +- .../impl/select/AbstractFormulaColumn.java | 5 + .../table/impl/select/SelectColumn.java | 9 +- .../impl/sources/ObjectSingleValueSource.java | 4 +- .../impl/sources/SingleValueColumnSource.java | 16 +- .../table/impl/updateby/UpdateByOperator.java | 3 +- .../updateby/UpdateByOperatorFactory.java | 98 +++- .../BigIntegerRollingAvgOperator.java | 1 - .../BaseRollingFormulaOperator.java | 64 ++- .../BooleanRollingFormulaOperator.java | 23 +- .../ByteRollingFormulaOperator.java | 34 +- .../CharRollingFormulaOperator.java | 34 +- .../DoubleRollingFormulaOperator.java | 34 +- .../FloatRollingFormulaOperator.java | 34 +- .../IntRollingFormulaOperator.java | 34 +- .../LongRollingFormulaOperator.java | 34 +- .../ObjectRollingFormulaOperator.java | 25 +- .../ShortRollingFormulaOperator.java | 34 +- .../ByteRingBufferVectorWrapper.java | 2 +- .../CharRingBufferVectorWrapper.java | 2 +- .../DoubleRingBufferVectorWrapper.java | 2 +- .../FloatRingBufferVectorWrapper.java | 2 +- .../IntRingBufferVectorWrapper.java | 2 +- .../LongRingBufferVectorWrapper.java | 2 +- .../ObjectRingBufferVectorWrapper.java | 2 +- .../RingBufferVectorWrapper.java | 40 +- .../ShortRingBufferVectorWrapper.java | 2 +- .../RollingFormulaMultiColumnOperator.java | 419 ++++++++++++++++++ .../ByteRingBufferWindowConsumer.java | 47 ++ .../CharRingBufferWindowConsumer.java | 43 ++ .../DoubleRingBufferWindowConsumer.java | 47 ++ .../FloatRingBufferWindowConsumer.java | 47 ++ .../IntRingBufferWindowConsumer.java | 47 ++ .../LongRingBufferWindowConsumer.java | 47 ++ .../ObjectRingBufferWindowConsumer.java | 43 ++ .../RingBufferWindowConsumer.java | 71 +++ .../ShortRingBufferWindowConsumer.java | 47 ++ .../rollingwavg/ByteRollingWAvgOperator.java | 11 - .../rollingwavg/CharRollingWAvgOperator.java | 11 - .../DoubleRollingWAvgOperator.java | 11 - .../rollingwavg/FloatRollingWAvgOperator.java | 11 - .../rollingwavg/IntRollingWAvgOperator.java | 11 - .../rollingwavg/LongRollingWAvgOperator.java | 11 - .../rollingwavg/ShortRollingWAvgOperator.java | 11 - .../impl/updateby/TestRollingFormula.java | 310 ++++++++++++- .../client/impl/UpdateByBuilder.java | 2 +- py/server/deephaven/updateby.py | 64 ++- py/server/tests/test_updateby.py | 37 +- .../replicators/ReplicateRingBuffers.java | 26 +- .../ReplicateSourcesAndChunks.java | 4 +- .../replicators/ReplicateUpdateBy.java | 8 + .../api/updateby/UpdateByOperation.java | 243 +++++++++- .../api/updateby/spec/RollingFormulaSpec.java | 91 +++- .../api/updateby/spec/RollingOpSpec.java | 28 +- .../api/updateby/spec/UpdateBySpec.java | 7 + .../api/updateby/spec/UpdateBySpecBase.java | 7 + 88 files changed, 2530 insertions(+), 410 deletions(-) create mode 100644 Base/src/main/java/io/deephaven/base/ringbuffer/RingBuffer.java rename Base/src/test/java/io/deephaven/base/ringbuffer/{AggregatingByteRingBufferTest.java => TestAggregatingByteRingBuffer.java} (97%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{AggregatingCharRingBufferTest.java => TestAggregatingCharRingBuffer.java} (99%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{AggregatingDoubleRingBufferTest.java => TestAggregatingDoubleRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{AggregatingFloatRingBufferTest.java => TestAggregatingFloatRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{AggregatingIntRingBufferTest.java => TestAggregatingIntRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{AggregatingLongRingBufferTest.java => TestAggregatingLongRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{AggregatingObjectRingBufferTest.java => TestAggregatingObjectRingBuffer.java} (99%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{AggregatingShortRingBufferTest.java => TestAggregatingShortRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{ByteRingBufferTest.java => TestByteRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{CharRingBufferTest.java => TestCharRingBuffer.java} (99%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{DoubleRingBufferTest.java => TestDoubleRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{FloatRingBufferTest.java => TestFloatRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{IntRingBufferTest.java => TestIntRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{LongRingBufferTest.java => TestLongRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{ObjectRingBufferTest.java => TestObjectRingBuffer.java} (98%) rename Base/src/test/java/io/deephaven/base/ringbuffer/{ShortRingBufferTest.java => TestShortRingBuffer.java} (98%) create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/RollingFormulaMultiColumnOperator.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ByteRingBufferWindowConsumer.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/CharRingBufferWindowConsumer.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/DoubleRingBufferWindowConsumer.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/FloatRingBufferWindowConsumer.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/IntRingBufferWindowConsumer.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/LongRingBufferWindowConsumer.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ObjectRingBufferWindowConsumer.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/RingBufferWindowConsumer.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ShortRingBufferWindowConsumer.java diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingByteRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingByteRingBuffer.java index f4f2ce461a8..5f42dfcd519 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingByteRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingByteRingBuffer.java @@ -434,11 +434,16 @@ public void clear() { final long prevHead = internalBuffer.head; final int prevSize = size(); - internalBuffer.clear(); + // Reset the pointers in the ring buffer without clearing the storage array. This leaves existing `identityVal` + // entries in place for the next `evaluate()` call. + internalBuffer.head = internalBuffer.tail = 0; calcHead = calcTail = 0; - // Reset the cleared storage entries to the identity value + + // Reset the previously populated storage entries to the identity value. After this call, all entries in the + // storage buffer are `identityVal` fillWithIdentityVal(prevHead, prevSize); + // Reset the tree buffer with the identity value Arrays.fill(treeStorage, identityVal); } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingCharRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingCharRingBuffer.java index e931ba3f1c6..f6148c481ed 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingCharRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingCharRingBuffer.java @@ -430,11 +430,16 @@ public void clear() { final long prevHead = internalBuffer.head; final int prevSize = size(); - internalBuffer.clear(); + // Reset the pointers in the ring buffer without clearing the storage array. This leaves existing `identityVal` + // entries in place for the next `evaluate()` call. + internalBuffer.head = internalBuffer.tail = 0; calcHead = calcTail = 0; - // Reset the cleared storage entries to the identity value + + // Reset the previously populated storage entries to the identity value. After this call, all entries in the + // storage buffer are `identityVal` fillWithIdentityVal(prevHead, prevSize); + // Reset the tree buffer with the identity value Arrays.fill(treeStorage, identityVal); } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingDoubleRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingDoubleRingBuffer.java index 516474e31ac..cd5963e54ec 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingDoubleRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingDoubleRingBuffer.java @@ -434,11 +434,16 @@ public void clear() { final long prevHead = internalBuffer.head; final int prevSize = size(); - internalBuffer.clear(); + // Reset the pointers in the ring buffer without clearing the storage array. This leaves existing `identityVal` + // entries in place for the next `evaluate()` call. + internalBuffer.head = internalBuffer.tail = 0; calcHead = calcTail = 0; - // Reset the cleared storage entries to the identity value + + // Reset the previously populated storage entries to the identity value. After this call, all entries in the + // storage buffer are `identityVal` fillWithIdentityVal(prevHead, prevSize); + // Reset the tree buffer with the identity value Arrays.fill(treeStorage, identityVal); } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingFloatRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingFloatRingBuffer.java index fedb32a7ab5..f6d6b43a619 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingFloatRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingFloatRingBuffer.java @@ -434,11 +434,16 @@ public void clear() { final long prevHead = internalBuffer.head; final int prevSize = size(); - internalBuffer.clear(); + // Reset the pointers in the ring buffer without clearing the storage array. This leaves existing `identityVal` + // entries in place for the next `evaluate()` call. + internalBuffer.head = internalBuffer.tail = 0; calcHead = calcTail = 0; - // Reset the cleared storage entries to the identity value + + // Reset the previously populated storage entries to the identity value. After this call, all entries in the + // storage buffer are `identityVal` fillWithIdentityVal(prevHead, prevSize); + // Reset the tree buffer with the identity value Arrays.fill(treeStorage, identityVal); } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingIntRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingIntRingBuffer.java index 9794c76ec0d..b542a07d8d5 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingIntRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingIntRingBuffer.java @@ -434,11 +434,16 @@ public void clear() { final long prevHead = internalBuffer.head; final int prevSize = size(); - internalBuffer.clear(); + // Reset the pointers in the ring buffer without clearing the storage array. This leaves existing `identityVal` + // entries in place for the next `evaluate()` call. + internalBuffer.head = internalBuffer.tail = 0; calcHead = calcTail = 0; - // Reset the cleared storage entries to the identity value + + // Reset the previously populated storage entries to the identity value. After this call, all entries in the + // storage buffer are `identityVal` fillWithIdentityVal(prevHead, prevSize); + // Reset the tree buffer with the identity value Arrays.fill(treeStorage, identityVal); } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingLongRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingLongRingBuffer.java index c402fe5db2f..da3b07750f5 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingLongRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingLongRingBuffer.java @@ -434,11 +434,16 @@ public void clear() { final long prevHead = internalBuffer.head; final int prevSize = size(); - internalBuffer.clear(); + // Reset the pointers in the ring buffer without clearing the storage array. This leaves existing `identityVal` + // entries in place for the next `evaluate()` call. + internalBuffer.head = internalBuffer.tail = 0; calcHead = calcTail = 0; - // Reset the cleared storage entries to the identity value + + // Reset the previously populated storage entries to the identity value. After this call, all entries in the + // storage buffer are `identityVal` fillWithIdentityVal(prevHead, prevSize); + // Reset the tree buffer with the identity value Arrays.fill(treeStorage, identityVal); } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingObjectRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingObjectRingBuffer.java index b064b8a4383..1c6eb1d8c46 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingObjectRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingObjectRingBuffer.java @@ -434,11 +434,16 @@ public void clear() { final long prevHead = internalBuffer.head; final int prevSize = size(); - internalBuffer.clear(); + // Reset the pointers in the ring buffer without clearing the storage array. This leaves existing `identityVal` + // entries in place for the next `evaluate()` call. + internalBuffer.head = internalBuffer.tail = 0; calcHead = calcTail = 0; - // Reset the cleared storage entries to the identity value + + // Reset the previously populated storage entries to the identity value. After this call, all entries in the + // storage buffer are `identityVal` fillWithIdentityVal(prevHead, prevSize); + // Reset the tree buffer with the identity value Arrays.fill(treeStorage, identityVal); } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingShortRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingShortRingBuffer.java index 1e6707cb473..9e4a1c27aca 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingShortRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/AggregatingShortRingBuffer.java @@ -434,11 +434,16 @@ public void clear() { final long prevHead = internalBuffer.head; final int prevSize = size(); - internalBuffer.clear(); + // Reset the pointers in the ring buffer without clearing the storage array. This leaves existing `identityVal` + // entries in place for the next `evaluate()` call. + internalBuffer.head = internalBuffer.tail = 0; calcHead = calcTail = 0; - // Reset the cleared storage entries to the identity value + + // Reset the previously populated storage entries to the identity value. After this call, all entries in the + // storage buffer are `identityVal` fillWithIdentityVal(prevHead, prevSize); + // Reset the tree buffer with the identity value Arrays.fill(treeStorage, identityVal); } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/ByteRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/ByteRingBuffer.java index aede7521309..fa8b5a891d1 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/ByteRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/ByteRingBuffer.java @@ -9,6 +9,7 @@ import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; +import org.jetbrains.annotations.TestOnly; import java.io.Serializable; import java.util.NoSuchElementException; @@ -19,7 +20,7 @@ * {@code long} values. Head and tail will not wrap around; instead we use storage arrays sized to 2^N to allow fast * determination of storage indices through a mask operation. */ -public class ByteRingBuffer implements Serializable { +public class ByteRingBuffer implements RingBuffer, Serializable { static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; byte[] storage; @@ -32,7 +33,7 @@ public class ByteRingBuffer implements Serializable { * * @param capacity minimum capacity of the ring buffer */ - public ByteRingBuffer(int capacity) { + public ByteRingBuffer(final int capacity) { this(capacity, true); } @@ -42,7 +43,7 @@ public ByteRingBuffer(int capacity) { * @param capacity minimum capacity of ring buffer * @param growable whether to allow growth when the buffer is full. */ - public ByteRingBuffer(int capacity, boolean growable) { + public ByteRingBuffer(final int capacity, final boolean growable) { Assert.leq(capacity, "ByteRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; @@ -59,7 +60,7 @@ public ByteRingBuffer(int capacity, boolean growable) { * * @param increase Increase amount. The ring buffer's capacity will be increased by at least this amount. */ - protected void grow(int increase) { + protected void grow(final int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible @@ -83,7 +84,7 @@ protected void grow(int increase) { * * @param dest The destination buffer. */ - protected void copyRingBufferToArray(byte[] dest) { + protected void copyRingBufferToArray(final byte[] dest) { final int size = size(); final int storageHead = (int) (head & mask); @@ -99,27 +100,35 @@ protected void copyRingBufferToArray(byte[] dest) { System.arraycopy(storage, 0, dest, firstCopyLen, secondCopyLen); } + @Override public boolean isFull() { return size() == storage.length; } + @Override public boolean isEmpty() { return tail == head; } + @Override public int size() { return Math.toIntExact(tail - head); } + @Override public int capacity() { return storage.length; } + @Override public int remaining() { return storage.length - size(); } + @Override public void clear() { + // region object-bulk-clear + // endregion object-bulk-clear tail = head = 0; } @@ -131,7 +140,7 @@ public void clear() { * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full * @return {@code true} if the byte was added successfully */ - public boolean add(byte e) { + public boolean add(final byte e) { if (isFull()) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -151,7 +160,8 @@ public boolean add(byte e) { * @param count the minimum number of empty entries in the buffer after this call * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full */ - public void ensureRemaining(int count) { + @Override + public void ensureRemaining(final int count) { if (remaining() < count) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -168,7 +178,7 @@ public void ensureRemaining(int count) { * * @param e the value to add to the buffer */ - public void addUnsafe(byte e) { + public void addUnsafe(final byte e) { // This is an extremely paranoid wrap check that in all likelihood will never run. With FIXUP_THRESHOLD at // 1 << 62, and the user pushing 2^32 values per second(!), it will take 68 years to wrap this counter . if (tail >= FIXUP_THRESHOLD) { @@ -188,7 +198,7 @@ public void addUnsafe(byte e) { * @param notFullResult value to return is the buffer is not full * @return the overwritten entry if the buffer is full, the provided value otherwise */ - public byte addOverwrite(byte e, byte notFullResult) { + public byte addOverwrite(final byte e, final byte notFullResult) { byte val = notFullResult; if (isFull()) { val = remove(); @@ -204,7 +214,7 @@ public byte addOverwrite(byte e, byte notFullResult) { * @param e the byte to be added to the buffer * @return true if the value was added successfully, false otherwise */ - public boolean offer(byte e) { + public boolean offer(final byte e) { if (isFull()) { return false; } @@ -218,7 +228,7 @@ public boolean offer(byte e) { * @param count The number of elements to remove. * @throws NoSuchElementException if the buffer is empty */ - public byte[] remove(int count) { + public byte[] remove(final int count) { final int size = size(); if (size < count) { throw new NoSuchElementException(); @@ -264,7 +274,7 @@ public byte removeUnsafe() { * @param onEmpty the value to return if the ring buffer is empty * @return The removed element if the ring buffer was non-empty, otherwise the value of 'onEmpty' */ - public byte poll(byte onEmpty) { + public byte poll(final byte onEmpty) { if (isEmpty()) { return onEmpty; } @@ -291,7 +301,7 @@ public byte element() { * @param onEmpty the value to return if the ring buffer is empty * @return The head element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public byte peek(byte onEmpty) { + public byte peek(final byte onEmpty) { if (isEmpty()) { return onEmpty; } @@ -314,7 +324,7 @@ public byte front() { * @throws NoSuchElementException if the buffer is empty * @return The element at the specified offset */ - public byte front(int offset) { + public byte front(final int offset) { if (offset < 0 || offset >= size()) { throw new NoSuchElementException(); } @@ -341,7 +351,7 @@ public byte back() { * @param onEmpty the value to return if the ring buffer is empty * @return The tail element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public byte peekBack(byte onEmpty) { + public byte peekBack(final byte onEmpty) { if (isEmpty()) { return onEmpty; } @@ -385,4 +395,14 @@ public void remove() { throw new UnsupportedOperationException(); } } + + /** + * Get the storage array for this ring buffer. This is intended for testing and debugging purposes only. + * + * @return The storage array for this ring buffer. + */ + @TestOnly + public byte[] getStorage() { + return storage; + } } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/CharRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/CharRingBuffer.java index 68c58a866e7..84bff19d07e 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/CharRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/CharRingBuffer.java @@ -5,6 +5,7 @@ import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; +import org.jetbrains.annotations.TestOnly; import java.io.Serializable; import java.util.NoSuchElementException; @@ -15,7 +16,7 @@ * {@code long} values. Head and tail will not wrap around; instead we use storage arrays sized to 2^N to allow fast * determination of storage indices through a mask operation. */ -public class CharRingBuffer implements Serializable { +public class CharRingBuffer implements RingBuffer, Serializable { static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; char[] storage; @@ -28,7 +29,7 @@ public class CharRingBuffer implements Serializable { * * @param capacity minimum capacity of the ring buffer */ - public CharRingBuffer(int capacity) { + public CharRingBuffer(final int capacity) { this(capacity, true); } @@ -38,7 +39,7 @@ public CharRingBuffer(int capacity) { * @param capacity minimum capacity of ring buffer * @param growable whether to allow growth when the buffer is full. */ - public CharRingBuffer(int capacity, boolean growable) { + public CharRingBuffer(final int capacity, final boolean growable) { Assert.leq(capacity, "CharRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; @@ -55,7 +56,7 @@ public CharRingBuffer(int capacity, boolean growable) { * * @param increase Increase amount. The ring buffer's capacity will be increased by at least this amount. */ - protected void grow(int increase) { + protected void grow(final int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible @@ -79,7 +80,7 @@ protected void grow(int increase) { * * @param dest The destination buffer. */ - protected void copyRingBufferToArray(char[] dest) { + protected void copyRingBufferToArray(final char[] dest) { final int size = size(); final int storageHead = (int) (head & mask); @@ -95,27 +96,35 @@ protected void copyRingBufferToArray(char[] dest) { System.arraycopy(storage, 0, dest, firstCopyLen, secondCopyLen); } + @Override public boolean isFull() { return size() == storage.length; } + @Override public boolean isEmpty() { return tail == head; } + @Override public int size() { return Math.toIntExact(tail - head); } + @Override public int capacity() { return storage.length; } + @Override public int remaining() { return storage.length - size(); } + @Override public void clear() { + // region object-bulk-clear + // endregion object-bulk-clear tail = head = 0; } @@ -127,7 +136,7 @@ public void clear() { * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full * @return {@code true} if the char was added successfully */ - public boolean add(char e) { + public boolean add(final char e) { if (isFull()) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -147,7 +156,8 @@ public boolean add(char e) { * @param count the minimum number of empty entries in the buffer after this call * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full */ - public void ensureRemaining(int count) { + @Override + public void ensureRemaining(final int count) { if (remaining() < count) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -164,7 +174,7 @@ public void ensureRemaining(int count) { * * @param e the value to add to the buffer */ - public void addUnsafe(char e) { + public void addUnsafe(final char e) { // This is an extremely paranoid wrap check that in all likelihood will never run. With FIXUP_THRESHOLD at // 1 << 62, and the user pushing 2^32 values per second(!), it will take 68 years to wrap this counter . if (tail >= FIXUP_THRESHOLD) { @@ -184,7 +194,7 @@ public void addUnsafe(char e) { * @param notFullResult value to return is the buffer is not full * @return the overwritten entry if the buffer is full, the provided value otherwise */ - public char addOverwrite(char e, char notFullResult) { + public char addOverwrite(final char e, final char notFullResult) { char val = notFullResult; if (isFull()) { val = remove(); @@ -200,7 +210,7 @@ public char addOverwrite(char e, char notFullResult) { * @param e the char to be added to the buffer * @return true if the value was added successfully, false otherwise */ - public boolean offer(char e) { + public boolean offer(final char e) { if (isFull()) { return false; } @@ -214,7 +224,7 @@ public boolean offer(char e) { * @param count The number of elements to remove. * @throws NoSuchElementException if the buffer is empty */ - public char[] remove(int count) { + public char[] remove(final int count) { final int size = size(); if (size < count) { throw new NoSuchElementException(); @@ -260,7 +270,7 @@ public char removeUnsafe() { * @param onEmpty the value to return if the ring buffer is empty * @return The removed element if the ring buffer was non-empty, otherwise the value of 'onEmpty' */ - public char poll(char onEmpty) { + public char poll(final char onEmpty) { if (isEmpty()) { return onEmpty; } @@ -287,7 +297,7 @@ public char element() { * @param onEmpty the value to return if the ring buffer is empty * @return The head element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public char peek(char onEmpty) { + public char peek(final char onEmpty) { if (isEmpty()) { return onEmpty; } @@ -310,7 +320,7 @@ public char front() { * @throws NoSuchElementException if the buffer is empty * @return The element at the specified offset */ - public char front(int offset) { + public char front(final int offset) { if (offset < 0 || offset >= size()) { throw new NoSuchElementException(); } @@ -337,7 +347,7 @@ public char back() { * @param onEmpty the value to return if the ring buffer is empty * @return The tail element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public char peekBack(char onEmpty) { + public char peekBack(final char onEmpty) { if (isEmpty()) { return onEmpty; } @@ -381,4 +391,14 @@ public void remove() { throw new UnsupportedOperationException(); } } + + /** + * Get the storage array for this ring buffer. This is intended for testing and debugging purposes only. + * + * @return The storage array for this ring buffer. + */ + @TestOnly + public char[] getStorage() { + return storage; + } } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/DoubleRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/DoubleRingBuffer.java index 8e942f8937a..2c28fbdd469 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/DoubleRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/DoubleRingBuffer.java @@ -9,6 +9,7 @@ import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; +import org.jetbrains.annotations.TestOnly; import java.io.Serializable; import java.util.NoSuchElementException; @@ -19,7 +20,7 @@ * {@code long} values. Head and tail will not wrap around; instead we use storage arrays sized to 2^N to allow fast * determination of storage indices through a mask operation. */ -public class DoubleRingBuffer implements Serializable { +public class DoubleRingBuffer implements RingBuffer, Serializable { static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; double[] storage; @@ -32,7 +33,7 @@ public class DoubleRingBuffer implements Serializable { * * @param capacity minimum capacity of the ring buffer */ - public DoubleRingBuffer(int capacity) { + public DoubleRingBuffer(final int capacity) { this(capacity, true); } @@ -42,7 +43,7 @@ public DoubleRingBuffer(int capacity) { * @param capacity minimum capacity of ring buffer * @param growable whether to allow growth when the buffer is full. */ - public DoubleRingBuffer(int capacity, boolean growable) { + public DoubleRingBuffer(final int capacity, final boolean growable) { Assert.leq(capacity, "DoubleRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; @@ -59,7 +60,7 @@ public DoubleRingBuffer(int capacity, boolean growable) { * * @param increase Increase amount. The ring buffer's capacity will be increased by at least this amount. */ - protected void grow(int increase) { + protected void grow(final int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible @@ -83,7 +84,7 @@ protected void grow(int increase) { * * @param dest The destination buffer. */ - protected void copyRingBufferToArray(double[] dest) { + protected void copyRingBufferToArray(final double[] dest) { final int size = size(); final int storageHead = (int) (head & mask); @@ -99,27 +100,35 @@ protected void copyRingBufferToArray(double[] dest) { System.arraycopy(storage, 0, dest, firstCopyLen, secondCopyLen); } + @Override public boolean isFull() { return size() == storage.length; } + @Override public boolean isEmpty() { return tail == head; } + @Override public int size() { return Math.toIntExact(tail - head); } + @Override public int capacity() { return storage.length; } + @Override public int remaining() { return storage.length - size(); } + @Override public void clear() { + // region object-bulk-clear + // endregion object-bulk-clear tail = head = 0; } @@ -131,7 +140,7 @@ public void clear() { * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full * @return {@code true} if the double was added successfully */ - public boolean add(double e) { + public boolean add(final double e) { if (isFull()) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -151,7 +160,8 @@ public boolean add(double e) { * @param count the minimum number of empty entries in the buffer after this call * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full */ - public void ensureRemaining(int count) { + @Override + public void ensureRemaining(final int count) { if (remaining() < count) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -168,7 +178,7 @@ public void ensureRemaining(int count) { * * @param e the value to add to the buffer */ - public void addUnsafe(double e) { + public void addUnsafe(final double e) { // This is an extremely paranoid wrap check that in all likelihood will never run. With FIXUP_THRESHOLD at // 1 << 62, and the user pushing 2^32 values per second(!), it will take 68 years to wrap this counter . if (tail >= FIXUP_THRESHOLD) { @@ -188,7 +198,7 @@ public void addUnsafe(double e) { * @param notFullResult value to return is the buffer is not full * @return the overwritten entry if the buffer is full, the provided value otherwise */ - public double addOverwrite(double e, double notFullResult) { + public double addOverwrite(final double e, final double notFullResult) { double val = notFullResult; if (isFull()) { val = remove(); @@ -204,7 +214,7 @@ public double addOverwrite(double e, double notFullResult) { * @param e the double to be added to the buffer * @return true if the value was added successfully, false otherwise */ - public boolean offer(double e) { + public boolean offer(final double e) { if (isFull()) { return false; } @@ -218,7 +228,7 @@ public boolean offer(double e) { * @param count The number of elements to remove. * @throws NoSuchElementException if the buffer is empty */ - public double[] remove(int count) { + public double[] remove(final int count) { final int size = size(); if (size < count) { throw new NoSuchElementException(); @@ -264,7 +274,7 @@ public double removeUnsafe() { * @param onEmpty the value to return if the ring buffer is empty * @return The removed element if the ring buffer was non-empty, otherwise the value of 'onEmpty' */ - public double poll(double onEmpty) { + public double poll(final double onEmpty) { if (isEmpty()) { return onEmpty; } @@ -291,7 +301,7 @@ public double element() { * @param onEmpty the value to return if the ring buffer is empty * @return The head element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public double peek(double onEmpty) { + public double peek(final double onEmpty) { if (isEmpty()) { return onEmpty; } @@ -314,7 +324,7 @@ public double front() { * @throws NoSuchElementException if the buffer is empty * @return The element at the specified offset */ - public double front(int offset) { + public double front(final int offset) { if (offset < 0 || offset >= size()) { throw new NoSuchElementException(); } @@ -341,7 +351,7 @@ public double back() { * @param onEmpty the value to return if the ring buffer is empty * @return The tail element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public double peekBack(double onEmpty) { + public double peekBack(final double onEmpty) { if (isEmpty()) { return onEmpty; } @@ -385,4 +395,14 @@ public void remove() { throw new UnsupportedOperationException(); } } + + /** + * Get the storage array for this ring buffer. This is intended for testing and debugging purposes only. + * + * @return The storage array for this ring buffer. + */ + @TestOnly + public double[] getStorage() { + return storage; + } } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/FloatRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/FloatRingBuffer.java index 16c8751c447..d432710383e 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/FloatRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/FloatRingBuffer.java @@ -9,6 +9,7 @@ import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; +import org.jetbrains.annotations.TestOnly; import java.io.Serializable; import java.util.NoSuchElementException; @@ -19,7 +20,7 @@ * {@code long} values. Head and tail will not wrap around; instead we use storage arrays sized to 2^N to allow fast * determination of storage indices through a mask operation. */ -public class FloatRingBuffer implements Serializable { +public class FloatRingBuffer implements RingBuffer, Serializable { static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; float[] storage; @@ -32,7 +33,7 @@ public class FloatRingBuffer implements Serializable { * * @param capacity minimum capacity of the ring buffer */ - public FloatRingBuffer(int capacity) { + public FloatRingBuffer(final int capacity) { this(capacity, true); } @@ -42,7 +43,7 @@ public FloatRingBuffer(int capacity) { * @param capacity minimum capacity of ring buffer * @param growable whether to allow growth when the buffer is full. */ - public FloatRingBuffer(int capacity, boolean growable) { + public FloatRingBuffer(final int capacity, final boolean growable) { Assert.leq(capacity, "FloatRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; @@ -59,7 +60,7 @@ public FloatRingBuffer(int capacity, boolean growable) { * * @param increase Increase amount. The ring buffer's capacity will be increased by at least this amount. */ - protected void grow(int increase) { + protected void grow(final int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible @@ -83,7 +84,7 @@ protected void grow(int increase) { * * @param dest The destination buffer. */ - protected void copyRingBufferToArray(float[] dest) { + protected void copyRingBufferToArray(final float[] dest) { final int size = size(); final int storageHead = (int) (head & mask); @@ -99,27 +100,35 @@ protected void copyRingBufferToArray(float[] dest) { System.arraycopy(storage, 0, dest, firstCopyLen, secondCopyLen); } + @Override public boolean isFull() { return size() == storage.length; } + @Override public boolean isEmpty() { return tail == head; } + @Override public int size() { return Math.toIntExact(tail - head); } + @Override public int capacity() { return storage.length; } + @Override public int remaining() { return storage.length - size(); } + @Override public void clear() { + // region object-bulk-clear + // endregion object-bulk-clear tail = head = 0; } @@ -131,7 +140,7 @@ public void clear() { * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full * @return {@code true} if the float was added successfully */ - public boolean add(float e) { + public boolean add(final float e) { if (isFull()) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -151,7 +160,8 @@ public boolean add(float e) { * @param count the minimum number of empty entries in the buffer after this call * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full */ - public void ensureRemaining(int count) { + @Override + public void ensureRemaining(final int count) { if (remaining() < count) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -168,7 +178,7 @@ public void ensureRemaining(int count) { * * @param e the value to add to the buffer */ - public void addUnsafe(float e) { + public void addUnsafe(final float e) { // This is an extremely paranoid wrap check that in all likelihood will never run. With FIXUP_THRESHOLD at // 1 << 62, and the user pushing 2^32 values per second(!), it will take 68 years to wrap this counter . if (tail >= FIXUP_THRESHOLD) { @@ -188,7 +198,7 @@ public void addUnsafe(float e) { * @param notFullResult value to return is the buffer is not full * @return the overwritten entry if the buffer is full, the provided value otherwise */ - public float addOverwrite(float e, float notFullResult) { + public float addOverwrite(final float e, final float notFullResult) { float val = notFullResult; if (isFull()) { val = remove(); @@ -204,7 +214,7 @@ public float addOverwrite(float e, float notFullResult) { * @param e the float to be added to the buffer * @return true if the value was added successfully, false otherwise */ - public boolean offer(float e) { + public boolean offer(final float e) { if (isFull()) { return false; } @@ -218,7 +228,7 @@ public boolean offer(float e) { * @param count The number of elements to remove. * @throws NoSuchElementException if the buffer is empty */ - public float[] remove(int count) { + public float[] remove(final int count) { final int size = size(); if (size < count) { throw new NoSuchElementException(); @@ -264,7 +274,7 @@ public float removeUnsafe() { * @param onEmpty the value to return if the ring buffer is empty * @return The removed element if the ring buffer was non-empty, otherwise the value of 'onEmpty' */ - public float poll(float onEmpty) { + public float poll(final float onEmpty) { if (isEmpty()) { return onEmpty; } @@ -291,7 +301,7 @@ public float element() { * @param onEmpty the value to return if the ring buffer is empty * @return The head element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public float peek(float onEmpty) { + public float peek(final float onEmpty) { if (isEmpty()) { return onEmpty; } @@ -314,7 +324,7 @@ public float front() { * @throws NoSuchElementException if the buffer is empty * @return The element at the specified offset */ - public float front(int offset) { + public float front(final int offset) { if (offset < 0 || offset >= size()) { throw new NoSuchElementException(); } @@ -341,7 +351,7 @@ public float back() { * @param onEmpty the value to return if the ring buffer is empty * @return The tail element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public float peekBack(float onEmpty) { + public float peekBack(final float onEmpty) { if (isEmpty()) { return onEmpty; } @@ -385,4 +395,14 @@ public void remove() { throw new UnsupportedOperationException(); } } + + /** + * Get the storage array for this ring buffer. This is intended for testing and debugging purposes only. + * + * @return The storage array for this ring buffer. + */ + @TestOnly + public float[] getStorage() { + return storage; + } } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/IntRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/IntRingBuffer.java index 80a47f0a389..ca83c1715d8 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/IntRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/IntRingBuffer.java @@ -9,6 +9,7 @@ import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; +import org.jetbrains.annotations.TestOnly; import java.io.Serializable; import java.util.NoSuchElementException; @@ -19,7 +20,7 @@ * {@code long} values. Head and tail will not wrap around; instead we use storage arrays sized to 2^N to allow fast * determination of storage indices through a mask operation. */ -public class IntRingBuffer implements Serializable { +public class IntRingBuffer implements RingBuffer, Serializable { static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; int[] storage; @@ -32,7 +33,7 @@ public class IntRingBuffer implements Serializable { * * @param capacity minimum capacity of the ring buffer */ - public IntRingBuffer(int capacity) { + public IntRingBuffer(final int capacity) { this(capacity, true); } @@ -42,7 +43,7 @@ public IntRingBuffer(int capacity) { * @param capacity minimum capacity of ring buffer * @param growable whether to allow growth when the buffer is full. */ - public IntRingBuffer(int capacity, boolean growable) { + public IntRingBuffer(final int capacity, final boolean growable) { Assert.leq(capacity, "IntRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; @@ -59,7 +60,7 @@ public IntRingBuffer(int capacity, boolean growable) { * * @param increase Increase amount. The ring buffer's capacity will be increased by at least this amount. */ - protected void grow(int increase) { + protected void grow(final int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible @@ -83,7 +84,7 @@ protected void grow(int increase) { * * @param dest The destination buffer. */ - protected void copyRingBufferToArray(int[] dest) { + protected void copyRingBufferToArray(final int[] dest) { final int size = size(); final int storageHead = (int) (head & mask); @@ -99,27 +100,35 @@ protected void copyRingBufferToArray(int[] dest) { System.arraycopy(storage, 0, dest, firstCopyLen, secondCopyLen); } + @Override public boolean isFull() { return size() == storage.length; } + @Override public boolean isEmpty() { return tail == head; } + @Override public int size() { return Math.toIntExact(tail - head); } + @Override public int capacity() { return storage.length; } + @Override public int remaining() { return storage.length - size(); } + @Override public void clear() { + // region object-bulk-clear + // endregion object-bulk-clear tail = head = 0; } @@ -131,7 +140,7 @@ public void clear() { * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full * @return {@code true} if the int was added successfully */ - public boolean add(int e) { + public boolean add(final int e) { if (isFull()) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -151,7 +160,8 @@ public boolean add(int e) { * @param count the minimum number of empty entries in the buffer after this call * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full */ - public void ensureRemaining(int count) { + @Override + public void ensureRemaining(final int count) { if (remaining() < count) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -168,7 +178,7 @@ public void ensureRemaining(int count) { * * @param e the value to add to the buffer */ - public void addUnsafe(int e) { + public void addUnsafe(final int e) { // This is an extremely paranoid wrap check that in all likelihood will never run. With FIXUP_THRESHOLD at // 1 << 62, and the user pushing 2^32 values per second(!), it will take 68 years to wrap this counter . if (tail >= FIXUP_THRESHOLD) { @@ -188,7 +198,7 @@ public void addUnsafe(int e) { * @param notFullResult value to return is the buffer is not full * @return the overwritten entry if the buffer is full, the provided value otherwise */ - public int addOverwrite(int e, int notFullResult) { + public int addOverwrite(final int e, final int notFullResult) { int val = notFullResult; if (isFull()) { val = remove(); @@ -204,7 +214,7 @@ public int addOverwrite(int e, int notFullResult) { * @param e the int to be added to the buffer * @return true if the value was added successfully, false otherwise */ - public boolean offer(int e) { + public boolean offer(final int e) { if (isFull()) { return false; } @@ -218,7 +228,7 @@ public boolean offer(int e) { * @param count The number of elements to remove. * @throws NoSuchElementException if the buffer is empty */ - public int[] remove(int count) { + public int[] remove(final int count) { final int size = size(); if (size < count) { throw new NoSuchElementException(); @@ -264,7 +274,7 @@ public int removeUnsafe() { * @param onEmpty the value to return if the ring buffer is empty * @return The removed element if the ring buffer was non-empty, otherwise the value of 'onEmpty' */ - public int poll(int onEmpty) { + public int poll(final int onEmpty) { if (isEmpty()) { return onEmpty; } @@ -291,7 +301,7 @@ public int element() { * @param onEmpty the value to return if the ring buffer is empty * @return The head element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public int peek(int onEmpty) { + public int peek(final int onEmpty) { if (isEmpty()) { return onEmpty; } @@ -314,7 +324,7 @@ public int front() { * @throws NoSuchElementException if the buffer is empty * @return The element at the specified offset */ - public int front(int offset) { + public int front(final int offset) { if (offset < 0 || offset >= size()) { throw new NoSuchElementException(); } @@ -341,7 +351,7 @@ public int back() { * @param onEmpty the value to return if the ring buffer is empty * @return The tail element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public int peekBack(int onEmpty) { + public int peekBack(final int onEmpty) { if (isEmpty()) { return onEmpty; } @@ -385,4 +395,14 @@ public void remove() { throw new UnsupportedOperationException(); } } + + /** + * Get the storage array for this ring buffer. This is intended for testing and debugging purposes only. + * + * @return The storage array for this ring buffer. + */ + @TestOnly + public int[] getStorage() { + return storage; + } } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/LongRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/LongRingBuffer.java index de5f8df9c90..43d9312711f 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/LongRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/LongRingBuffer.java @@ -9,6 +9,7 @@ import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; +import org.jetbrains.annotations.TestOnly; import java.io.Serializable; import java.util.NoSuchElementException; @@ -19,7 +20,7 @@ * {@code long} values. Head and tail will not wrap around; instead we use storage arrays sized to 2^N to allow fast * determination of storage indices through a mask operation. */ -public class LongRingBuffer implements Serializable { +public class LongRingBuffer implements RingBuffer, Serializable { static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; long[] storage; @@ -32,7 +33,7 @@ public class LongRingBuffer implements Serializable { * * @param capacity minimum capacity of the ring buffer */ - public LongRingBuffer(int capacity) { + public LongRingBuffer(final int capacity) { this(capacity, true); } @@ -42,7 +43,7 @@ public LongRingBuffer(int capacity) { * @param capacity minimum capacity of ring buffer * @param growable whether to allow growth when the buffer is full. */ - public LongRingBuffer(int capacity, boolean growable) { + public LongRingBuffer(final int capacity, final boolean growable) { Assert.leq(capacity, "LongRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; @@ -59,7 +60,7 @@ public LongRingBuffer(int capacity, boolean growable) { * * @param increase Increase amount. The ring buffer's capacity will be increased by at least this amount. */ - protected void grow(int increase) { + protected void grow(final int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible @@ -83,7 +84,7 @@ protected void grow(int increase) { * * @param dest The destination buffer. */ - protected void copyRingBufferToArray(long[] dest) { + protected void copyRingBufferToArray(final long[] dest) { final int size = size(); final int storageHead = (int) (head & mask); @@ -99,27 +100,35 @@ protected void copyRingBufferToArray(long[] dest) { System.arraycopy(storage, 0, dest, firstCopyLen, secondCopyLen); } + @Override public boolean isFull() { return size() == storage.length; } + @Override public boolean isEmpty() { return tail == head; } + @Override public int size() { return Math.toIntExact(tail - head); } + @Override public int capacity() { return storage.length; } + @Override public int remaining() { return storage.length - size(); } + @Override public void clear() { + // region object-bulk-clear + // endregion object-bulk-clear tail = head = 0; } @@ -131,7 +140,7 @@ public void clear() { * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full * @return {@code true} if the long was added successfully */ - public boolean add(long e) { + public boolean add(final long e) { if (isFull()) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -151,7 +160,8 @@ public boolean add(long e) { * @param count the minimum number of empty entries in the buffer after this call * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full */ - public void ensureRemaining(int count) { + @Override + public void ensureRemaining(final int count) { if (remaining() < count) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -168,7 +178,7 @@ public void ensureRemaining(int count) { * * @param e the value to add to the buffer */ - public void addUnsafe(long e) { + public void addUnsafe(final long e) { // This is an extremely paranoid wrap check that in all likelihood will never run. With FIXUP_THRESHOLD at // 1 << 62, and the user pushing 2^32 values per second(!), it will take 68 years to wrap this counter . if (tail >= FIXUP_THRESHOLD) { @@ -188,7 +198,7 @@ public void addUnsafe(long e) { * @param notFullResult value to return is the buffer is not full * @return the overwritten entry if the buffer is full, the provided value otherwise */ - public long addOverwrite(long e, long notFullResult) { + public long addOverwrite(final long e, final long notFullResult) { long val = notFullResult; if (isFull()) { val = remove(); @@ -204,7 +214,7 @@ public long addOverwrite(long e, long notFullResult) { * @param e the long to be added to the buffer * @return true if the value was added successfully, false otherwise */ - public boolean offer(long e) { + public boolean offer(final long e) { if (isFull()) { return false; } @@ -218,7 +228,7 @@ public boolean offer(long e) { * @param count The number of elements to remove. * @throws NoSuchElementException if the buffer is empty */ - public long[] remove(int count) { + public long[] remove(final int count) { final int size = size(); if (size < count) { throw new NoSuchElementException(); @@ -264,7 +274,7 @@ public long removeUnsafe() { * @param onEmpty the value to return if the ring buffer is empty * @return The removed element if the ring buffer was non-empty, otherwise the value of 'onEmpty' */ - public long poll(long onEmpty) { + public long poll(final long onEmpty) { if (isEmpty()) { return onEmpty; } @@ -291,7 +301,7 @@ public long element() { * @param onEmpty the value to return if the ring buffer is empty * @return The head element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public long peek(long onEmpty) { + public long peek(final long onEmpty) { if (isEmpty()) { return onEmpty; } @@ -314,7 +324,7 @@ public long front() { * @throws NoSuchElementException if the buffer is empty * @return The element at the specified offset */ - public long front(int offset) { + public long front(final int offset) { if (offset < 0 || offset >= size()) { throw new NoSuchElementException(); } @@ -341,7 +351,7 @@ public long back() { * @param onEmpty the value to return if the ring buffer is empty * @return The tail element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public long peekBack(long onEmpty) { + public long peekBack(final long onEmpty) { if (isEmpty()) { return onEmpty; } @@ -385,4 +395,14 @@ public void remove() { throw new UnsupportedOperationException(); } } + + /** + * Get the storage array for this ring buffer. This is intended for testing and debugging purposes only. + * + * @return The storage array for this ring buffer. + */ + @TestOnly + public long[] getStorage() { + return storage; + } } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/ObjectRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/ObjectRingBuffer.java index ad93ddd1f43..4c7dc788602 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/ObjectRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/ObjectRingBuffer.java @@ -11,6 +11,7 @@ import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; +import org.jetbrains.annotations.TestOnly; import java.io.Serializable; import java.util.NoSuchElementException; @@ -21,7 +22,7 @@ * {@code long} values. Head and tail will not wrap around; instead we use storage arrays sized to 2^N to allow fast * determination of storage indices through a mask operation. */ -public class ObjectRingBuffer implements Serializable { +public class ObjectRingBuffer implements RingBuffer, Serializable { static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; T[] storage; @@ -34,7 +35,7 @@ public class ObjectRingBuffer implements Serializable { * * @param capacity minimum capacity of the ring buffer */ - public ObjectRingBuffer(int capacity) { + public ObjectRingBuffer(final int capacity) { this(capacity, true); } @@ -44,7 +45,7 @@ public ObjectRingBuffer(int capacity) { * @param capacity minimum capacity of ring buffer * @param growable whether to allow growth when the buffer is full. */ - public ObjectRingBuffer(int capacity, boolean growable) { + public ObjectRingBuffer(final int capacity, final boolean growable) { Assert.leq(capacity, "ObjectRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; @@ -61,7 +62,7 @@ public ObjectRingBuffer(int capacity, boolean growable) { * * @param increase Increase amount. The ring buffer's capacity will be increased by at least this amount. */ - protected void grow(int increase) { + protected void grow(final int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible @@ -85,7 +86,7 @@ protected void grow(int increase) { * * @param dest The destination buffer. */ - protected void copyRingBufferToArray(T[] dest) { + protected void copyRingBufferToArray(final T[] dest) { final int size = size(); final int storageHead = (int) (head & mask); @@ -101,27 +102,43 @@ protected void copyRingBufferToArray(T[] dest) { System.arraycopy(storage, 0, dest, firstCopyLen, secondCopyLen); } + @Override public boolean isFull() { return size() == storage.length; } + @Override public boolean isEmpty() { return tail == head; } + @Override public int size() { return Math.toIntExact(tail - head); } + @Override public int capacity() { return storage.length; } + @Override public int remaining() { return storage.length - size(); } + @Override public void clear() { + // region object-bulk-clear + final int storageHead = (int) (head & mask); + final int size = size(); + // firstLen is either the size of the ring buffer or the distance from head to the end of the storage array. + final int firstLen = Math.min(storage.length - storageHead, size); + // secondLen is the number of elements remaining from the first clear. + final int secondLen = size - firstLen; + Arrays.fill(storage, storageHead, storageHead + firstLen, null); + Arrays.fill(storage, 0, secondLen, null); + // endregion object-bulk-clear tail = head = 0; } @@ -133,7 +150,7 @@ public void clear() { * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full * @return {@code true} if the Object was added successfully */ - public boolean add(T e) { + public boolean add(final T e) { if (isFull()) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -153,7 +170,8 @@ public boolean add(T e) { * @param count the minimum number of empty entries in the buffer after this call * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full */ - public void ensureRemaining(int count) { + @Override + public void ensureRemaining(final int count) { if (remaining() < count) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -170,7 +188,7 @@ public void ensureRemaining(int count) { * * @param e the value to add to the buffer */ - public void addUnsafe(T e) { + public void addUnsafe(final T e) { // This is an extremely paranoid wrap check that in all likelihood will never run. With FIXUP_THRESHOLD at // 1 << 62, and the user pushing 2^32 values per second(!), it will take 68 years to wrap this counter . if (tail >= FIXUP_THRESHOLD) { @@ -190,7 +208,7 @@ public void addUnsafe(T e) { * @param notFullResult value to return is the buffer is not full * @return the overwritten entry if the buffer is full, the provided value otherwise */ - public T addOverwrite(T e, T notFullResult) { + public T addOverwrite(final T e, final T notFullResult) { T val = notFullResult; if (isFull()) { val = remove(); @@ -206,7 +224,7 @@ public T addOverwrite(T e, T notFullResult) { * @param e the Object to be added to the buffer * @return true if the value was added successfully, false otherwise */ - public boolean offer(T e) { + public boolean offer(final T e) { if (isFull()) { return false; } @@ -220,7 +238,7 @@ public boolean offer(T e) { * @param count The number of elements to remove. * @throws NoSuchElementException if the buffer is empty */ - public T[] remove(int count) { + public T[] remove(final int count) { final int size = size(); if (size < count) { throw new NoSuchElementException(); @@ -281,7 +299,7 @@ public T removeUnsafe() { * @param onEmpty the value to return if the ring buffer is empty * @return The removed element if the ring buffer was non-empty, otherwise the value of 'onEmpty' */ - public T poll(T onEmpty) { + public T poll(final T onEmpty) { if (isEmpty()) { return onEmpty; } @@ -308,7 +326,7 @@ public T element() { * @param onEmpty the value to return if the ring buffer is empty * @return The head element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public T peek(T onEmpty) { + public T peek(final T onEmpty) { if (isEmpty()) { return onEmpty; } @@ -331,7 +349,7 @@ public T front() { * @throws NoSuchElementException if the buffer is empty * @return The element at the specified offset */ - public T front(int offset) { + public T front(final int offset) { if (offset < 0 || offset >= size()) { throw new NoSuchElementException(); } @@ -358,7 +376,7 @@ public T back() { * @param onEmpty the value to return if the ring buffer is empty * @return The tail element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public T peekBack(T onEmpty) { + public T peekBack(final T onEmpty) { if (isEmpty()) { return onEmpty; } @@ -402,4 +420,14 @@ public void remove() { throw new UnsupportedOperationException(); } } + + /** + * Get the storage array for this ring buffer. This is intended for testing and debugging purposes only. + * + * @return The storage array for this ring buffer. + */ + @TestOnly + public T[] getStorage() { + return storage; + } } diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/RingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/RingBuffer.java new file mode 100644 index 00000000000..4c9f0b55a3e --- /dev/null +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/RingBuffer.java @@ -0,0 +1,79 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.base.ringbuffer; + +import org.jetbrains.annotations.NotNull; + +/** + * Generic interface for a ring buffer and factory methods for buffer creation. + */ +public interface RingBuffer { + @NotNull + static RingBuffer makeRingBuffer( + @NotNull final Class dataType, + final int capacity, + final boolean growable) { + final RingBuffer result; + if (dataType == char.class || dataType == Character.class) { + result = new CharRingBuffer(capacity, growable); + } else if (dataType == byte.class || dataType == Byte.class) { + result = new ByteRingBuffer(capacity, growable); + } else if (dataType == double.class || dataType == Double.class) { + result = new DoubleRingBuffer(capacity, growable); + } else if (dataType == float.class || dataType == Float.class) { + result = new FloatRingBuffer(capacity, growable); + } else if (dataType == int.class || dataType == Integer.class) { + result = new IntRingBuffer(capacity, growable); + } else if (dataType == long.class || dataType == Long.class) { + result = new LongRingBuffer(capacity, growable); + } else if (dataType == short.class || dataType == Short.class) { + result = new ShortRingBuffer(capacity, growable); + } else { + result = new ObjectRingBuffer(capacity, growable); + } + return result; + } + + /** + * Whether the buffer is completely full. + */ + boolean isFull(); + + /** + * Whether the buffer is entirely empty. + */ + boolean isEmpty(); + + /** + * Return how many items are currently in the buffer. + */ + int size(); + + /** + * Return how many items can fit in the buffer at its current capacity. If the buffer can grow, this number can + * change. + */ + int capacity(); + + /** + * Return how many free slots exist in this buffer at its current capacity. + */ + int remaining(); + + /** + * Clear the buffer of all values. If this is an object ring buffer, this will additionally set all values to + * {@code null}. + */ + void clear(); + + /** + * Ensure that at least {@code count} free slots are available in the buffer. If the buffer is growable, the + * capacity may be increased to accommodate the new slots. If the buffer is not growable and there are insufficient + * free slots, an {@link UnsupportedOperationException} will be thrown. + * + * @param count the number of free slots to ensure are available + * @throws UnsupportedOperationException if the buffer is not growable and there are insufficient free slots + */ + void ensureRemaining(int count); +} diff --git a/Base/src/main/java/io/deephaven/base/ringbuffer/ShortRingBuffer.java b/Base/src/main/java/io/deephaven/base/ringbuffer/ShortRingBuffer.java index 89619fb839c..94076d3a40f 100644 --- a/Base/src/main/java/io/deephaven/base/ringbuffer/ShortRingBuffer.java +++ b/Base/src/main/java/io/deephaven/base/ringbuffer/ShortRingBuffer.java @@ -9,6 +9,7 @@ import io.deephaven.base.MathUtil; import io.deephaven.base.verify.Assert; +import org.jetbrains.annotations.TestOnly; import java.io.Serializable; import java.util.NoSuchElementException; @@ -19,7 +20,7 @@ * {@code long} values. Head and tail will not wrap around; instead we use storage arrays sized to 2^N to allow fast * determination of storage indices through a mask operation. */ -public class ShortRingBuffer implements Serializable { +public class ShortRingBuffer implements RingBuffer, Serializable { static final long FIXUP_THRESHOLD = 1L << 62; final boolean growable; short[] storage; @@ -32,7 +33,7 @@ public class ShortRingBuffer implements Serializable { * * @param capacity minimum capacity of the ring buffer */ - public ShortRingBuffer(int capacity) { + public ShortRingBuffer(final int capacity) { this(capacity, true); } @@ -42,7 +43,7 @@ public ShortRingBuffer(int capacity) { * @param capacity minimum capacity of ring buffer * @param growable whether to allow growth when the buffer is full. */ - public ShortRingBuffer(int capacity, boolean growable) { + public ShortRingBuffer(final int capacity, final boolean growable) { Assert.leq(capacity, "ShortRingBuffer capacity", MathUtil.MAX_POWER_OF_2); this.growable = growable; @@ -59,7 +60,7 @@ public ShortRingBuffer(int capacity, boolean growable) { * * @param increase Increase amount. The ring buffer's capacity will be increased by at least this amount. */ - protected void grow(int increase) { + protected void grow(final int increase) { final int size = size(); final long newCapacity = (long) storage.length + increase; // assert that we are not asking for the impossible @@ -83,7 +84,7 @@ protected void grow(int increase) { * * @param dest The destination buffer. */ - protected void copyRingBufferToArray(short[] dest) { + protected void copyRingBufferToArray(final short[] dest) { final int size = size(); final int storageHead = (int) (head & mask); @@ -99,27 +100,35 @@ protected void copyRingBufferToArray(short[] dest) { System.arraycopy(storage, 0, dest, firstCopyLen, secondCopyLen); } + @Override public boolean isFull() { return size() == storage.length; } + @Override public boolean isEmpty() { return tail == head; } + @Override public int size() { return Math.toIntExact(tail - head); } + @Override public int capacity() { return storage.length; } + @Override public int remaining() { return storage.length - size(); } + @Override public void clear() { + // region object-bulk-clear + // endregion object-bulk-clear tail = head = 0; } @@ -131,7 +140,7 @@ public void clear() { * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full * @return {@code true} if the short was added successfully */ - public boolean add(short e) { + public boolean add(final short e) { if (isFull()) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -151,7 +160,8 @@ public boolean add(short e) { * @param count the minimum number of empty entries in the buffer after this call * @throws UnsupportedOperationException when {@code growable} is {@code false} and buffer is full */ - public void ensureRemaining(int count) { + @Override + public void ensureRemaining(final int count) { if (remaining() < count) { if (!growable) { throw new UnsupportedOperationException("Ring buffer is full and growth is disabled"); @@ -168,7 +178,7 @@ public void ensureRemaining(int count) { * * @param e the value to add to the buffer */ - public void addUnsafe(short e) { + public void addUnsafe(final short e) { // This is an extremely paranoid wrap check that in all likelihood will never run. With FIXUP_THRESHOLD at // 1 << 62, and the user pushing 2^32 values per second(!), it will take 68 years to wrap this counter . if (tail >= FIXUP_THRESHOLD) { @@ -188,7 +198,7 @@ public void addUnsafe(short e) { * @param notFullResult value to return is the buffer is not full * @return the overwritten entry if the buffer is full, the provided value otherwise */ - public short addOverwrite(short e, short notFullResult) { + public short addOverwrite(final short e, final short notFullResult) { short val = notFullResult; if (isFull()) { val = remove(); @@ -204,7 +214,7 @@ public short addOverwrite(short e, short notFullResult) { * @param e the short to be added to the buffer * @return true if the value was added successfully, false otherwise */ - public boolean offer(short e) { + public boolean offer(final short e) { if (isFull()) { return false; } @@ -218,7 +228,7 @@ public boolean offer(short e) { * @param count The number of elements to remove. * @throws NoSuchElementException if the buffer is empty */ - public short[] remove(int count) { + public short[] remove(final int count) { final int size = size(); if (size < count) { throw new NoSuchElementException(); @@ -264,7 +274,7 @@ public short removeUnsafe() { * @param onEmpty the value to return if the ring buffer is empty * @return The removed element if the ring buffer was non-empty, otherwise the value of 'onEmpty' */ - public short poll(short onEmpty) { + public short poll(final short onEmpty) { if (isEmpty()) { return onEmpty; } @@ -291,7 +301,7 @@ public short element() { * @param onEmpty the value to return if the ring buffer is empty * @return The head element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public short peek(short onEmpty) { + public short peek(final short onEmpty) { if (isEmpty()) { return onEmpty; } @@ -314,7 +324,7 @@ public short front() { * @throws NoSuchElementException if the buffer is empty * @return The element at the specified offset */ - public short front(int offset) { + public short front(final int offset) { if (offset < 0 || offset >= size()) { throw new NoSuchElementException(); } @@ -341,7 +351,7 @@ public short back() { * @param onEmpty the value to return if the ring buffer is empty * @return The tail element if the ring buffer is non-empty, otherwise the value of 'onEmpty' */ - public short peekBack(short onEmpty) { + public short peekBack(final short onEmpty) { if (isEmpty()) { return onEmpty; } @@ -385,4 +395,14 @@ public void remove() { throw new UnsupportedOperationException(); } } + + /** + * Get the storage array for this ring buffer. This is intended for testing and debugging purposes only. + * + * @return The storage array for this ring buffer. + */ + @TestOnly + public short[] getStorage() { + return storage; + } } diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingByteRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingByteRingBuffer.java similarity index 97% rename from Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingByteRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingByteRingBuffer.java index 553f3bf7222..38e238159f5 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingByteRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingByteRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit AggregatingCharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestAggregatingCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -12,7 +12,7 @@ import java.util.NoSuchElementException; import java.util.Random; -public class AggregatingByteRingBufferTest extends TestCase { +public class TestAggregatingByteRingBuffer extends TestCase { private void assertEmpty(AggregatingByteRingBuffer rb) { assertTrue(rb.isEmpty()); diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingCharRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingCharRingBuffer.java similarity index 99% rename from Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingCharRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingCharRingBuffer.java index f0d682477d8..9ac48986faa 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingCharRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingCharRingBuffer.java @@ -8,7 +8,7 @@ import java.util.NoSuchElementException; import java.util.Random; -public class AggregatingCharRingBufferTest extends TestCase { +public class TestAggregatingCharRingBuffer extends TestCase { private void assertEmpty(AggregatingCharRingBuffer rb) { assertTrue(rb.isEmpty()); diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingDoubleRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingDoubleRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingDoubleRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingDoubleRingBuffer.java index 078dc150a6e..d5883a5a598 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingDoubleRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingDoubleRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit AggregatingCharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestAggregatingCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -12,7 +12,7 @@ import java.util.NoSuchElementException; import java.util.Random; -public class AggregatingDoubleRingBufferTest extends TestCase { +public class TestAggregatingDoubleRingBuffer extends TestCase { private void assertEmpty(AggregatingDoubleRingBuffer rb) { assertTrue(rb.isEmpty()); diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingFloatRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingFloatRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingFloatRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingFloatRingBuffer.java index 08ea6c76190..9116bea7b9d 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingFloatRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingFloatRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit AggregatingCharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestAggregatingCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -12,7 +12,7 @@ import java.util.NoSuchElementException; import java.util.Random; -public class AggregatingFloatRingBufferTest extends TestCase { +public class TestAggregatingFloatRingBuffer extends TestCase { private void assertEmpty(AggregatingFloatRingBuffer rb) { assertTrue(rb.isEmpty()); diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingIntRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingIntRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingIntRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingIntRingBuffer.java index 0a8445f6eab..1d225bfd233 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingIntRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingIntRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit AggregatingCharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestAggregatingCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -12,7 +12,7 @@ import java.util.NoSuchElementException; import java.util.Random; -public class AggregatingIntRingBufferTest extends TestCase { +public class TestAggregatingIntRingBuffer extends TestCase { private void assertEmpty(AggregatingIntRingBuffer rb) { assertTrue(rb.isEmpty()); diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingLongRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingLongRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingLongRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingLongRingBuffer.java index afb705fa793..cd765139f27 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingLongRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingLongRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit AggregatingCharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestAggregatingCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -12,7 +12,7 @@ import java.util.NoSuchElementException; import java.util.Random; -public class AggregatingLongRingBufferTest extends TestCase { +public class TestAggregatingLongRingBuffer extends TestCase { private void assertEmpty(AggregatingLongRingBuffer rb) { assertTrue(rb.isEmpty()); diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingObjectRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingObjectRingBuffer.java similarity index 99% rename from Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingObjectRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingObjectRingBuffer.java index 0f226b6e18d..09595fba70b 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingObjectRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingObjectRingBuffer.java @@ -9,7 +9,7 @@ import java.util.NoSuchElementException; import java.util.Random; -public class AggregatingObjectRingBufferTest extends TestCase { +public class TestAggregatingObjectRingBuffer extends TestCase { private void assertEmpty(AggregatingObjectRingBuffer rb) { assertTrue(rb.isEmpty()); diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingShortRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingShortRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingShortRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingShortRingBuffer.java index fe77b88efb3..7c50f50faac 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingShortRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingShortRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit AggregatingCharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestAggregatingCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -12,7 +12,7 @@ import java.util.NoSuchElementException; import java.util.Random; -public class AggregatingShortRingBufferTest extends TestCase { +public class TestAggregatingShortRingBuffer extends TestCase { private void assertEmpty(AggregatingShortRingBuffer rb) { assertTrue(rb.isEmpty()); diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/ByteRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestByteRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/ByteRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestByteRingBuffer.java index 14e22525eb0..70e8914efcb 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/ByteRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestByteRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit CharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -15,7 +15,7 @@ import static org.junit.Assert.assertThrows; -public class ByteRingBufferTest extends TestCase { +public class TestByteRingBuffer extends TestCase { final byte SENTINEL = Byte.MIN_VALUE; @@ -38,6 +38,8 @@ private void assertEmpty(ByteRingBuffer rb) { } catch (NoSuchElementException x) { // expected } + // region empty-test + // endregion empty-test } private void assertFull(ByteRingBuffer rb) { diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/CharRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestCharRingBuffer.java similarity index 99% rename from Base/src/test/java/io/deephaven/base/ringbuffer/CharRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestCharRingBuffer.java index e548ea634bf..ec76bb60bdc 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/CharRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestCharRingBuffer.java @@ -11,7 +11,7 @@ import static org.junit.Assert.assertThrows; -public class CharRingBufferTest extends TestCase { +public class TestCharRingBuffer extends TestCase { final char SENTINEL = Character.MIN_VALUE; @@ -34,6 +34,8 @@ private void assertEmpty(CharRingBuffer rb) { } catch (NoSuchElementException x) { // expected } + // region empty-test + // endregion empty-test } private void assertFull(CharRingBuffer rb) { diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/DoubleRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestDoubleRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/DoubleRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestDoubleRingBuffer.java index 9503dbf246b..df5286cbf02 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/DoubleRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestDoubleRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit CharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -15,7 +15,7 @@ import static org.junit.Assert.assertThrows; -public class DoubleRingBufferTest extends TestCase { +public class TestDoubleRingBuffer extends TestCase { final double SENTINEL = Double.MIN_VALUE; @@ -38,6 +38,8 @@ private void assertEmpty(DoubleRingBuffer rb) { } catch (NoSuchElementException x) { // expected } + // region empty-test + // endregion empty-test } private void assertFull(DoubleRingBuffer rb) { diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/FloatRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestFloatRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/FloatRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestFloatRingBuffer.java index ab8b865ccca..be56827979b 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/FloatRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestFloatRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit CharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -15,7 +15,7 @@ import static org.junit.Assert.assertThrows; -public class FloatRingBufferTest extends TestCase { +public class TestFloatRingBuffer extends TestCase { final float SENTINEL = Float.MIN_VALUE; @@ -38,6 +38,8 @@ private void assertEmpty(FloatRingBuffer rb) { } catch (NoSuchElementException x) { // expected } + // region empty-test + // endregion empty-test } private void assertFull(FloatRingBuffer rb) { diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/IntRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestIntRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/IntRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestIntRingBuffer.java index 55fe5492f30..afc4509b8ef 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/IntRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestIntRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit CharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -15,7 +15,7 @@ import static org.junit.Assert.assertThrows; -public class IntRingBufferTest extends TestCase { +public class TestIntRingBuffer extends TestCase { final int SENTINEL = Integer.MIN_VALUE; @@ -38,6 +38,8 @@ private void assertEmpty(IntRingBuffer rb) { } catch (NoSuchElementException x) { // expected } + // region empty-test + // endregion empty-test } private void assertFull(IntRingBuffer rb) { diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/LongRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestLongRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/LongRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestLongRingBuffer.java index cd2bfdb1dff..f1aa4105a1e 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/LongRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestLongRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit CharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -15,7 +15,7 @@ import static org.junit.Assert.assertThrows; -public class LongRingBufferTest extends TestCase { +public class TestLongRingBuffer extends TestCase { final long SENTINEL = Long.MIN_VALUE; @@ -38,6 +38,8 @@ private void assertEmpty(LongRingBuffer rb) { } catch (NoSuchElementException x) { // expected } + // region empty-test + // endregion empty-test } private void assertFull(LongRingBuffer rb) { diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/ObjectRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestObjectRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/ObjectRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestObjectRingBuffer.java index b7a9db4817e..1dab28f6731 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/ObjectRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestObjectRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit CharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -15,7 +15,7 @@ import static org.junit.Assert.assertThrows; -public class ObjectRingBufferTest extends TestCase { +public class TestObjectRingBuffer extends TestCase { final Object SENTINEL = new Object(); @@ -38,6 +38,11 @@ private void assertEmpty(ObjectRingBuffer rb) { } catch (NoSuchElementException x) { // expected } + // region empty-test + for (Object val : rb.getStorage()) { + assertNull(val); + } + // endregion empty-test } private void assertFull(ObjectRingBuffer rb) { diff --git a/Base/src/test/java/io/deephaven/base/ringbuffer/ShortRingBufferTest.java b/Base/src/test/java/io/deephaven/base/ringbuffer/TestShortRingBuffer.java similarity index 98% rename from Base/src/test/java/io/deephaven/base/ringbuffer/ShortRingBufferTest.java rename to Base/src/test/java/io/deephaven/base/ringbuffer/TestShortRingBuffer.java index baff6f06fbd..c436efdead7 100644 --- a/Base/src/test/java/io/deephaven/base/ringbuffer/ShortRingBufferTest.java +++ b/Base/src/test/java/io/deephaven/base/ringbuffer/TestShortRingBuffer.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit CharRingBufferTest and run "./gradlew replicateRingBuffers" to regenerate +// ****** Edit TestCharRingBuffer and run "./gradlew replicateRingBuffers" to regenerate // // @formatter:off package io.deephaven.base.ringbuffer; @@ -15,7 +15,7 @@ import static org.junit.Assert.assertThrows; -public class ShortRingBufferTest extends TestCase { +public class TestShortRingBuffer extends TestCase { final short SENTINEL = Short.MIN_VALUE; @@ -38,6 +38,8 @@ private void assertEmpty(ShortRingBuffer rb) { } catch (NoSuchElementException x) { // expected } + // region empty-test + // endregion empty-test } private void assertFull(ShortRingBuffer rb) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java index ace064b9353..b1240adb367 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java @@ -340,6 +340,11 @@ public boolean isRetain() { return false; } + @Override + public boolean hasVirtualRowVariables() { + return usesI || usesII || usesK; + } + @Override public String toString() { return formulaString; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java index 9cb52213e3b..2af2eedd5ae 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java @@ -171,7 +171,7 @@ default List initDef( /** * Get a MatchPair for this column, if applicable. * - * @return + * @return the MatchPair for this column, if applicable. */ MatchPair getMatchPair(); @@ -218,6 +218,13 @@ default void validateSafeForRefresh(final BaseTable sourceTable) { */ boolean isStateless(); + /** + * Returns true if this column uses row virtual offset columns of {@code i}, {@code ii} or {@code k}. + */ + default boolean hasVirtualRowVariables() { + return false; + } + /** * Create a copy of this SelectColumn. * diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ObjectSingleValueSource.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ObjectSingleValueSource.java index 94ebdd1bbbd..3785bfadfce 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ObjectSingleValueSource.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ObjectSingleValueSource.java @@ -36,8 +36,8 @@ public class ObjectSingleValueSource extends SingleValueColumnSource private transient T prev; // region Constructor - public ObjectSingleValueSource(Class type) { - super(type); + public ObjectSingleValueSource(Class type, Class componentType) { + super(type, componentType); current = null; prev = null; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/SingleValueColumnSource.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/SingleValueColumnSource.java index bf343470884..e2e44684989 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/SingleValueColumnSource.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/SingleValueColumnSource.java @@ -8,6 +8,8 @@ import io.deephaven.engine.table.ChunkSink; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.table.impl.AbstractColumnSource; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; public abstract class SingleValueColumnSource extends AbstractColumnSource implements WritableColumnSource, ChunkSink, InMemoryColumnSource, @@ -16,8 +18,12 @@ public abstract class SingleValueColumnSource extends AbstractColumnSource protected transient long changeTime; protected boolean isTrackingPrevValues; - SingleValueColumnSource(Class type) { - super(type); + SingleValueColumnSource(@NotNull final Class type) { + this(type, null); + } + + SingleValueColumnSource(@NotNull final Class type, @Nullable final Class elementType) { + super(type, elementType); } @Override @@ -26,6 +32,10 @@ public final void startTrackingPrevValues() { } public static SingleValueColumnSource getSingleValueColumnSource(Class type) { + return getSingleValueColumnSource(type, null); + } + + public static SingleValueColumnSource getSingleValueColumnSource(Class type, Class componentType) { SingleValueColumnSource result; if (type == Byte.class || type == byte.class) { result = new ByteSingleValueSource(); @@ -44,7 +54,7 @@ public static SingleValueColumnSource getSingleValueColumnSource(Class } else if (type == Boolean.class || type == boolean.class) { result = new BooleanSingleValueSource(); } else { - result = new ObjectSingleValueSource<>(type); + result = new ObjectSingleValueSource<>(type, componentType); } // noinspection unchecked return (SingleValueColumnSource) result; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperator.java index b82ef5c83eb..95ef56f059e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperator.java @@ -130,7 +130,8 @@ public abstract void accumulateRolling(RowSequence inputKeys, protected abstract void reset(); } - protected UpdateByOperator(@NotNull final MatchPair pair, + protected UpdateByOperator( + @NotNull final MatchPair pair, @NotNull final String[] affectingColumns, @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java index 6e551476520..2463c7413c3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java @@ -4,6 +4,7 @@ package io.deephaven.engine.table.impl.updateby; import io.deephaven.api.Pair; +import io.deephaven.api.Selectable; import io.deephaven.api.updateby.ColumnUpdateOperation; import io.deephaven.api.updateby.OperationControl; import io.deephaven.api.updateby.UpdateByControl; @@ -14,6 +15,7 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; +import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.updateby.delta.*; import io.deephaven.engine.table.impl.updateby.em.*; import io.deephaven.engine.table.impl.updateby.emstd.*; @@ -23,6 +25,7 @@ import io.deephaven.engine.table.impl.updateby.rollingavg.*; import io.deephaven.engine.table.impl.updateby.rollingcount.*; import io.deephaven.engine.table.impl.updateby.rollingformula.*; +import io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.RollingFormulaMultiColumnOperator; import io.deephaven.engine.table.impl.updateby.rollinggroup.RollingGroupOperator; import io.deephaven.engine.table.impl.updateby.rollingminmax.*; import io.deephaven.engine.table.impl.updateby.rollingproduct.*; @@ -32,6 +35,8 @@ import io.deephaven.engine.table.impl.updateby.sum.*; import io.deephaven.hash.KeyedObjectHashMap; import io.deephaven.hash.KeyedObjectKey; +import io.deephaven.vector.VectorFactory; +import org.apache.commons.lang3.ArrayUtils; import org.jetbrains.annotations.NotNull; import java.math.BigDecimal; @@ -54,6 +59,7 @@ public class UpdateByOperatorFactory { private final MatchPair[] groupByColumns; @NotNull private final UpdateByControl control; + private Map> vectorColumnNameMap; public UpdateByOperatorFactory( @NotNull final TableDefinition tableDef, @@ -539,6 +545,12 @@ public Void visit(@NotNull final RollingFormulaSpec spec) { // These operators can re-use formula columns when the types match. final Map, FormulaColumn> formulaColumnMap = new HashMap<>(); + // noinspection deprecation + if (spec.paramToken().isEmpty()) { + ops.add(makeRollingFormulaMultiColumnOperator(tableDef, spec)); + return null; + } + Arrays.stream(pairs) .filter(p -> !isTimeBased || !p.rightColumn().equals(timestampCol)) .map(fc -> makeRollingFormulaOperator(fc, tableDef, formulaColumnMap, spec)) @@ -1371,52 +1383,120 @@ private UpdateByOperator makeRollingFormulaOperator(@NotNull final MatchPair pai final long prevWindowScaleUnits = rs.revWindowScale().getTimeScaleUnits(); final long fwdWindowScaleUnits = rs.fwdWindowScale().getTimeScaleUnits(); + final String formula = rs.formula(); + // noinspection deprecation + final String paramToken = rs.paramToken().orElseThrow(); + if (csType == boolean.class || csType == Boolean.class) { return new BooleanRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), - prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), + prevWindowScaleUnits, fwdWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef, compilationProcessor); } else if (csType == byte.class || csType == Byte.class) { return new ByteRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), - prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), + prevWindowScaleUnits, fwdWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef, compilationProcessor); } else if (csType == char.class || csType == Character.class) { return new CharRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), - prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), + prevWindowScaleUnits, fwdWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef, compilationProcessor); } else if (csType == short.class || csType == Short.class) { return new ShortRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), - prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), + prevWindowScaleUnits, fwdWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef, compilationProcessor); } else if (csType == int.class || csType == Integer.class) { return new IntRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), - prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), + prevWindowScaleUnits, fwdWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef, compilationProcessor); } else if (csType == long.class || csType == Long.class) { return new LongRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), - prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), + prevWindowScaleUnits, fwdWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef, compilationProcessor); } else if (csType == float.class || csType == Float.class) { return new FloatRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), - prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), + prevWindowScaleUnits, fwdWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef, compilationProcessor); } else if (csType == double.class || csType == Double.class) { return new DoubleRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), - prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), + prevWindowScaleUnits, fwdWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef, compilationProcessor); } return new ObjectRollingFormulaOperator<>(pair, affectingColumns, rs.revWindowScale().timestampCol(), - prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), + prevWindowScaleUnits, fwdWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef, compilationProcessor); } + private UpdateByOperator makeRollingFormulaMultiColumnOperator( + @NotNull final TableDefinition tableDef, + @NotNull final RollingFormulaSpec rs) { + + final long prevWindowScaleUnits = rs.revWindowScale().getTimeScaleUnits(); + final long fwdWindowScaleUnits = rs.fwdWindowScale().getTimeScaleUnits(); + + final Map> columnDefinitionMap = tableDef.getColumnNameMap(); + + // Create the colum + final SelectColumn selectColumn = SelectColumn.of(Selectable.parse(rs.formula())); + + // Get or create a column definition map where the definitions are vectors of the original column types. + if (vectorColumnNameMap == null) { + vectorColumnNameMap = new HashMap<>(); + columnDefinitionMap.forEach((key, value) -> { + final ColumnDefinition columnDef = ColumnDefinition.fromGenericType( + key, + VectorFactory.forElementType(value.getDataType()).vectorType(), + value.getDataType()); + vectorColumnNameMap.put(key, columnDef); + }); + } + + // Get the input column names and data types from the formula. + final String[] inputColumnNames = + selectColumn.initDef(vectorColumnNameMap, compilationProcessor).toArray(String[]::new); + if (!selectColumn.getColumnArrays().isEmpty()) { + throw new IllegalArgumentException("RollingFormulaMultiColumnOperator does not support column arrays (" + + selectColumn.getColumnArrays() + ")"); + } + if (selectColumn.hasVirtualRowVariables()) { + throw new IllegalArgumentException("RollingFormula does not support virtual row variables"); + } + final Class[] inputColumnTypes = new Class[inputColumnNames.length]; + final Class[] inputVectorTypes = new Class[inputColumnNames.length]; + + for (int i = 0; i < inputColumnNames.length; i++) { + final ColumnDefinition columnDef = columnDefinitionMap.get(inputColumnNames[i]); + inputColumnTypes[i] = columnDef.getDataType(); + inputVectorTypes[i] = vectorColumnNameMap.get(inputColumnNames[i]).getDataType(); + } + + final String[] affectingColumns; + if (rs.revWindowScale().timestampCol() == null) { + affectingColumns = inputColumnNames; + } else { + affectingColumns = ArrayUtils.add(inputColumnNames, rs.revWindowScale().timestampCol()); + } + + // Create a new column pair with the same name for the left and right columns + final MatchPair pair = new MatchPair(selectColumn.getName(), selectColumn.getName()); + + return new RollingFormulaMultiColumnOperator( + pair, + affectingColumns, + rs.revWindowScale().timestampCol(), + prevWindowScaleUnits, + fwdWindowScaleUnits, + selectColumn, + inputColumnNames, + inputColumnTypes, + inputVectorTypes); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/BigIntegerRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/BigIntegerRollingAvgOperator.java index 817bfeee1d8..6a869d58aa1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/BigIntegerRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/BigIntegerRollingAvgOperator.java @@ -10,7 +10,6 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.updateby.UpdateByOperator; import io.deephaven.engine.table.impl.updateby.internal.BaseObjectUpdateByOperator; -import io.deephaven.engine.table.impl.util.RowRedirection; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java index b5e0a9560a9..5f153a1fe24 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java @@ -38,6 +38,8 @@ abstract class BaseRollingFormulaOperator extends UpdateByOperator { final TableDefinition tableDef; final FormulaColumn formulaColumn; + final Class inputColumnType; + final Class inputComponentType; final Class inputVectorType; protected WritableColumnSource primitiveOutputSource; @@ -104,7 +106,9 @@ public BaseRollingFormulaOperator( final String outputColumnName = pair.leftColumn; - final Class inputColumnType = tableDef.getColumn(pair.rightColumn).getDataType(); + final ColumnDefinition columnDefinition = tableDef.getColumn(pair.rightColumn); + inputColumnType = columnDefinition.getDataType(); + inputComponentType = columnDefinition.getComponentType(); inputVectorType = VectorFactory.forElementType(inputColumnType).vectorType(); // Re-use the formula column if it's already been created for this type. No need to synchronize; these @@ -126,15 +130,17 @@ protected BaseRollingFormulaOperator( @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, final long forwardWindowScaleUnits, - final Class inputVectorType, + final Class columnType, + final Class componentType, + final Class vectorType, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, true); this.formulaColumnMap = formulaColumnMap; this.tableDef = tableDef; - - final Class columnType = tableDef.getColumn(pair.rightColumn).getDataType(); - this.inputVectorType = inputVectorType; + this.inputColumnType = columnType; + this.inputComponentType = componentType; + this.inputVectorType = vectorType; // Re-use the formula column already created for this type. formulaColumn = formulaColumnMap.computeIfAbsent(columnType, t -> { @@ -172,49 +178,35 @@ protected static IntConsumer getChunkSetter( "Output chunk type should not be Boolean but should have been reinterpreted to byte"); } if (chunkType == ChunkType.Byte) { - return i -> { - final WritableByteChunk writableChunk = valueChunk.asWritableByteChunk(); - writableChunk.set(i, formulaOutputSource.getByte(0)); - }; + final WritableByteChunk writableChunk = valueChunk.asWritableByteChunk(); + return index -> writableChunk.set(index, formulaOutputSource.getByte(0)); } if (chunkType == ChunkType.Char) { - return i -> { - final WritableCharChunk writableChunk = valueChunk.asWritableCharChunk(); - writableChunk.set(i, formulaOutputSource.getChar(0)); - }; + final WritableCharChunk writableChunk = valueChunk.asWritableCharChunk(); + return index -> writableChunk.set(index, formulaOutputSource.getChar(0)); } if (chunkType == ChunkType.Double) { - return i -> { - final WritableDoubleChunk writableChunk = valueChunk.asWritableDoubleChunk(); - writableChunk.set(i, formulaOutputSource.getDouble(0)); - }; + final WritableDoubleChunk writableChunk = valueChunk.asWritableDoubleChunk(); + return index -> writableChunk.set(index, formulaOutputSource.getDouble(0)); } if (chunkType == ChunkType.Float) { - return i -> { - final WritableFloatChunk writableChunk = valueChunk.asWritableFloatChunk(); - writableChunk.set(i, formulaOutputSource.getFloat(0)); - }; + final WritableFloatChunk writableChunk = valueChunk.asWritableFloatChunk(); + return index -> writableChunk.set(index, formulaOutputSource.getFloat(0)); } if (chunkType == ChunkType.Int) { - return i -> { - final WritableIntChunk writableChunk = valueChunk.asWritableIntChunk(); - writableChunk.set(i, formulaOutputSource.getInt(0)); - }; + final WritableIntChunk writableChunk = valueChunk.asWritableIntChunk(); + return index -> writableChunk.set(index, formulaOutputSource.getInt(0)); } if (chunkType == ChunkType.Long) { - return i -> { - final WritableLongChunk writableChunk = valueChunk.asWritableLongChunk(); - writableChunk.set(i, formulaOutputSource.getLong(0)); - }; + final WritableLongChunk writableChunk = valueChunk.asWritableLongChunk(); + return index -> writableChunk.set(index, formulaOutputSource.getLong(0)); } if (chunkType == ChunkType.Short) { - return i -> { - final WritableShortChunk writableChunk = valueChunk.asWritableShortChunk(); - writableChunk.set(i, formulaOutputSource.getShort(0)); - }; + final WritableShortChunk writableChunk = valueChunk.asWritableShortChunk(); + return index -> writableChunk.set(index, formulaOutputSource.getShort(0)); } - return i -> { - final WritableObjectChunk writableChunk = valueChunk.asWritableObjectChunk(); + final WritableObjectChunk writableChunk = valueChunk.asWritableObjectChunk(); + return index -> { Object result = formulaOutputSource.get(0); if (result instanceof RingBufferVectorWrapper) { // Handle the rare (and probably not useful) case where the formula is an identity. We need to @@ -222,7 +214,7 @@ protected static IntConsumer getChunkSetter( // live data in the ring. result = ((Vector) result).getDirect(); } - writableChunk.set(i, result); + writableChunk.set(index, result); }; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java index b54d4dcc02d..7144e591683 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java @@ -41,7 +41,6 @@ public class BooleanRollingFormulaOperator extends BaseRollingFormulaOperator { private static final int BUFFER_INITIAL_CAPACITY = 128; protected class Context extends BaseRollingFormulaOperator.Context { - private final ColumnSource formulaOutputSource; private final IntConsumer outputSetter; private ByteChunk influencerValuesChunk; @@ -64,11 +63,12 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { final SingleValueColumnSource> formulaInputSource = (SingleValueColumnSource>) SingleValueColumnSource .getSingleValueColumnSource(inputVectorType); - formulaInputSource.set(new ObjectRingBufferVectorWrapper(windowValues, inputVectorType)); + // noinspection rawtypes + formulaInputSource.set(new ObjectRingBufferVectorWrapper(windowValues, inputComponentType)); formulaCopy.initInputs(RowSetFactory.flat(1).toTracking(), Collections.singletonMap(PARAM_COLUMN_NAME, formulaInputSource)); - formulaOutputSource = formulaCopy.getDataView(); + final ColumnSource formulaOutputSource = formulaCopy.getDataView(); outputSetter = getChunkSetter(outputValues, formulaOutputSource); } @@ -173,11 +173,22 @@ protected BooleanRollingFormulaOperator( @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, final long forwardWindowScaleUnits, + final Class columnType, + final Class componentType, final Class vectorType, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, vectorType, - formulaColumnMap, tableDef); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + columnType, + componentType, + vectorType, + formulaColumnMap, + tableDef); } @Override @@ -187,6 +198,8 @@ public UpdateByOperator copy() { timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, + inputColumnType, + inputComponentType, inputVectorType, formulaColumnMap, tableDef); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java index 4cb47097afa..477d949f647 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java @@ -48,7 +48,6 @@ public class ByteRollingFormulaOperator extends BaseRollingFormulaOperator { // endregion extra-fields protected class Context extends BaseRollingFormulaOperator.Context { - private final ColumnSource formulaOutputSource; private final IntConsumer outputSetter; private ByteChunk influencerValuesChunk; @@ -71,7 +70,8 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { formulaCopy.initInputs(RowSetFactory.flat(1).toTracking(), Collections.singletonMap(PARAM_COLUMN_NAME, formulaInputSource)); - formulaOutputSource = ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); + final ColumnSource formulaOutputSource = + ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); outputSetter = getChunkSetter(outputValues, formulaOutputSource); } @@ -169,8 +169,17 @@ public ByteRollingFormulaOperator( // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, compilationProcessor); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + formula, + paramToken, + formulaColumnMap, + tableDef, + compilationProcessor); // region constructor // endregion constructor } @@ -181,14 +190,25 @@ protected ByteRollingFormulaOperator( @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, final long forwardWindowScaleUnits, + final Class columnType, + final Class componentType, final Class vectorType, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, vectorType, - formulaColumnMap, tableDef); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + columnType, + componentType, + vectorType, + formulaColumnMap, + tableDef); // region constructor // endregion constructor } @@ -200,6 +220,8 @@ public UpdateByOperator copy() { timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, + inputColumnType, + inputComponentType, inputVectorType, formulaColumnMap, tableDef diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java index 7018c431482..1882f881aad 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java @@ -44,7 +44,6 @@ public class CharRollingFormulaOperator extends BaseRollingFormulaOperator { // endregion extra-fields protected class Context extends BaseRollingFormulaOperator.Context { - private final ColumnSource formulaOutputSource; private final IntConsumer outputSetter; private CharChunk influencerValuesChunk; @@ -67,7 +66,8 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { formulaCopy.initInputs(RowSetFactory.flat(1).toTracking(), Collections.singletonMap(PARAM_COLUMN_NAME, formulaInputSource)); - formulaOutputSource = ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); + final ColumnSource formulaOutputSource = + ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); outputSetter = getChunkSetter(outputValues, formulaOutputSource); } @@ -165,8 +165,17 @@ public CharRollingFormulaOperator( // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, compilationProcessor); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + formula, + paramToken, + formulaColumnMap, + tableDef, + compilationProcessor); // region constructor // endregion constructor } @@ -177,14 +186,25 @@ protected CharRollingFormulaOperator( @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, final long forwardWindowScaleUnits, + final Class columnType, + final Class componentType, final Class vectorType, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, vectorType, - formulaColumnMap, tableDef); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + columnType, + componentType, + vectorType, + formulaColumnMap, + tableDef); // region constructor // endregion constructor } @@ -196,6 +216,8 @@ public UpdateByOperator copy() { timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, + inputColumnType, + inputComponentType, inputVectorType, formulaColumnMap, tableDef diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java index bb0c449e394..2a4f4026415 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java @@ -48,7 +48,6 @@ public class DoubleRollingFormulaOperator extends BaseRollingFormulaOperator { // endregion extra-fields protected class Context extends BaseRollingFormulaOperator.Context { - private final ColumnSource formulaOutputSource; private final IntConsumer outputSetter; private DoubleChunk influencerValuesChunk; @@ -71,7 +70,8 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { formulaCopy.initInputs(RowSetFactory.flat(1).toTracking(), Collections.singletonMap(PARAM_COLUMN_NAME, formulaInputSource)); - formulaOutputSource = ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); + final ColumnSource formulaOutputSource = + ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); outputSetter = getChunkSetter(outputValues, formulaOutputSource); } @@ -169,8 +169,17 @@ public DoubleRollingFormulaOperator( // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, compilationProcessor); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + formula, + paramToken, + formulaColumnMap, + tableDef, + compilationProcessor); // region constructor // endregion constructor } @@ -181,14 +190,25 @@ protected DoubleRollingFormulaOperator( @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, final long forwardWindowScaleUnits, + final Class columnType, + final Class componentType, final Class vectorType, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, vectorType, - formulaColumnMap, tableDef); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + columnType, + componentType, + vectorType, + formulaColumnMap, + tableDef); // region constructor // endregion constructor } @@ -200,6 +220,8 @@ public UpdateByOperator copy() { timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, + inputColumnType, + inputComponentType, inputVectorType, formulaColumnMap, tableDef diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java index 66a65898978..398685f8da9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java @@ -48,7 +48,6 @@ public class FloatRollingFormulaOperator extends BaseRollingFormulaOperator { // endregion extra-fields protected class Context extends BaseRollingFormulaOperator.Context { - private final ColumnSource formulaOutputSource; private final IntConsumer outputSetter; private FloatChunk influencerValuesChunk; @@ -71,7 +70,8 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { formulaCopy.initInputs(RowSetFactory.flat(1).toTracking(), Collections.singletonMap(PARAM_COLUMN_NAME, formulaInputSource)); - formulaOutputSource = ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); + final ColumnSource formulaOutputSource = + ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); outputSetter = getChunkSetter(outputValues, formulaOutputSource); } @@ -169,8 +169,17 @@ public FloatRollingFormulaOperator( // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, compilationProcessor); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + formula, + paramToken, + formulaColumnMap, + tableDef, + compilationProcessor); // region constructor // endregion constructor } @@ -181,14 +190,25 @@ protected FloatRollingFormulaOperator( @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, final long forwardWindowScaleUnits, + final Class columnType, + final Class componentType, final Class vectorType, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, vectorType, - formulaColumnMap, tableDef); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + columnType, + componentType, + vectorType, + formulaColumnMap, + tableDef); // region constructor // endregion constructor } @@ -200,6 +220,8 @@ public UpdateByOperator copy() { timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, + inputColumnType, + inputComponentType, inputVectorType, formulaColumnMap, tableDef diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java index c0b07b52af9..e3b432dde19 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java @@ -48,7 +48,6 @@ public class IntRollingFormulaOperator extends BaseRollingFormulaOperator { // endregion extra-fields protected class Context extends BaseRollingFormulaOperator.Context { - private final ColumnSource formulaOutputSource; private final IntConsumer outputSetter; private IntChunk influencerValuesChunk; @@ -71,7 +70,8 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { formulaCopy.initInputs(RowSetFactory.flat(1).toTracking(), Collections.singletonMap(PARAM_COLUMN_NAME, formulaInputSource)); - formulaOutputSource = ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); + final ColumnSource formulaOutputSource = + ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); outputSetter = getChunkSetter(outputValues, formulaOutputSource); } @@ -169,8 +169,17 @@ public IntRollingFormulaOperator( // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, compilationProcessor); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + formula, + paramToken, + formulaColumnMap, + tableDef, + compilationProcessor); // region constructor // endregion constructor } @@ -181,14 +190,25 @@ protected IntRollingFormulaOperator( @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, final long forwardWindowScaleUnits, + final Class columnType, + final Class componentType, final Class vectorType, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, vectorType, - formulaColumnMap, tableDef); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + columnType, + componentType, + vectorType, + formulaColumnMap, + tableDef); // region constructor // endregion constructor } @@ -200,6 +220,8 @@ public UpdateByOperator copy() { timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, + inputColumnType, + inputComponentType, inputVectorType, formulaColumnMap, tableDef diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java index 9b3eb63de7b..dc2051b2701 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java @@ -48,7 +48,6 @@ public class LongRollingFormulaOperator extends BaseRollingFormulaOperator { // endregion extra-fields protected class Context extends BaseRollingFormulaOperator.Context { - private final ColumnSource formulaOutputSource; private final IntConsumer outputSetter; private LongChunk influencerValuesChunk; @@ -71,7 +70,8 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { formulaCopy.initInputs(RowSetFactory.flat(1).toTracking(), Collections.singletonMap(PARAM_COLUMN_NAME, formulaInputSource)); - formulaOutputSource = ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); + final ColumnSource formulaOutputSource = + ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); outputSetter = getChunkSetter(outputValues, formulaOutputSource); } @@ -169,8 +169,17 @@ public LongRollingFormulaOperator( // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, compilationProcessor); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + formula, + paramToken, + formulaColumnMap, + tableDef, + compilationProcessor); // region constructor // endregion constructor } @@ -181,14 +190,25 @@ protected LongRollingFormulaOperator( @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, final long forwardWindowScaleUnits, + final Class columnType, + final Class componentType, final Class vectorType, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, vectorType, - formulaColumnMap, tableDef); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + columnType, + componentType, + vectorType, + formulaColumnMap, + tableDef); // region constructor // endregion constructor } @@ -200,6 +220,8 @@ public UpdateByOperator copy() { timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, + inputColumnType, + inputComponentType, inputVectorType, formulaColumnMap, tableDef diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java index 8b062025e52..c344a150ca5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java @@ -13,7 +13,6 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; -import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; @@ -43,7 +42,6 @@ public class ObjectRollingFormulaOperator extends BaseRollingFormulaOperator // endregion extra-fields protected class Context extends BaseRollingFormulaOperator.Context { - private final ColumnSource formulaOutputSource; private final IntConsumer outputSetter; private ObjectChunk influencerValuesChunk; @@ -62,12 +60,12 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { final SingleValueColumnSource> formulaInputSource = (SingleValueColumnSource>) SingleValueColumnSource .getSingleValueColumnSource(inputVectorType); - formulaInputSource.set(new ObjectRingBufferVectorWrapper(windowValues, inputVectorType)); + // noinspection rawtypes + formulaInputSource.set(new ObjectRingBufferVectorWrapper(windowValues, inputComponentType)); formulaCopy.initInputs(RowSetFactory.flat(1).toTracking(), Collections.singletonMap(PARAM_COLUMN_NAME, formulaInputSource)); - formulaOutputSource = formulaCopy.getDataView(); - outputSetter = getChunkSetter(outputValues, formulaOutputSource); + outputSetter = getChunkSetter(outputValues, formulaCopy.getDataView()); } @Override @@ -172,11 +170,22 @@ protected ObjectRollingFormulaOperator( @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, final long forwardWindowScaleUnits, + final Class columnType, + final Class componentType, final Class vectorType, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, vectorType, - formulaColumnMap, tableDef); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + columnType, + componentType, + vectorType, + formulaColumnMap, + tableDef); } @Override @@ -186,6 +195,8 @@ public UpdateByOperator copy() { timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, + inputColumnType, + inputComponentType, inputVectorType, formulaColumnMap, tableDef); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java index 7cce65c7cb5..1ab9962c011 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java @@ -48,7 +48,6 @@ public class ShortRollingFormulaOperator extends BaseRollingFormulaOperator { // endregion extra-fields protected class Context extends BaseRollingFormulaOperator.Context { - private final ColumnSource formulaOutputSource; private final IntConsumer outputSetter; private ShortChunk influencerValuesChunk; @@ -71,7 +70,8 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { formulaCopy.initInputs(RowSetFactory.flat(1).toTracking(), Collections.singletonMap(PARAM_COLUMN_NAME, formulaInputSource)); - formulaOutputSource = ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); + final ColumnSource formulaOutputSource = + ReinterpretUtils.maybeConvertToPrimitive(formulaCopy.getDataView()); outputSetter = getChunkSetter(outputValues, formulaOutputSource); } @@ -169,8 +169,17 @@ public ShortRollingFormulaOperator( // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, compilationProcessor); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + formula, + paramToken, + formulaColumnMap, + tableDef, + compilationProcessor); // region constructor // endregion constructor } @@ -181,14 +190,25 @@ protected ShortRollingFormulaOperator( @Nullable final String timestampColumnName, final long reverseWindowScaleUnits, final long forwardWindowScaleUnits, + final Class columnType, + final Class componentType, final Class vectorType, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, vectorType, - formulaColumnMap, tableDef); + super( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + columnType, + componentType, + vectorType, + formulaColumnMap, + tableDef); // region constructor // endregion constructor } @@ -200,6 +220,8 @@ public UpdateByOperator copy() { timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, + inputColumnType, + inputComponentType, inputVectorType, formulaColumnMap, tableDef diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ByteRingBufferVectorWrapper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ByteRingBufferVectorWrapper.java index 5fc15cb3280..e3323a3a689 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ByteRingBufferVectorWrapper.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ByteRingBufferVectorWrapper.java @@ -13,7 +13,7 @@ import io.deephaven.vector.ByteVectorDirect; import io.deephaven.vector.ByteVectorSlice; -public class ByteRingBufferVectorWrapper implements ByteVector, RingBufferVectorWrapper { +public class ByteRingBufferVectorWrapper implements ByteVector, RingBufferVectorWrapper { private final ByteRingBuffer ringBuffer; public ByteRingBufferVectorWrapper(final ByteRingBuffer ringBuffer) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/CharRingBufferVectorWrapper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/CharRingBufferVectorWrapper.java index b30dd79b925..6c1cb68a3c2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/CharRingBufferVectorWrapper.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/CharRingBufferVectorWrapper.java @@ -9,7 +9,7 @@ import io.deephaven.vector.CharVectorDirect; import io.deephaven.vector.CharVectorSlice; -public class CharRingBufferVectorWrapper implements CharVector, RingBufferVectorWrapper { +public class CharRingBufferVectorWrapper implements CharVector, RingBufferVectorWrapper { private final CharRingBuffer ringBuffer; public CharRingBufferVectorWrapper(final CharRingBuffer ringBuffer) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/DoubleRingBufferVectorWrapper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/DoubleRingBufferVectorWrapper.java index b2e07f33a4e..b8a336a1117 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/DoubleRingBufferVectorWrapper.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/DoubleRingBufferVectorWrapper.java @@ -13,7 +13,7 @@ import io.deephaven.vector.DoubleVectorDirect; import io.deephaven.vector.DoubleVectorSlice; -public class DoubleRingBufferVectorWrapper implements DoubleVector, RingBufferVectorWrapper { +public class DoubleRingBufferVectorWrapper implements DoubleVector, RingBufferVectorWrapper { private final DoubleRingBuffer ringBuffer; public DoubleRingBufferVectorWrapper(final DoubleRingBuffer ringBuffer) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/FloatRingBufferVectorWrapper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/FloatRingBufferVectorWrapper.java index 1e636dfe397..46f92a96db5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/FloatRingBufferVectorWrapper.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/FloatRingBufferVectorWrapper.java @@ -13,7 +13,7 @@ import io.deephaven.vector.FloatVectorDirect; import io.deephaven.vector.FloatVectorSlice; -public class FloatRingBufferVectorWrapper implements FloatVector, RingBufferVectorWrapper { +public class FloatRingBufferVectorWrapper implements FloatVector, RingBufferVectorWrapper { private final FloatRingBuffer ringBuffer; public FloatRingBufferVectorWrapper(final FloatRingBuffer ringBuffer) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/IntRingBufferVectorWrapper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/IntRingBufferVectorWrapper.java index 28c33bb0c22..6c150a43949 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/IntRingBufferVectorWrapper.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/IntRingBufferVectorWrapper.java @@ -13,7 +13,7 @@ import io.deephaven.vector.IntVectorDirect; import io.deephaven.vector.IntVectorSlice; -public class IntRingBufferVectorWrapper implements IntVector, RingBufferVectorWrapper { +public class IntRingBufferVectorWrapper implements IntVector, RingBufferVectorWrapper { private final IntRingBuffer ringBuffer; public IntRingBufferVectorWrapper(final IntRingBuffer ringBuffer) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/LongRingBufferVectorWrapper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/LongRingBufferVectorWrapper.java index b7e2a838d3d..719c355af87 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/LongRingBufferVectorWrapper.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/LongRingBufferVectorWrapper.java @@ -13,7 +13,7 @@ import io.deephaven.vector.LongVectorDirect; import io.deephaven.vector.LongVectorSlice; -public class LongRingBufferVectorWrapper implements LongVector, RingBufferVectorWrapper { +public class LongRingBufferVectorWrapper implements LongVector, RingBufferVectorWrapper { private final LongRingBuffer ringBuffer; public LongRingBufferVectorWrapper(final LongRingBuffer ringBuffer) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ObjectRingBufferVectorWrapper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ObjectRingBufferVectorWrapper.java index fde4452e6ea..775f792e82c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ObjectRingBufferVectorWrapper.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ObjectRingBufferVectorWrapper.java @@ -9,7 +9,7 @@ import io.deephaven.vector.ObjectVectorDirect; import io.deephaven.vector.ObjectVectorSlice; -public class ObjectRingBufferVectorWrapper implements ObjectVector, RingBufferVectorWrapper { +public class ObjectRingBufferVectorWrapper implements ObjectVector, RingBufferVectorWrapper> { private final ObjectRingBuffer ringBuffer; private final Class componentType; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/RingBufferVectorWrapper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/RingBufferVectorWrapper.java index 48892888ac0..88932bde0af 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/RingBufferVectorWrapper.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/RingBufferVectorWrapper.java @@ -3,5 +3,43 @@ // package io.deephaven.engine.table.impl.updateby.rollingformula.ringbuffervectorwrapper; -public interface RingBufferVectorWrapper { +import io.deephaven.base.ringbuffer.*; +import io.deephaven.vector.Vector; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * This helper class will expose a {@link Vector} interface for a {@link RingBuffer}. + */ +public interface RingBufferVectorWrapper> extends Vector { + /** + * Create a {@link RingBufferVectorWrapper} for the given {@link RingBuffer}. Optionally, a component type can be + * supplied (although it will be ignored for all wrappers but {@link ObjectRingBufferVectorWrapper}). + */ + static > RingBufferVectorWrapper makeRingBufferVectorWrapper( + @NotNull final RingBuffer buffer, + @Nullable final Class componentType) { + final RingBufferVectorWrapper result; + final Class bufferClass = buffer.getClass(); + if (bufferClass == CharRingBuffer.class) { + result = new CharRingBufferVectorWrapper((CharRingBuffer) buffer); + } else if (bufferClass == ByteRingBuffer.class) { + result = new ByteRingBufferVectorWrapper((ByteRingBuffer) buffer); + } else if (bufferClass == DoubleRingBuffer.class) { + result = new DoubleRingBufferVectorWrapper((DoubleRingBuffer) buffer); + } else if (bufferClass == FloatRingBuffer.class) { + result = new FloatRingBufferVectorWrapper((FloatRingBuffer) buffer); + } else if (bufferClass == IntRingBuffer.class) { + result = new IntRingBufferVectorWrapper((IntRingBuffer) buffer); + } else if (bufferClass == LongRingBuffer.class) { + result = new LongRingBufferVectorWrapper((LongRingBuffer) buffer); + } else if (bufferClass == ShortRingBuffer.class) { + result = new ShortRingBufferVectorWrapper((ShortRingBuffer) buffer); + } else { + // noinspection unchecked + result = new ObjectRingBufferVectorWrapper<>((ObjectRingBuffer) buffer, (Class) componentType); + } + // noinspection unchecked + return (RingBufferVectorWrapper) result; + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ShortRingBufferVectorWrapper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ShortRingBufferVectorWrapper.java index 35b36a303a6..29fb081133d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ShortRingBufferVectorWrapper.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ringbuffervectorwrapper/ShortRingBufferVectorWrapper.java @@ -13,7 +13,7 @@ import io.deephaven.vector.ShortVectorDirect; import io.deephaven.vector.ShortVectorSlice; -public class ShortRingBufferVectorWrapper implements ShortVector, RingBufferVectorWrapper { +public class ShortRingBufferVectorWrapper implements ShortVector, RingBufferVectorWrapper { private final ShortRingBuffer ringBuffer; public ShortRingBufferVectorWrapper(final ShortRingBuffer ringBuffer) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/RollingFormulaMultiColumnOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/RollingFormulaMultiColumnOperator.java new file mode 100644 index 00000000000..a97dc9d1617 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/RollingFormulaMultiColumnOperator.java @@ -0,0 +1,419 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn; + +import io.deephaven.base.ringbuffer.*; +import io.deephaven.chunk.*; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; +import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.select.SelectColumn; +import io.deephaven.engine.table.impl.sources.*; +import io.deephaven.engine.table.impl.updateby.UpdateByOperator; +import io.deephaven.engine.table.impl.updateby.rollingformula.ringbuffervectorwrapper.RingBufferVectorWrapper; +import io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.windowconsumer.RingBufferWindowConsumer; +import io.deephaven.engine.table.impl.util.ChunkUtils; +import io.deephaven.engine.table.impl.util.RowRedirection; +import io.deephaven.vector.Vector; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.*; +import java.util.function.IntConsumer; + +import static io.deephaven.util.QueryConstants.*; + +public class RollingFormulaMultiColumnOperator extends UpdateByOperator { + private static final int BUFFER_INITIAL_CAPACITY = 512; + + private final SelectColumn selectColumn; + private final String[] inputColumnNames; + private final Class[] inputColumnTypes; + private final Class[] inputVectorTypes; + + private WritableColumnSource primitiveOutputSource; + private WritableColumnSource outputSource; + private WritableColumnSource maybeInnerSource; + private ChunkType outputChunkType; + + private class Context extends UpdateByOperator.Context { + private final ChunkSink.FillFromContext outputFillContext; + private final WritableChunk outputValues; + + private final IntConsumer outputSetter; + private final IntConsumer outputNullSetter; + + private final RingBufferWindowConsumer[] inputConsumers; + + @SuppressWarnings("unused") + private Context(final int affectedChunkSize, final int influencerChunkSize) { + outputFillContext = primitiveOutputSource.makeFillFromContext(affectedChunkSize); + outputValues = outputChunkType.makeWritableChunk(affectedChunkSize); + + // Make a copy of the operator formula column. + final SelectColumn contextSelectColumn = selectColumn.copy(); + + inputConsumers = new RingBufferWindowConsumer[inputColumnNames.length]; + + // To perform the calculation, we will leverage SelectColumn and for its input sources we create a set of + // SingleValueColumnSources, each containing a Vector of values. This vector will contain exactly the + // values from the input columns that are appropriate for output row given the window configuration. + // The formula column is evaluated once per output row and the result written to the output column + // source. + + // The SingleValueColumnSources is backed by RingBuffers through use of a RingBufferVectorWrapper. + // The underlying RingBuffer is updated with the values from the input columns with assistance from + // the RingBufferWindowConsumer class, which abstracts the process of pushing and popping values from input + // column data chunks into the RingBuffer. + + final Map> inputSources = new HashMap<>(); + for (int i = 0; i < inputColumnNames.length; i++) { + final String inputColumnName = inputColumnNames[i]; + final Class inputColumnType = inputColumnTypes[i]; + final Class inputVectorType = inputVectorTypes[i]; + + // Create and store the ring buffer for the input column. + final RingBuffer ringBuffer = RingBuffer.makeRingBuffer( + inputColumnType, + BUFFER_INITIAL_CAPACITY, + true); + + // Create a single value column source wrapping the ring buffer. + // noinspection unchecked + final SingleValueColumnSource> formulaInputSource = + (SingleValueColumnSource>) SingleValueColumnSource + .getSingleValueColumnSource(inputVectorType, inputColumnType); + + final RingBufferVectorWrapper wrapper = RingBufferVectorWrapper.makeRingBufferVectorWrapper( + ringBuffer, + inputColumnType); + + formulaInputSource.set(wrapper); + + inputSources.put(inputColumnName, formulaInputSource); + + inputConsumers[i] = RingBufferWindowConsumer.create(ringBuffer); + } + contextSelectColumn.initInputs(RowSetFactory.flat(1).toTracking(), inputSources); + + final ColumnSource formulaOutputSource = + ReinterpretUtils.maybeConvertToPrimitive(contextSelectColumn.getDataView()); + + outputSetter = getChunkSetter(outputValues, formulaOutputSource); + outputNullSetter = getChunkNullSetter(outputValues); + } + + @Override + protected void setValueChunks(@NotNull Chunk[] valueChunks) { + // Assign the influencer values chunks to the input consumers. + for (int i = 0; i < valueChunks.length; i++) { + inputConsumers[i].setInputChunk(valueChunks[i]); + } + } + + @Override + protected void push(int pos, int count) { + throw new IllegalStateException("RollingFormulaMultiColumnOperator.Context.push should never be called."); + } + + @Override + public void accumulateCumulative( + @NotNull final RowSequence inputKeys, + @NotNull final Chunk[] valueChunkArr, + @Nullable final LongChunk tsChunk, + final int len) { + throw new UnsupportedOperationException("RollingFormula is not supported in cumulative operations."); + } + + @Override + public void accumulateRolling( + @NotNull final RowSequence inputKeys, + @NotNull final Chunk[] influencerValueChunkArr, + @Nullable final LongChunk affectedPosChunk, + @Nullable final LongChunk influencerPosChunk, + @NotNull final IntChunk pushChunk, + @NotNull final IntChunk popChunk, + final int len) { + + setValueChunks(influencerValueChunkArr); + setPosChunks(affectedPosChunk, influencerPosChunk); + + int pushIndex = 0; + + // chunk processing + for (int ii = 0; ii < len; ii++) { + final int pushCount = pushChunk.get(ii); + final int popCount = popChunk.get(ii); + + if (pushCount == NULL_INT) { + outputNullSetter.accept(ii); + continue; + } + + // pop for this row + if (popCount > 0) { + for (RingBufferWindowConsumer consumer : inputConsumers) { + consumer.pop(popCount); + } + } + + // push for this row + if (pushCount > 0) { + for (RingBufferWindowConsumer consumer : inputConsumers) { + consumer.push(pushIndex, pushCount); + } + pushIndex += pushCount; + } + + // If not empty (even if completely full of null), run the formula over the window values. + outputSetter.accept(ii); + } + + // chunk output to column + writeToOutputColumn(inputKeys); + } + + @Override + protected void writeToOutputChunk(int outIdx) { + throw new IllegalStateException( + "RollingFormulaMultiColumnOperator.Context.writeToOutputChunk should never be called."); + } + + @Override + public void writeToOutputColumn(@NotNull final RowSequence inputKeys) { + primitiveOutputSource.fillFromChunk(outputFillContext, outputValues, inputKeys); + } + + @Override + public void reset() { + // Clear all the ring buffers for re-use + for (RingBufferWindowConsumer consumer : inputConsumers) { + consumer.reset(); + } + nullCount = 0; + } + + @Override + public void close() { + outputValues.close(); + outputFillContext.close(); + } + } + + /** + * Create a new RollingFormulaMultiColumnOperator. + * + * @param pair Contains the output column name as a MatchPair + * @param affectingColumns The names of the columns that when changed would affect this formula output + * @param timestampColumnName The name of the column containing timestamps for time-based calculations (or null when + * not time-based) + * @param reverseWindowScaleUnits The size of the reverse window in ticks (or nanoseconds when time-based) + * @param forwardWindowScaleUnits The size of the forward window in ticks (or nanoseconds when time-based) + * @param selectColumn The {@link SelectColumn} specifying the calculation to be performed + * @param inputColumnNames The names of the columns to be used as inputs + * @param inputColumnTypes The types of the columns to be used as inputs + * @param inputVectorTypes The vector types of the columns to be used as inputs + */ + public RollingFormulaMultiColumnOperator( + @NotNull final MatchPair pair, + @NotNull final String[] affectingColumns, + @Nullable final String timestampColumnName, + final long reverseWindowScaleUnits, + final long forwardWindowScaleUnits, + @NotNull final SelectColumn selectColumn, + @NotNull final String[] inputColumnNames, + @NotNull final Class[] inputColumnTypes, + @NotNull final Class[] inputVectorTypes) { + super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, true); + this.selectColumn = selectColumn; + this.inputColumnNames = inputColumnNames; + this.inputColumnTypes = inputColumnTypes; + this.inputVectorTypes = inputVectorTypes; + } + + @Override + public UpdateByOperator copy() { + return new RollingFormulaMultiColumnOperator( + pair, + affectingColumns, + timestampColumnName, + reverseWindowScaleUnits, + forwardWindowScaleUnits, + selectColumn, + inputColumnNames, + inputColumnTypes, + inputVectorTypes); + } + + @Override + public void initializeSources(@NotNull final Table source, @Nullable final RowRedirection rowRedirection) { + this.rowRedirection = rowRedirection; + + if (rowRedirection != null) { + // region create-dense + maybeInnerSource = ArrayBackedColumnSource.getMemoryColumnSource(0, selectColumn.getReturnedType(), + selectColumn.getReturnedComponentType()); + // endregion create-dense + outputSource = WritableRedirectedColumnSource.maybeRedirect(rowRedirection, maybeInnerSource, 0); + } else { + maybeInnerSource = null; + // region create-sparse + outputSource = SparseArrayColumnSource.getSparseMemoryColumnSource(0, selectColumn.getReturnedType(), + selectColumn.getReturnedComponentType()); + // endregion create-sparse + } + + primitiveOutputSource = ReinterpretUtils.maybeConvertToWritablePrimitive(outputSource); + + outputChunkType = primitiveOutputSource.getChunkType(); + } + + protected static IntConsumer getChunkSetter( + final WritableChunk valueChunk, + final ColumnSource formulaOutputSource) { + final ChunkType chunkType = valueChunk.getChunkType(); + switch (chunkType) { + case Boolean: + throw new IllegalStateException( + "Output chunk type should not be Boolean but should have been reinterpreted to byte"); + case Byte: + final WritableByteChunk byteChunk = valueChunk.asWritableByteChunk(); + return index -> byteChunk.set(index, formulaOutputSource.getByte(0)); + + case Char: + final WritableCharChunk charChunk = valueChunk.asWritableCharChunk(); + return index -> charChunk.set(index, formulaOutputSource.getChar(0)); + + case Double: + final WritableDoubleChunk doubleChunk = valueChunk.asWritableDoubleChunk(); + return index -> doubleChunk.set(index, formulaOutputSource.getDouble(0)); + + case Float: + final WritableFloatChunk floatChunk = valueChunk.asWritableFloatChunk(); + return index -> floatChunk.set(index, formulaOutputSource.getFloat(0)); + + case Int: + final WritableIntChunk intChunk = valueChunk.asWritableIntChunk(); + return index -> intChunk.set(index, formulaOutputSource.getInt(0)); + + case Long: + final WritableLongChunk longChunk = valueChunk.asWritableLongChunk(); + return index -> longChunk.set(index, formulaOutputSource.getLong(0)); + + case Short: + final WritableShortChunk shortChunk = valueChunk.asWritableShortChunk(); + return index -> shortChunk.set(index, formulaOutputSource.getShort(0)); + + default: + final WritableObjectChunk objectChunk = valueChunk.asWritableObjectChunk(); + return index -> { + Object result = formulaOutputSource.get(0); + if (result instanceof RingBufferVectorWrapper) { + // Handle the rare (and probably not useful) case where the formula is an identity. We need to + // copy the data in the RingBuffer and store that as a DirectVector. If not, we will point to + // the live data in the ring. + result = ((Vector) result).getDirect(); + } + objectChunk.set(index, result); + }; + } + } + + protected static IntConsumer getChunkNullSetter(final WritableChunk valueChunk) { + final ChunkType chunkType = valueChunk.getChunkType(); + switch (chunkType) { + case Boolean: + throw new IllegalStateException( + "Output chunk type should not be Boolean but should have been reinterpreted to byte"); + case Byte: + final WritableByteChunk byteChunk = valueChunk.asWritableByteChunk(); + return index -> byteChunk.set(index, NULL_BYTE); + + case Char: + final WritableCharChunk charChunk = valueChunk.asWritableCharChunk(); + return index -> charChunk.set(index, NULL_CHAR); + + case Double: + final WritableDoubleChunk doubleChunk = valueChunk.asWritableDoubleChunk(); + return index -> doubleChunk.set(index, NULL_DOUBLE); + + case Float: + final WritableFloatChunk floatChunk = valueChunk.asWritableFloatChunk(); + return index -> floatChunk.set(index, NULL_FLOAT); + + case Int: + final WritableIntChunk intChunk = valueChunk.asWritableIntChunk(); + return index -> intChunk.set(index, NULL_INT); + + case Long: + final WritableLongChunk longChunk = valueChunk.asWritableLongChunk(); + return index -> longChunk.set(index, NULL_LONG); + + case Short: + final WritableShortChunk shortChunk = valueChunk.asWritableShortChunk(); + return index -> shortChunk.set(index, NULL_SHORT); + + default: + final WritableObjectChunk objectChunk = valueChunk.asWritableObjectChunk(); + return index -> objectChunk.set(index, null); + } + } + + @Override + public void startTrackingPrev() { + outputSource.startTrackingPrevValues(); + if (rowRedirection != null) { + assert maybeInnerSource != null; + maybeInnerSource.startTrackingPrevValues(); + } + } + + @Override + public UpdateByOperator.@NotNull Context makeUpdateContext(int affectedChunkSize, int influencerChunkSize) { + return new Context(affectedChunkSize, influencerChunkSize); + } + + @Override + public void prepareForParallelPopulation(final RowSet changedRows) { + if (rowRedirection != null) { + assert maybeInnerSource != null; + ((WritableSourceWithPrepareForParallelPopulation) maybeInnerSource) + .prepareForParallelPopulation(changedRows); + } else { + ((WritableSourceWithPrepareForParallelPopulation) outputSource).prepareForParallelPopulation(changedRows); + } + } + + @NotNull + @Override + public Map> getOutputColumns() { + return Collections.singletonMap(pair.leftColumn, outputSource); + } + + // region clear-output + @Override + public void clearOutputRows(final RowSet toClear) { + // if we are redirected, clear the inner source + if (rowRedirection != null) { + ChunkUtils.fillWithNullValue(maybeInnerSource, toClear); + } else { + ChunkUtils.fillWithNullValue(outputSource, toClear); + } + } + + @Override + public void applyOutputShift(@NotNull final RowSet subRowSetToShift, final long delta) { + ((SparseArrayColumnSource) outputSource).shift(subRowSetToShift, delta); + } + + @Override + @NotNull + protected String[] getInputColumnNames() { + return inputColumnNames; + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ByteRingBufferWindowConsumer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ByteRingBufferWindowConsumer.java new file mode 100644 index 00000000000..5f1c6978077 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ByteRingBufferWindowConsumer.java @@ -0,0 +1,47 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit CharRingBufferWindowConsumer and run "./gradlew replicateUpdateBy" to regenerate +// +// @formatter:off +package io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.windowconsumer; + +import io.deephaven.base.ringbuffer.ByteRingBuffer; +import io.deephaven.chunk.ByteChunk; +import io.deephaven.chunk.Chunk; + +class ByteRingBufferWindowConsumer extends RingBufferWindowConsumer { + private final ByteRingBuffer byteRingBuffer; + + private ByteChunk influencerValuesChunk; + + ByteRingBufferWindowConsumer(ByteRingBuffer byteRingBuffer) { + this.byteRingBuffer = byteRingBuffer; + } + + @Override + public void setInputChunk(final Chunk inputChunk) { + this.influencerValuesChunk = inputChunk.asByteChunk(); + } + + @Override + public void push(int index, int count) { + byteRingBuffer.ensureRemaining(count); + for (int i = 0; i < count; i++) { + byteRingBuffer.add(influencerValuesChunk.get(index + i)); + } + } + + @Override + public void pop(int count) { + for (int i = 0; i < count; i++) { + byteRingBuffer.remove(); + } + } + + @Override + public void reset() { + byteRingBuffer.clear(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/CharRingBufferWindowConsumer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/CharRingBufferWindowConsumer.java new file mode 100644 index 00000000000..f15aca8a02f --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/CharRingBufferWindowConsumer.java @@ -0,0 +1,43 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.windowconsumer; + +import io.deephaven.base.ringbuffer.CharRingBuffer; +import io.deephaven.chunk.CharChunk; +import io.deephaven.chunk.Chunk; + +class CharRingBufferWindowConsumer extends RingBufferWindowConsumer { + private final CharRingBuffer charRingBuffer; + + private CharChunk influencerValuesChunk; + + CharRingBufferWindowConsumer(CharRingBuffer charRingBuffer) { + this.charRingBuffer = charRingBuffer; + } + + @Override + public void setInputChunk(final Chunk inputChunk) { + this.influencerValuesChunk = inputChunk.asCharChunk(); + } + + @Override + public void push(int index, int count) { + charRingBuffer.ensureRemaining(count); + for (int i = 0; i < count; i++) { + charRingBuffer.add(influencerValuesChunk.get(index + i)); + } + } + + @Override + public void pop(int count) { + for (int i = 0; i < count; i++) { + charRingBuffer.remove(); + } + } + + @Override + public void reset() { + charRingBuffer.clear(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/DoubleRingBufferWindowConsumer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/DoubleRingBufferWindowConsumer.java new file mode 100644 index 00000000000..08c3feac049 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/DoubleRingBufferWindowConsumer.java @@ -0,0 +1,47 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit CharRingBufferWindowConsumer and run "./gradlew replicateUpdateBy" to regenerate +// +// @formatter:off +package io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.windowconsumer; + +import io.deephaven.base.ringbuffer.DoubleRingBuffer; +import io.deephaven.chunk.DoubleChunk; +import io.deephaven.chunk.Chunk; + +class DoubleRingBufferWindowConsumer extends RingBufferWindowConsumer { + private final DoubleRingBuffer doubleRingBuffer; + + private DoubleChunk influencerValuesChunk; + + DoubleRingBufferWindowConsumer(DoubleRingBuffer doubleRingBuffer) { + this.doubleRingBuffer = doubleRingBuffer; + } + + @Override + public void setInputChunk(final Chunk inputChunk) { + this.influencerValuesChunk = inputChunk.asDoubleChunk(); + } + + @Override + public void push(int index, int count) { + doubleRingBuffer.ensureRemaining(count); + for (int i = 0; i < count; i++) { + doubleRingBuffer.add(influencerValuesChunk.get(index + i)); + } + } + + @Override + public void pop(int count) { + for (int i = 0; i < count; i++) { + doubleRingBuffer.remove(); + } + } + + @Override + public void reset() { + doubleRingBuffer.clear(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/FloatRingBufferWindowConsumer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/FloatRingBufferWindowConsumer.java new file mode 100644 index 00000000000..0b5797800db --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/FloatRingBufferWindowConsumer.java @@ -0,0 +1,47 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit CharRingBufferWindowConsumer and run "./gradlew replicateUpdateBy" to regenerate +// +// @formatter:off +package io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.windowconsumer; + +import io.deephaven.base.ringbuffer.FloatRingBuffer; +import io.deephaven.chunk.FloatChunk; +import io.deephaven.chunk.Chunk; + +class FloatRingBufferWindowConsumer extends RingBufferWindowConsumer { + private final FloatRingBuffer floatRingBuffer; + + private FloatChunk influencerValuesChunk; + + FloatRingBufferWindowConsumer(FloatRingBuffer floatRingBuffer) { + this.floatRingBuffer = floatRingBuffer; + } + + @Override + public void setInputChunk(final Chunk inputChunk) { + this.influencerValuesChunk = inputChunk.asFloatChunk(); + } + + @Override + public void push(int index, int count) { + floatRingBuffer.ensureRemaining(count); + for (int i = 0; i < count; i++) { + floatRingBuffer.add(influencerValuesChunk.get(index + i)); + } + } + + @Override + public void pop(int count) { + for (int i = 0; i < count; i++) { + floatRingBuffer.remove(); + } + } + + @Override + public void reset() { + floatRingBuffer.clear(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/IntRingBufferWindowConsumer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/IntRingBufferWindowConsumer.java new file mode 100644 index 00000000000..23d8ca6d2a3 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/IntRingBufferWindowConsumer.java @@ -0,0 +1,47 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit CharRingBufferWindowConsumer and run "./gradlew replicateUpdateBy" to regenerate +// +// @formatter:off +package io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.windowconsumer; + +import io.deephaven.base.ringbuffer.IntRingBuffer; +import io.deephaven.chunk.IntChunk; +import io.deephaven.chunk.Chunk; + +class IntRingBufferWindowConsumer extends RingBufferWindowConsumer { + private final IntRingBuffer intRingBuffer; + + private IntChunk influencerValuesChunk; + + IntRingBufferWindowConsumer(IntRingBuffer intRingBuffer) { + this.intRingBuffer = intRingBuffer; + } + + @Override + public void setInputChunk(final Chunk inputChunk) { + this.influencerValuesChunk = inputChunk.asIntChunk(); + } + + @Override + public void push(int index, int count) { + intRingBuffer.ensureRemaining(count); + for (int i = 0; i < count; i++) { + intRingBuffer.add(influencerValuesChunk.get(index + i)); + } + } + + @Override + public void pop(int count) { + for (int i = 0; i < count; i++) { + intRingBuffer.remove(); + } + } + + @Override + public void reset() { + intRingBuffer.clear(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/LongRingBufferWindowConsumer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/LongRingBufferWindowConsumer.java new file mode 100644 index 00000000000..e77d06e8425 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/LongRingBufferWindowConsumer.java @@ -0,0 +1,47 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit CharRingBufferWindowConsumer and run "./gradlew replicateUpdateBy" to regenerate +// +// @formatter:off +package io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.windowconsumer; + +import io.deephaven.base.ringbuffer.LongRingBuffer; +import io.deephaven.chunk.LongChunk; +import io.deephaven.chunk.Chunk; + +class LongRingBufferWindowConsumer extends RingBufferWindowConsumer { + private final LongRingBuffer longRingBuffer; + + private LongChunk influencerValuesChunk; + + LongRingBufferWindowConsumer(LongRingBuffer longRingBuffer) { + this.longRingBuffer = longRingBuffer; + } + + @Override + public void setInputChunk(final Chunk inputChunk) { + this.influencerValuesChunk = inputChunk.asLongChunk(); + } + + @Override + public void push(int index, int count) { + longRingBuffer.ensureRemaining(count); + for (int i = 0; i < count; i++) { + longRingBuffer.add(influencerValuesChunk.get(index + i)); + } + } + + @Override + public void pop(int count) { + for (int i = 0; i < count; i++) { + longRingBuffer.remove(); + } + } + + @Override + public void reset() { + longRingBuffer.clear(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ObjectRingBufferWindowConsumer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ObjectRingBufferWindowConsumer.java new file mode 100644 index 00000000000..206d2cf15d3 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ObjectRingBufferWindowConsumer.java @@ -0,0 +1,43 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.windowconsumer; + +import io.deephaven.base.ringbuffer.ObjectRingBuffer; +import io.deephaven.chunk.ObjectChunk; +import io.deephaven.chunk.Chunk; + +class ObjectRingBufferWindowConsumer extends RingBufferWindowConsumer { + private final ObjectRingBuffer objectRingBuffer; + + private ObjectChunk influencerValuesChunk; + + ObjectRingBufferWindowConsumer(ObjectRingBuffer objectRingBuffer) { + this.objectRingBuffer = objectRingBuffer; + } + + @Override + public void setInputChunk(final Chunk inputChunk) { + this.influencerValuesChunk = inputChunk.asObjectChunk(); + } + + @Override + public void push(int index, int count) { + objectRingBuffer.ensureRemaining(count); + for (int i = 0; i < count; i++) { + objectRingBuffer.add(influencerValuesChunk.get(index + i)); + } + } + + @Override + public void pop(int count) { + for (int i = 0; i < count; i++) { + objectRingBuffer.remove(); + } + } + + @Override + public void reset() { + objectRingBuffer.clear(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/RingBufferWindowConsumer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/RingBufferWindowConsumer.java new file mode 100644 index 00000000000..c98088ee61d --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/RingBufferWindowConsumer.java @@ -0,0 +1,71 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.windowconsumer; + +import io.deephaven.base.ringbuffer.*; +import io.deephaven.chunk.Chunk; + +/** + * This helper provides an abstract interface for consuming UpdateBy window input data values and storing in a + * {@link RingBuffer}. The UpdateBy code produces a chunk of input data and lists of push/pop instructions for managing + * the current window values. This class abstracts the underlying buffer type and provides a common interface for the + * UpdateBy code to interact with the buffer. + */ +public abstract class RingBufferWindowConsumer { + /** + * Create a new {@link RingBufferWindowConsumer} for the given buffer. + * + * @param buffer the buffer to manage + * @return a new RingBufferWindowConsumer + */ + public static RingBufferWindowConsumer create(final RingBuffer buffer) { + final Class bufferClass = buffer.getClass(); + + if (bufferClass == CharRingBuffer.class) { + return new CharRingBufferWindowConsumer((CharRingBuffer) buffer); + } else if (bufferClass == ByteRingBuffer.class) { + return new ByteRingBufferWindowConsumer((ByteRingBuffer) buffer); + } else if (bufferClass == DoubleRingBuffer.class) { + return new DoubleRingBufferWindowConsumer((DoubleRingBuffer) buffer); + } else if (bufferClass == FloatRingBuffer.class) { + return new FloatRingBufferWindowConsumer((FloatRingBuffer) buffer); + } else if (bufferClass == IntRingBuffer.class) { + return new IntRingBufferWindowConsumer((IntRingBuffer) buffer); + } else if (bufferClass == LongRingBuffer.class) { + return new LongRingBufferWindowConsumer((LongRingBuffer) buffer); + } else if (bufferClass == ShortRingBuffer.class) { + return new ShortRingBufferWindowConsumer((ShortRingBuffer) buffer); + } + return new ObjectRingBufferWindowConsumer<>((ObjectRingBuffer) buffer); + } + + /** + * Set the input chunk for this consumer. This will be stored and the push instructions will index into this buffer + * for newly provided values. + * + * @param inputChunk the input chunk + */ + public abstract void setInputChunk(Chunk inputChunk); + + /** + * Push {@code count} values from the input chunk into the ring buffer, beginning at {@code index}. + * + * @param index the index of the first value to push + * @param count the count of values to push + */ + public abstract void push(int index, int count); + + /** + * Pop {@code count} values from the ring buffer. + * + * @param count the count of values to pop + */ + public abstract void pop(int count); + + /** + * Reset the ring buffer to its initial state. If this is an object ring buffer, this will set all values to + * {@code null}. + */ + public abstract void reset(); +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ShortRingBufferWindowConsumer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ShortRingBufferWindowConsumer.java new file mode 100644 index 00000000000..187b34fa603 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/ShortRingBufferWindowConsumer.java @@ -0,0 +1,47 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY +// ****** Edit CharRingBufferWindowConsumer and run "./gradlew replicateUpdateBy" to regenerate +// +// @formatter:off +package io.deephaven.engine.table.impl.updateby.rollingformulamulticolumn.windowconsumer; + +import io.deephaven.base.ringbuffer.ShortRingBuffer; +import io.deephaven.chunk.ShortChunk; +import io.deephaven.chunk.Chunk; + +class ShortRingBufferWindowConsumer extends RingBufferWindowConsumer { + private final ShortRingBuffer shortRingBuffer; + + private ShortChunk influencerValuesChunk; + + ShortRingBufferWindowConsumer(ShortRingBuffer shortRingBuffer) { + this.shortRingBuffer = shortRingBuffer; + } + + @Override + public void setInputChunk(final Chunk inputChunk) { + this.influencerValuesChunk = inputChunk.asShortChunk(); + } + + @Override + public void push(int index, int count) { + shortRingBuffer.ensureRemaining(count); + for (int i = 0; i < count; i++) { + shortRingBuffer.add(influencerValuesChunk.get(index + i)); + } + } + + @Override + public void pop(int count) { + for (int i = 0; i < count; i++) { + shortRingBuffer.remove(); + } + } + + @Override + public void reset() { + shortRingBuffer.clear(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/ByteRollingWAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/ByteRollingWAvgOperator.java index 70b3537a674..25f26774957 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/ByteRollingWAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/ByteRollingWAvgOperator.java @@ -93,15 +93,4 @@ public UpdateByOperator copy() { // endregion extra-copy-args ); } - - /** - * Get the names of the input column(s) for this operator. - * - * @return the names of the input column - */ - @NotNull - @Override - protected String[] getInputColumnNames() { - return new String[] {pair.rightColumn, weightColumnName}; - } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/CharRollingWAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/CharRollingWAvgOperator.java index 37e41347661..d892f43771a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/CharRollingWAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/CharRollingWAvgOperator.java @@ -89,15 +89,4 @@ public UpdateByOperator copy() { // endregion extra-copy-args ); } - - /** - * Get the names of the input column(s) for this operator. - * - * @return the names of the input column - */ - @NotNull - @Override - protected String[] getInputColumnNames() { - return new String[] {pair.rightColumn, weightColumnName}; - } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/DoubleRollingWAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/DoubleRollingWAvgOperator.java index c1e3a717d9a..1547753a81a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/DoubleRollingWAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/DoubleRollingWAvgOperator.java @@ -93,15 +93,4 @@ public UpdateByOperator copy() { // endregion extra-copy-args ); } - - /** - * Get the names of the input column(s) for this operator. - * - * @return the names of the input column - */ - @NotNull - @Override - protected String[] getInputColumnNames() { - return new String[] {pair.rightColumn, weightColumnName}; - } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/FloatRollingWAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/FloatRollingWAvgOperator.java index 0d4e0392595..0c79c04de30 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/FloatRollingWAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/FloatRollingWAvgOperator.java @@ -93,15 +93,4 @@ public UpdateByOperator copy() { // endregion extra-copy-args ); } - - /** - * Get the names of the input column(s) for this operator. - * - * @return the names of the input column - */ - @NotNull - @Override - protected String[] getInputColumnNames() { - return new String[] {pair.rightColumn, weightColumnName}; - } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/IntRollingWAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/IntRollingWAvgOperator.java index 5aeda3a07f1..58b4515becf 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/IntRollingWAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/IntRollingWAvgOperator.java @@ -93,15 +93,4 @@ public UpdateByOperator copy() { // endregion extra-copy-args ); } - - /** - * Get the names of the input column(s) for this operator. - * - * @return the names of the input column - */ - @NotNull - @Override - protected String[] getInputColumnNames() { - return new String[] {pair.rightColumn, weightColumnName}; - } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/LongRollingWAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/LongRollingWAvgOperator.java index 60bbc0e8d1d..59939e67891 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/LongRollingWAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/LongRollingWAvgOperator.java @@ -93,15 +93,4 @@ public UpdateByOperator copy() { // endregion extra-copy-args ); } - - /** - * Get the names of the input column(s) for this operator. - * - * @return the names of the input column - */ - @NotNull - @Override - protected String[] getInputColumnNames() { - return new String[] {pair.rightColumn, weightColumnName}; - } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/ShortRollingWAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/ShortRollingWAvgOperator.java index 4f9f11f5c83..459598dc86c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/ShortRollingWAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingwavg/ShortRollingWAvgOperator.java @@ -93,15 +93,4 @@ public UpdateByOperator copy() { // endregion extra-copy-args ); } - - /** - * Get the names of the input column(s) for this operator. - * - * @return the names of the input column - */ - @NotNull - @Override - protected String[] getInputColumnNames() { - return new String[] {pair.rightColumn, weightColumnName}; - } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingFormula.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingFormula.java index 7238a47c1d0..314b96faf37 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingFormula.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingFormula.java @@ -5,10 +5,12 @@ import io.deephaven.api.updateby.UpdateByControl; import io.deephaven.api.updateby.UpdateByOperation; +import io.deephaven.base.verify.Assert; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.table.PartitionedTable; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.impl.QueryTable; +import io.deephaven.engine.table.impl.select.FormulaCompilationException; import io.deephaven.engine.testutil.ControlledUpdateGraph; import io.deephaven.engine.testutil.EvalNugget; import io.deephaven.engine.testutil.GenerateTableUpdates; @@ -204,6 +206,15 @@ private void doTestStaticZeroKey(final int prevTicks, final int postTicks) { new String[] {"charCol"}, new TestDataGenerator[] {new CharGenerator('A', 'z', 0.1)}).t; + // Verify that the RollingFormula spec errors out when provided a paramToken and multiple input columns. + try { + t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "out=sum(x)", "x", primitiveColumns)); + Assert.statementNeverExecuted(); + } catch (Exception exception) { + Assert.assertion(exception instanceof FormulaCompilationException, + "exception instanceof FormulaCompilationException"); + } + Table actual; Table expected; String[] updateStrings; @@ -261,7 +272,8 @@ private void doTestStaticZeroKey(final int prevTicks, final int postTicks) { // Count vs. RollingCount //////////////////////////////////////////////////////////////////////////////////////////////////// - actual = t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "count(x)", "x", primitiveColumns)); + actual = t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "count(x)", "x", + primitiveColumns)); expected = t.updateBy(UpdateByOperation.RollingCount(prevTicks, postTicks, primitiveColumns)); TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); @@ -311,6 +323,49 @@ private void doTestStaticZeroKey(final int prevTicks, final int postTicks) { expected = t.updateBy(UpdateByOperation.RollingCount(prevTicks, postTicks, "boolCol")); TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // Multi-column formula tests + //////////////////////////////////////////////////////////////////////////////////////////////////// + + // zero input columns + actual = t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "out_val=-1L")); + expected = t.update("out_val=-1L"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // single input column + actual = t + .updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "out_val=sum(intCol)")); + expected = t.updateBy(UpdateByOperation.RollingGroup(prevTicks, postTicks, "a=intCol")) + .update("out_val=sum(a)").dropColumns("a"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // two input columns + actual = t + .updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "out_val=min(intCol) - max(longCol)")); + expected = t.updateBy(UpdateByOperation.RollingGroup(prevTicks, postTicks, "a=intCol", "b=longCol")) + .update("out_val=min(a) - max(b)").dropColumns("a", "b"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // three input columns + actual = t + .updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, + "out_val=sum(intCol) - max(longCol) + min(doubleCol)")); + expected = + t.updateBy(UpdateByOperation.RollingGroup(prevTicks, postTicks, "a=intCol", "b=longCol", "c=doubleCol")) + .update("out_val=sum(a) - max(b) + min(c)").dropColumns("a", "b", "c"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + actual = t + .updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "out_val=sum(intCol) - max(longCol)")); + expected = t.updateBy(UpdateByOperation.RollingGroup(prevTicks, postTicks, "a=intCol", "b=longCol")) + .update("out_val=sum(a) - max(b)").dropColumns("a", "b"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); } private void doTestStaticZeroKeyTimed(final Duration prevTime, final Duration postTime) { @@ -320,6 +375,15 @@ private void doTestStaticZeroKeyTimed(final Duration prevTime, final Duration po DateTimeUtils.parseInstant("2022-03-09T16:30:00.000 NY")), new CharGenerator('A', 'z', 0.1)}).t; + // Verify that the RollingFormula spec errors out when provided a paramToken and multiple input columns. + try { + t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "out=sum(x)", "x", primitiveColumns)); + Assert.statementNeverExecuted(); + } catch (Exception exception) { + Assert.assertion(exception instanceof FormulaCompilationException, + "exception instanceof FormulaCompilationException"); + } + Table actual; Table expected; String[] updateStrings; @@ -431,6 +495,52 @@ private void doTestStaticZeroKeyTimed(final Duration prevTime, final Duration po expected = t.updateBy(UpdateByOperation.RollingCount("ts", prevTime, postTime, "boolCol")); TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // Multi-column formula tests + //////////////////////////////////////////////////////////////////////////////////////////////////// + + // zero input columns + actual = t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "out_val=-1L")); + expected = t.update("out_val=-1L"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // single input column + actual = t + .updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "out_val=sum(intCol)")); + expected = t.updateBy(UpdateByOperation.RollingGroup("ts", prevTime, postTime, "a=intCol")) + .update("out_val=sum(a)").dropColumns("a"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // two input columns + actual = t + .updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "out_val=min(intCol) - max(longCol)")); + expected = t.updateBy(UpdateByOperation.RollingGroup("ts", prevTime, postTime, "a=intCol", "b=longCol")) + .update("out_val=min(a) - max(b)").dropColumns("a", "b"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // three input columns + actual = t + .updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "out_val=sum(intCol) - max(longCol) + min(doubleCol)")); + expected = t + .updateBy(UpdateByOperation.RollingGroup("ts", prevTime, postTime, "a=intCol", "b=longCol", + "c=doubleCol")) + .update("out_val=sum(a) - max(b) + min(c)").dropColumns("a", "b", "c"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + actual = t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "out_val=intCol + longCol")); + expected = t.updateBy(UpdateByOperation.RollingGroup("ts", prevTime, postTime, + "a=intCol", "b=longCol")) + .update("out_val=a + b").dropColumns("a", "b"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); } // endregion @@ -640,6 +750,53 @@ private void doTestStaticBucketed(boolean grouped, int prevTicks, int postTicks) expected = t.updateBy(UpdateByOperation.RollingCount(prevTicks, postTicks, "boolCol"), "Sym"); TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // Multi-column formula tests + //////////////////////////////////////////////////////////////////////////////////////////////////// + + // zero input columns + actual = t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "out_val=-1L"), "Sym"); + expected = t.update("out_val=-1L"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // single input column + actual = t + .updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "out_val=sum(intCol)"), "Sym"); + expected = t.updateBy(UpdateByOperation.RollingGroup(prevTicks, postTicks, "a=intCol"), "Sym") + .update("out_val=sum(a)").dropColumns("a"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // two input columns + actual = t + .updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "out_val=min(intCol) - max(longCol)"), + "Sym"); + expected = t.updateBy(UpdateByOperation.RollingGroup(prevTicks, postTicks, "a=intCol", "b=longCol"), "Sym") + .update("out_val=min(a) - max(b)").dropColumns("a", "b"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // three input columns + actual = t + .updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, + "out_val=sum(intCol) - max(longCol) + min(doubleCol)"), "Sym"); + expected = + t.updateBy(UpdateByOperation.RollingGroup(prevTicks, postTicks, "a=intCol", "b=longCol", "c=doubleCol"), + "Sym") + .update("out_val=sum(a) - max(b) + min(c)").dropColumns("a", "b", "c"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + actual = t + .updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "out_val=sum(intCol) - max(longCol)"), + "Sym"); + expected = t.updateBy(UpdateByOperation.RollingGroup(prevTicks, postTicks, "a=intCol", "b=longCol"), "Sym") + .update("out_val=sum(a) - max(b)").dropColumns("a", "b"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + } private void doTestStaticBucketedTimed(boolean grouped, Duration prevTime, Duration postTime) { @@ -762,6 +919,52 @@ private void doTestStaticBucketedTimed(boolean grouped, Duration prevTime, Durat expected = t.updateBy(UpdateByOperation.RollingCount("ts", prevTime, postTime, "boolCol"), "Sym"); TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // Multi-column formula tests + //////////////////////////////////////////////////////////////////////////////////////////////////// + + // zero input columns + actual = t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "out_val=-1L"), "Sym"); + expected = t.update("out_val=-1L"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // single input column + actual = t + .updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "out_val=sum(intCol)"), "Sym"); + expected = t.updateBy(UpdateByOperation.RollingGroup("ts", prevTime, postTime, "a=intCol"), "Sym") + .update("out_val=sum(a)").dropColumns("a"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // two input columns + actual = t + .updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "out_val=min(intCol) - max(longCol)"), "Sym"); + expected = t.updateBy(UpdateByOperation.RollingGroup("ts", prevTime, postTime, "a=intCol", "b=longCol"), "Sym") + .update("out_val=min(a) - max(b)").dropColumns("a", "b"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + // three input columns + actual = t + .updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "out_val=sum(intCol) - max(longCol) + min(doubleCol)"), "Sym"); + expected = t + .updateBy(UpdateByOperation.RollingGroup("ts", prevTime, postTime, "a=intCol", "b=longCol", + "c=doubleCol"), "Sym") + .update("out_val=sum(a) - max(b) + min(c)").dropColumns("a", "b", "c"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); + + actual = t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "out_val=intCol + longCol"), "Sym"); + expected = t.updateBy(UpdateByOperation.RollingGroup("ts", prevTime, postTime, + "a=intCol", "b=longCol"), "Sym") + .update("out_val=a + b").dropColumns("a", "b"); + + TstUtils.assertTableEquals(expected, actual, TableDiff.DiffItems.DoublesExact); } // endregion @@ -930,6 +1133,7 @@ private void doTestAppendOnly(boolean bucketed, int prevTicks, int postTicks) { ExecutionContext.getContext().getQueryLibrary().importStatic(Helpers.class); final EvalNugget[] nuggets = new EvalNugget[] { + // Single column formula tests EvalNugget.from(() -> bucketed ? t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "sum(x + 1)", "x", primitiveColumns), "Sym") @@ -945,9 +1149,32 @@ private void doTestAppendOnly(boolean bucketed, int prevTicks, int postTicks) { "x", "bigDecimalCol"), "Sym") : t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "sumBigDecimal(x)", "x", "bigDecimalCol"))), + // Multi-column formula tests + EvalNugget.from(() -> bucketed + ? t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "const_val=5"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "const_val=5"))), + EvalNugget.from(() -> bucketed + ? t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "sum=sum(longCol) / 2"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "sum=sum(longCol) / 2"))), + EvalNugget.from(() -> bucketed + ? t.updateBy( + UpdateByOperation.RollingFormula(prevTicks, postTicks, + "sum=sum(intCol) + sum(longCol)"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, + "sum=sum(intCol) + sum(longCol)"))), + EvalNugget.from(() -> bucketed + ? t.updateBy( + UpdateByOperation.RollingFormula(prevTicks, postTicks, + "sum=min(intCol) + min(longCol) * max(doubleCol)"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, + "sum=min(intCol) + min(longCol) * max(doubleCol)"))), }; - final Random billy = new Random(0xB177B177); + final Random billy = new Random(0xB177B177L); for (int ii = 0; ii < DYNAMIC_UPDATE_STEPS; ii++) { ExecutionContext.getContext().getUpdateGraph().cast().runWithinUnitTestCycle( () -> generateAppends(DYNAMIC_UPDATE_SIZE, billy, t, result.infos)); @@ -967,6 +1194,7 @@ private void doTestAppendOnlyTimed(boolean bucketed, Duration prevTime, Duration ExecutionContext.getContext().getQueryLibrary().importStatic(Helpers.class); final EvalNugget[] nuggets = new EvalNugget[] { + // Single column formula tests EvalNugget.from(() -> bucketed ? t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "sum(x + 1)", "x", primitiveColumns), "Sym") @@ -982,9 +1210,33 @@ private void doTestAppendOnlyTimed(boolean bucketed, Duration prevTime, Duration "sumBigDecimal(x)", "x", "bigDecimalCol"), "Sym") : t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "sumBigDecimal(x)", "x", "bigDecimalCol"))), + // Multi-column formula tests + EvalNugget.from(() -> bucketed + ? t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "const_val=5"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "const_val=5"))), + EvalNugget.from(() -> bucketed + ? t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "sum=sum(longCol) / 2"), + "Sym") + : t.updateBy( + UpdateByOperation.RollingFormula("ts", prevTime, postTime, "sum=sum(longCol) / 2"))), + EvalNugget.from(() -> bucketed + ? t.updateBy( + UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "sum=sum(intCol) + sum(longCol)"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "sum=sum(intCol) + sum(longCol)"))), + EvalNugget.from(() -> bucketed + ? t.updateBy( + UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "sum=min(intCol) + min(longCol) * max(doubleCol)"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "sum=min(intCol) + min(longCol) * max(doubleCol)"))), }; - final Random billy = new Random(0xB177B177); + final Random billy = new Random(0xB177B177L); for (int ii = 0; ii < DYNAMIC_UPDATE_STEPS; ii++) { ExecutionContext.getContext().getUpdateGraph().cast().runWithinUnitTestCycle( () -> generateAppends(DYNAMIC_UPDATE_SIZE, billy, t, result.infos)); @@ -1132,6 +1384,29 @@ private void doTestTicking(final boolean bucketed, final long prevTicks, final l "x", "bigDecimalCol"), "Sym") : t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "sumBigDecimal(x)", "x", "bigDecimalCol"))), + // Multi-column formula tests + EvalNugget.from(() -> bucketed + ? t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "const_val=5"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "const_val=5"))), + EvalNugget.from(() -> bucketed + ? t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "sum=sum(longCol) / 2"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "sum=sum(longCol) / 2"))), + EvalNugget.from(() -> bucketed + ? t.updateBy( + UpdateByOperation.RollingFormula(prevTicks, postTicks, + "sum=sum(intCol) + sum(longCol)"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, + "sum=sum(intCol) + sum(longCol)"))), + EvalNugget.from(() -> bucketed + ? t.updateBy( + UpdateByOperation.RollingFormula(prevTicks, postTicks, + "sum=min(intCol) + min(longCol) * max(doubleCol)"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, + "sum=min(intCol) + min(longCol) * max(doubleCol)"))), }; final Random billy = new Random(0xB177B177); @@ -1169,6 +1444,30 @@ private void doTestTickingTimed(final boolean bucketed, final Duration prevTime, "sumBigDecimal(x)", "x", "bigDecimalCol"), "Sym") : t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "sumBigDecimal(x)", "x", "bigDecimalCol"))), + // Multi-column formula tests + EvalNugget.from(() -> bucketed + ? t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "const_val=5"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "const_val=5"))), + EvalNugget.from(() -> bucketed + ? t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, "sum=sum(longCol) / 2"), + "Sym") + : t.updateBy( + UpdateByOperation.RollingFormula("ts", prevTime, postTime, "sum=sum(longCol) / 2"))), + EvalNugget.from(() -> bucketed + ? t.updateBy( + UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "sum=sum(intCol) + sum(longCol)"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "sum=sum(intCol) + sum(longCol)"))), + EvalNugget.from(() -> bucketed + ? t.updateBy( + UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "sum=min(intCol) + min(longCol) * max(doubleCol)"), + "Sym") + : t.updateBy(UpdateByOperation.RollingFormula("ts", prevTime, postTime, + "sum=min(intCol) + min(longCol) * max(doubleCol)"))), }; final Random billy = new Random(0xB177B177); @@ -1294,11 +1593,14 @@ public void testProxy() { final int postTicks = 0; Table actual; - Table expected; PartitionedTable pt = t.partitionBy("Sym"); actual = pt.proxy().updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "count(x)", "x", "intCol")) .target().merge(); + + actual = pt.proxy() + .updateBy(UpdateByOperation.RollingFormula(prevTicks, postTicks, "intCol_count=count(intCol)")) + .target().merge(); } // endregion diff --git a/java-client/session/src/main/java/io/deephaven/client/impl/UpdateByBuilder.java b/java-client/session/src/main/java/io/deephaven/client/impl/UpdateByBuilder.java index f41161b048b..b0ddd59a18a 100644 --- a/java-client/session/src/main/java/io/deephaven/client/impl/UpdateByBuilder.java +++ b/java-client/session/src/main/java/io/deephaven/client/impl/UpdateByBuilder.java @@ -312,7 +312,7 @@ public UpdateByColumn.UpdateBySpec visit(RollingFormulaSpec rs) { .setReverseWindowScale(adapt(rs.revWindowScale())) .setForwardWindowScale(adapt(rs.fwdWindowScale())) .setFormula(rs.formula()) - .setParamToken(rs.paramToken()); + .setParamToken(rs.paramToken().orElse(null)); return UpdateByColumn.UpdateBySpec.newBuilder() .setRollingFormula(builder.build()) .build(); diff --git a/py/server/deephaven/updateby.py b/py/server/deephaven/updateby.py index ef2f04b32d4..8c68f5ffbe7 100644 --- a/py/server/deephaven/updateby.py +++ b/py/server/deephaven/updateby.py @@ -1361,13 +1361,27 @@ def rolling_wavg_time(ts_col: str, wcol: str, cols: Union[str, List[str]], rev_t raise DHError(e, "failed to create a rolling weighted average (time) UpdateByOperation.") from e -def rolling_formula_tick(formula: str, formula_param: str, cols: Union[str, List[str]], rev_ticks: int, fwd_ticks: int = 0) -> UpdateByOperation: +def rolling_formula_tick(formula: str, formula_param: str = None, cols: Union[str, List[str]] = None, + rev_ticks: int = 0, fwd_ticks: int = 0) -> UpdateByOperation: """Creates a rolling formula UpdateByOperation for the supplied column names, using ticks as the windowing unit. Ticks are row counts, and you may specify the reverse and forward window in number of rows to include. The current row is considered to belong to the reverse window but not the forward window. Also, negative values are allowed and can be used to generate completely forward or completely reverse windows. - User-defined formula can contain a combination of any of the following: + There are two variants of this call. The preferred variant requires the formula to provide the output column name + and specific input column names in the following format: + | rolling_formula_tick(formula='output_col=(input_col1 + input_col2) * input_col3', rev_ticks=10, fwd_ticks=0) + This form does not accept `formula_param` or `cols` arguments because the input and output columns are explicitly + set within the formula string. + + The second (deprecated) variant allows the user to apply a formula expression to one input column, producing one + input column. In this call the `formula_param` is used as a placeholder for the input column name and the `cols` + argument is used to identify the output column name and the input source column when applying the formula. If + multiple input/output pairs are specified in the `cols` argument, the formula will be applied to each column in the + list. The format for this call is the following: + | rolling_formula_tick(formula='min(x * x + 5)', formula_param='x', cols=['out1=inputCol1','out2=inputCol2'], rev_ticks=10, fwd_ticks=0) + + User-defined formula can contain a combination of the following: | Built-in functions such as `min`, `max`, etc. | Mathematical arithmetic such as `*`, `+`, `/`, etc. | User-defined functions @@ -1386,10 +1400,12 @@ def rolling_formula_tick(formula: str, formula_param: str, cols: Union[str, List Args: formula (str): the user defined formula to apply to each group. - formula_param (str): the parameter name for the input column's vector within the formula. If formula is - `max(each)`, then `each` is the formula_param. - cols (Union[str, List[str]]): the column(s) to be operated on, can include expressions to rename the output, - i.e. "new_col = col"; when empty, update_by performs the rolling formula operation on all columns. + formula_param (str): If provided, supplies the parameter name for the input column's vector within the formula. + If formula is `max(each)`, then `each` is the formula_param. Default is None, implying the `formula` + argument specifies the input and output columns. + cols (Union[str, List[str]]): If provided, supplies the column(s) to operate on, can include expressions to + rename the output, i.e. "new_col = col". If omitted and the `formula_param` is provided, update_by performs + the rolling formula operation on all columns rev_ticks (int): the look-behind window size (in rows/ticks) fwd_ticks (int): the look-forward window size (int rows/ticks), default is 0 @@ -1400,20 +1416,36 @@ def rolling_formula_tick(formula: str, formula_param: str, cols: Union[str, List DHError """ try: + if formula_param is None: + # Use the multi-column formula call + return UpdateByOperation(j_updateby_op=_JUpdateByOperation.RollingFormula(rev_ticks, fwd_ticks, formula)) cols = to_sequence(cols) return UpdateByOperation(j_updateby_op=_JUpdateByOperation.RollingFormula(rev_ticks, fwd_ticks, formula, formula_param, *cols)) except Exception as e: raise DHError(e, "failed to create a rolling formula (tick) UpdateByOperation.") from e -def rolling_formula_time(ts_col: str, formula: str, formula_param: str, cols: Union[str, List[str]], rev_time: Union[int, str], - fwd_time: Union[int, str] = 0) -> UpdateByOperation: +def rolling_formula_time(ts_col: str, formula: str, formula_param: str = None, cols: Union[str, List[str]] = None, + rev_time: Union[int, str] = 0, fwd_time: Union[int, str] = 0) -> UpdateByOperation: """Creates a rolling formula UpdateByOperation for the supplied column names, using time as the windowing unit. This function accepts nanoseconds or time strings as the reverse and forward window parameters. Negative values are allowed and can be used to generate completely forward or completely reverse windows. A row containing a null in the timestamp column belongs to no window and will not be considered in the windows of other rows; its output will be null. + There are two variants of this call. The preferred variant requires the formula to provide the output column name + and specific input column names in the following format: + | rolling_formula_time(ts_col='tstamp', formula='output_col=(input_col1 + input_col2) * input_col3', rev_time='PT00:10:00', fwd_time='0'`) + This form does not accept `formula_param` or `cols` arguments because the input and output columns are explicitly + set within the formula string. + + The second (deprecated) variant allows the user to apply a formula expression to one input column, producing one + input column. In this call the `formula_param` is used as a placeholder for the input column name and the `cols` + argument is used to identify the output column name and the input source column when applying the formula. If + multiple input/output pairs are specified in the `cols` argument, the formula will be applied to each column in the + list. The format for this call is the following: + | rolling_formula_time(ts_col='tstamp', formula='min(x * x + 5)', formula_param='x', rev_time='PT00:10:00', fwd_time='0'`) + User-defined formula can contain a combination of any of the following: | Built-in functions such as `min`, `max`, etc. | Mathematical arithmetic such as `*`, `+`, `/`, etc. @@ -1435,10 +1467,12 @@ def rolling_formula_time(ts_col: str, formula: str, formula_param: str, cols: Un Args: ts_col (str): the timestamp column for determining the window formula (str): the user defined formula to apply to each group. - formula_param (str): the parameter name for the input column's vector within the formula. If formula is - `max(each)`, then `each` is the formula_param. - cols (Union[str, List[str]]): the column(s) to be operated on, can include expressions to rename the output, - i.e. "new_col = col"; when empty, update_by performs the rolling formula operation on all columns. + formula_param (str): If provided, supplies the parameter name for the input column's vector within the formula. + If formula is `max(each)`, then `each` is the formula_param. Default is None, implying the `formula` + argument specifies the input and output columns. + cols (Union[str, List[str]]): If provided, supplies the column(s) to operate on, can include expressions to + rename the output, i.e. "new_col = col". If omitted and the `formula_param` is provided, update_by performs + the rolling formula operation on all columns rev_time (int): the look-behind window size, can be expressed as an integer in nanoseconds or a time interval string, e.g. "PT00:00:00.001" or "PT5M" fwd_time (int): the look-ahead window size, can be expressed as an integer in nanoseconds or a time @@ -1451,9 +1485,13 @@ def rolling_formula_time(ts_col: str, formula: str, formula_param: str, cols: Un DHError """ try: - cols = to_sequence(cols) rev_time = _JDateTimeUtils.parseDurationNanos(rev_time) if isinstance(rev_time, str) else rev_time fwd_time = _JDateTimeUtils.parseDurationNanos(fwd_time) if isinstance(fwd_time, str) else fwd_time + if formula_param is None: + # Use the multi-column formula call + return UpdateByOperation(j_updateby_op=_JUpdateByOperation.RollingFormula(ts_col, rev_time, fwd_time, formula)) + + cols = to_sequence(cols) return UpdateByOperation(j_updateby_op=_JUpdateByOperation.RollingFormula(ts_col, rev_time, fwd_time, formula, formula_param, *cols)) except Exception as e: raise DHError(e, "failed to create a rolling formula (time) UpdateByOperation.") from e \ No newline at end of file diff --git a/py/server/tests/test_updateby.py b/py/server/tests/test_updateby.py index e58ce542539..150d41d51e5 100644 --- a/py/server/tests/test_updateby.py +++ b/py/server/tests/test_updateby.py @@ -157,12 +157,20 @@ def setUpClass(cls) -> None: rolling_formula_tick(formula="sum(x)", formula_param="x", cols=["formula_a = a", "formula_d = d"], rev_ticks=10), rolling_formula_tick(formula="avg(x)", formula_param="x", cols=["formula_a = a", "formula_d = d"], rev_ticks=10, fwd_ticks=10), rolling_formula_time(formula="sum(x)", formula_param="x", ts_col="Timestamp", cols=["formula_b = b", "formula_e = e"], rev_time="PT00:00:10"), - rolling_formula_time(formula="avg(x)", formula_param="x", ts_col="Timestamp", cols=["formula_b = b", "formula_e = e"], rev_time=10_000_000_000, - fwd_time=-10_000_000_00), - rolling_formula_time(formula="sum(x)", formula_param="x", ts_col="Timestamp", cols=["formula_b = b", "formula_e = e"], rev_time="PT30S", - fwd_time="-PT00:00:20"), + rolling_formula_time(formula="avg(x)", formula_param="x", ts_col="Timestamp", cols=["formula_b = b", "formula_e = e"], rev_time=10_000_000_000, fwd_time=-10_000_000_00), + rolling_formula_time(formula="sum(x)", formula_param="x", ts_col="Timestamp", cols=["formula_b = b", "formula_e = e"], rev_time="PT30S", fwd_time="-PT00:00:20"), ] + # Rolling Operators list shared with test_rolling_ops / test_rolling_ops_proxy that produce a single output column + cls.rolling_ops_one_output = [ + rolling_formula_tick(formula="formula_ad=sum(a) + sum(d)", rev_ticks=10), + rolling_formula_tick(formula="formula_ad=avg(a) + avg(b)", rev_ticks=10, fwd_ticks=10), + rolling_formula_time(formula="formula_be=sum(b) + sum(e)", ts_col="Timestamp", rev_time="PT00:00:10"), + rolling_formula_time(formula="formula_be=avg(b) + avg(e)", ts_col="Timestamp", rev_time=10_000_000_000, fwd_time=-10_000_000_00), + rolling_formula_time(formula="formula_be=sum(b) + sum(b)", ts_col="Timestamp", rev_time="PT30S", fwd_time="-PT00:00:20"), + ] + + @classmethod def tearDownClass(cls) -> None: del cls.em_op_ctrl @@ -225,6 +233,7 @@ def test_simple_ops_proxy(self): self.assertEqual(ct.size, rct.size) def test_rolling_ops(self): + # Test rolling operators that produce 2 output columns for op in self.rolling_ops: with self.subTest(op): for t in (self.static_table, self.ticking_table): @@ -233,12 +242,22 @@ def test_rolling_ops(self): self.assertEqual(len(rt.definition), 2 + len(t.definition)) with update_graph.exclusive_lock(self.test_update_graph): self.assertEqual(rt.size, t.size) + # Test rolling operators that produce a single output column + for op in self.rolling_ops_one_output: + with self.subTest(op): + for t in (self.static_table, self.ticking_table): + rt = t.update_by(ops=op, by="c") + self.assertTrue(rt.is_refreshing is t.is_refreshing) + self.assertEqual(len(rt.definition), 1 + len(t.definition)) + with update_graph.exclusive_lock(self.test_update_graph): + self.assertEqual(rt.size, t.size) def test_rolling_ops_proxy(self): pt_proxies = [self.static_table.partition_by("b").proxy(), self.ticking_table.partition_by("b").proxy(), ] + # Test rolling operators that produce 2 output columns for op in self.rolling_ops: with self.subTest(op): for pt_proxy in pt_proxies: @@ -248,6 +267,16 @@ def test_rolling_ops_proxy(self): self.assertEqual(len(rct.definition), 2 + len(ct.definition)) with update_graph.exclusive_lock(self.test_update_graph): self.assertEqual(ct.size, rct.size) + # Test rolling operators that produce a single output column + for op in self.rolling_ops_one_output: + with self.subTest(op): + for pt_proxy in pt_proxies: + rt_proxy = pt_proxy.update_by(op, by="c") + for ct, rct in zip(pt_proxy.target.constituent_tables, rt_proxy.target.constituent_tables): + self.assertTrue(rct.is_refreshing is ct.is_refreshing) + self.assertEqual(len(rct.definition), 1 + len(ct.definition)) + with update_graph.exclusive_lock(self.test_update_graph): + self.assertEqual(ct.size, rct.size) def test_multiple_ops(self): multiple_ops = [ diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateRingBuffers.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateRingBuffers.java index 71b96c5b254..b0af725f4a3 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateRingBuffers.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateRingBuffers.java @@ -65,6 +65,19 @@ public static void main(String... args) throws IOException { " System.arraycopy(storage, 0, result, firstCopyLen, secondCopyLen);\n" + " Arrays.fill(storage, 0, secondCopyLen, null);" + "\n")); + + lines = ReplicationUtils.replaceRegion(lines, "object-bulk-clear", + Collections.singletonList( + " final int storageHead = (int) (head & mask);\n" + + " final int size = size();\n" + + " // firstLen is either the size of the ring buffer or the distance from head to the end of the storage array.\n" + + + " final int firstLen = Math.min(storage.length - storageHead, size);\n" + + " // secondLen is the number of elements remaining from the first clear.\n" + + " final int secondLen = size - firstLen;\n" + + " Arrays.fill(storage, storageHead, storageHead + firstLen, null);\n" + + " Arrays.fill(storage, 0, secondLen, null);")); + FileUtils.writeLines(objectFile, lines); charToAllButBoolean(TASK, @@ -103,17 +116,24 @@ public static void main(String... args) throws IOException { // replicate the tests - charToAllButBoolean(TASK, "Base/src/test/java/io/deephaven/base/ringbuffer/CharRingBufferTest.java"); + charToAllButBoolean(TASK, "Base/src/test/java/io/deephaven/base/ringbuffer/TestCharRingBuffer.java"); objectResult = ReplicatePrimitiveCode.charToObject(TASK, - "Base/src/test/java/io/deephaven/base/ringbuffer/CharRingBufferTest.java"); + "Base/src/test/java/io/deephaven/base/ringbuffer/TestCharRingBuffer.java"); objectFile = new File(objectResult); lines = FileUtils.readLines(objectFile, Charset.defaultCharset()); lines = ReplicationUtils.globalReplacements(lines, "Object.MIN_VALUE", "new Object()"); + + lines = ReplicationUtils.replaceRegion(lines, "empty-test", + Collections.singletonList( + " for (Object val : rb.getStorage()) {\n" + + " assertNull(val);\n" + + " }")); + FileUtils.writeLines(objectFile, lines); List files = charToAllButBoolean(TASK, - "Base/src/test/java/io/deephaven/base/ringbuffer/AggregatingCharRingBufferTest.java"); + "Base/src/test/java/io/deephaven/base/ringbuffer/TestAggregatingCharRingBuffer.java"); for (final String f : files) { if (f.contains("Byte")) { objectFile = new File(f); diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateSourcesAndChunks.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateSourcesAndChunks.java index e7c66cec5b7..467933b2654 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateSourcesAndChunks.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateSourcesAndChunks.java @@ -397,8 +397,8 @@ private static void replicateObjectSingleValue() throws IOException { "Object value", "T value"); lines = ReplicationUtils.removeRegion(lines, "UnboxedSetter"); lines = ReplicationUtils.replaceRegion(lines, "Constructor", Arrays.asList( - " public ObjectSingleValueSource(Class type) {", - " super(type);", + " public ObjectSingleValueSource(Class type, Class componentType) {", + " super(type, componentType);", " current = null;", " prev = null;", " }")); diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateUpdateBy.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateUpdateBy.java index 43d66fd3158..77a2730a5e6 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateUpdateBy.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateUpdateBy.java @@ -242,6 +242,14 @@ public static void main(String[] args) throws IOException { } } + files = ReplicatePrimitiveCode.charToAllButBoolean(TASK, + "engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformulamulticolumn/windowconsumer/CharRingBufferWindowConsumer.java"); + for (final String f : files) { + if (f.contains("Int")) { + fixupInteger(f); + } + } + } private static void replicateNumericOperator(@NotNull final String shortClass, @NotNull final String floatClass) diff --git a/table-api/src/main/java/io/deephaven/api/updateby/UpdateByOperation.java b/table-api/src/main/java/io/deephaven/api/updateby/UpdateByOperation.java index 8977fe9608d..baab6690ce0 100644 --- a/table-api/src/main/java/io/deephaven/api/updateby/UpdateByOperation.java +++ b/table-api/src/main/java/io/deephaven/api/updateby/UpdateByOperation.java @@ -1661,9 +1661,9 @@ static UpdateByOperation RollingStd(long revTicks, String... pairs) { * following the current row (inclusive) * * - * Sample standard deviation is computed using Bessel's correction - * (https://en.wikipedia.org/wiki/Bessel%27s_correction), which ensures that the sample variance will be an unbiased - * estimator of population variance. + * Sample standard deviation is computed using + * Bessel's correction, which ensures that the + * sample variance will be an unbiased estimator of population variance. * * @param revTicks the look-behind window size (in rows/ticks) * @param fwdTicks the look-ahead window size (in rows/ticks) @@ -1686,9 +1686,9 @@ static UpdateByOperation RollingStd(long revTicks, long fwdTicks, String... pair *
  • {@code revDuration = 10m} - contains rows from 10m earlier through the current row timestamp (inclusive)
    • * * - * Sample standard deviation is computed using Bessel's correction - * (https://en.wikipedia.org/wiki/Bessel%27s_correction), which ensures that the sample variance will be an unbiased - * estimator of population variance. + * Sample standard deviation is computed using + * Bessel's correction, which ensures that the + * sample variance will be an unbiased estimator of population variance. * * @param timestampCol the name of the timestamp column * @param revDuration the look-behind window size (in Duration) @@ -1910,9 +1910,8 @@ static UpdateByOperation RollingWAvg(String timestampCol, long revTime, long fwd } - /** - * Create a {@link RollingFormulaSpec rolling forumla} for the supplied column name pairs, using ticks as the + * Create a {@link RollingFormulaSpec rolling formula} for the supplied column name pairs, using ticks as the * windowing unit. Ticks are row counts and you may specify the previous window in number of rows to include. The * current row is considered to belong to the reverse window, so calling this with {@code revTicks = 1} will simply * return the current row. Specifying {@code revTicks = 10} will include the previous 9 rows to this one and this @@ -1936,7 +1935,7 @@ static UpdateByOperation RollingFormula(long revTicks, String formula, String pa } /** - * Create a {@link RollingFormulaSpec rolling forumla} for the supplied column name pairs, using ticks as the + * Create a {@link RollingFormulaSpec rolling formula} for the supplied column name pairs, using ticks as the * windowing unit. Ticks are row counts and you may specify the reverse and forward window in number of rows to * include. The current row is considered to belong to the reverse window but not the forward window. Also, negative * values are allowed and can be used to generate completely forward or completely reverse windows. @@ -1976,7 +1975,7 @@ static UpdateByOperation RollingFormula(long revTicks, long fwdTicks, String for } /** - * Create a {@link RollingFormulaSpec rolling forumla} for the supplied column name pairs, using time as the + * Create a {@link RollingFormulaSpec rolling formula} for the supplied column name pairs, using time as the * windowing unit. This function accepts {@link Duration duration} as the reverse window parameter. A row containing * a {@code null} in the timestamp column belongs to no window and will not have a value computed or be considered * in the windows of other rows. @@ -2007,7 +2006,7 @@ static UpdateByOperation RollingFormula(String timestampCol, Duration revDuratio } /** - * Create a {@link RollingFormulaSpec rolling forumla} for the supplied column name pairs, using time as the + * Create a {@link RollingFormulaSpec rolling formula} for the supplied column name pairs, using time as the * windowing unit. This function accepts {@link Duration durations} as the reverse and forward window parameters. * Negative values are allowed and can be used to generate completely forward or completely reverse windows. A row * containing a {@code null} in the timestamp column belongs to no window and will not have a value computed or be @@ -2049,7 +2048,7 @@ static UpdateByOperation RollingFormula(String timestampCol, Duration revDuratio } /** - * Create a {@link RollingFormulaSpec rolling forumla} for the supplied column name pairs, using time as the + * Create a {@link RollingFormulaSpec rolling formula} for the supplied column name pairs, using time as the * windowing unit. This function accepts {@code nanoseconds} as the reverse window parameters. A row containing a * {@code null} in the timestamp column belongs to no window and will not have a value computed or be considered in * the windows of other rows. @@ -2074,7 +2073,7 @@ static UpdateByOperation RollingFormula(String timestampCol, long revTime, Strin } /** - * Create a {@link RollingFormulaSpec rolling forumla} for the supplied column name pairs, using time as the + * Create a {@link RollingFormulaSpec rolling formula} for the supplied column name pairs, using time as the * windowing unit. This function accepts {@code nanoseconds} as the reverse and forward window parameters. Negative * values are allowed and can be used to generate completely forward or completely reverse windows. A row containing * a {@code null} in the timestamp column belongs to no window and will not have a value computed or be considered @@ -2101,6 +2100,224 @@ static UpdateByOperation RollingFormula(String timestampCol, long revTime, long return RollingFormulaSpec.ofTime(timestampCol, revTime, fwdTime, formula, paramToken).clause(pairs); } + // New methods for the multi-column formula + + /** + * Create a {@link RollingFormulaSpec rolling formula} using ticks as the windowing unit. Ticks are row counts and + * you may specify the previous window in number of rows to include. The current row is considered to belong to the + * reverse window, so calling this with {@code revTicks = 1} will simply return the current row. Specifying + * {@code revTicks = 10} will include the previous 9 rows to this one and this row for a total of 10 rows. + *

    + * The provided {@code formula} should specify the output column name. Some examples of formula are: + * + * 

    +     * sum_AB = sum(col_A) + sum(col_B)
+     * max_AB = max(col_A) + max(col_B)
+     * values = col_A + col_B
+     *
    + * + * @param revTicks the look-behind window size (in rows/ticks) + * @param formula the user-defined formula to apply to each group. This formula includes the output column name and + * can contain any combination of the following: + *
      + *
    • Built-in functions such as min, max, etc.
      • + *
    • Mathematical arithmetic such as *, +, /, etc.
      • + *
    • User-defined functions
      • + *
    + * + * @return The aggregation + */ + static UpdateByOperation RollingFormula(long revTicks, String formula) { + return RollingFormulaSpec.ofTicks(revTicks, formula).clause(); + } + + /** + * Create a {@link RollingFormulaSpec rolling formula} using ticks as the windowing unit. Ticks are row counts and + * you may specify the reverse and forward window in number of rows to include. The current row is considered to + * belong to the reverse window but not the forward window. Also, negative values are allowed and can be used to + * generate completely forward or completely reverse windows. + *

    + * Here are some examples of window values: + *

      + *
    • {@code revTicks = 1, fwdTicks = 0} - contains only the current row
      • + *
    • {@code revTicks = 10, fwdTicks = 0} - contains 9 previous rows and the current row
      • + *
    • {@code revTicks = 0, fwdTicks = 10} - contains the following 10 rows, excludes the current row
      • + *
    • {@code revTicks = 10, fwdTicks = 10} - contains the previous 9 rows, the current row and the 10 rows + * following
      • + *
    • {@code revTicks = 10, fwdTicks = -5} - contains 5 rows, beginning at 9 rows before, ending at 5 rows before + * the current row (inclusive)
      • + *
    • {@code revTicks = 11, fwdTicks = -1} - contains 10 rows, beginning at 10 rows before, ending at 1 row before + * the current row (inclusive)
      • + *
    • {@code revTicks = -5, fwdTicks = 10} - contains 5 rows, beginning 5 rows following, ending at 10 rows + * following the current row (inclusive)
      • + *
    + * + *

    + * The provided {@code formula} should specify the output column name. Some examples of formula are: + * + * 

    +     * sum_AB = sum(col_A) + sum(col_B)
+     * max_AB = max(col_A) + max(col_B)
+     * values = col_A + col_B
+     *
    + * + * @param revTicks the look-behind window size (in rows/ticks) + * @param fwdTicks the look-ahead window size (in rows/ticks) + * @param formula the user-defined formula to apply to each group. This formula can contain any combination of the + * following: + *
      + *
    • Built-in functions such as min, max, etc.
      • + *
    • Mathematical arithmetic such as *, +, /, etc.
      • + *
    • User-defined functions
      • + *
    + * @return The aggregation + */ + static UpdateByOperation RollingFormula(long revTicks, long fwdTicks, String formula) { + return RollingFormulaSpec.ofTicks(revTicks, fwdTicks, formula).clause(); + } + + /** + * Create a {@link RollingFormulaSpec rolling formula} using time as the windowing unit. This function accepts + * {@link Duration duration} as the reverse window parameter. A row containing a {@code null} in the timestamp + * column belongs to no window and will not have a value computed or be considered in the windows of other rows. + *

    + * Here are some examples of window values: + *

      + *
    • {@code revDuration = 0m} - contains rows that exactly match the current row timestamp
      • + *
    • {@code revDuration = 10m} - contains rows from 10m earlier through the current row timestamp (inclusive)
      • + *
    + * + *

    + * The provided {@code formula} should specify the output column name. Some examples of formula are: + * + * 

    +     * sum_AB = sum(col_A) + sum(col_B)
+     * max_AB = max(col_A) + max(col_B)
+     * values = col_A + col_B
+     *
    + * + * @param timestampCol the name of the timestamp column + * @param revDuration the look-behind window size (in Duration) + * @param formula the user-defined {@link RollingFormulaSpec#formula() formula} to apply to each group. This formula + * can contain any combination of the following: + *
      + *
    • Built-in functions such as min, max, etc.
      • + *
    • Mathematical arithmetic such as *, +, /, etc.
      • + *
    • User-defined functions
      • + *
    + * @return The aggregation + */ + static UpdateByOperation RollingFormula(String timestampCol, Duration revDuration, String formula) { + return RollingFormulaSpec.ofTime(timestampCol, revDuration, formula).clause(); + } + + /** + * Create a {@link RollingFormulaSpec rolling formula} using time as the windowing unit. This function accepts + * {@link Duration durations} as the reverse and forward window parameters. Negative values are allowed and can be + * used to generate completely forward or completely reverse windows. A row containing a {@code null} in the + * timestamp column belongs to no window and will not have a value computed or be considered in the windows of other + * rows. + *

    + * Here are some examples of window values: + *

      + *
    • {@code revDuration = 0m, fwdDuration = 0m} - contains rows that exactly match the current row timestamp
      • + *
    • {@code revDuration = 10m, fwdDuration = 0m} - contains rows from 10m earlier through the current row + * timestamp (inclusive)
      • + *
    • {@code revDuration = 0m, fwdDuration = 10m} - contains rows from the current row through 10m following the + * current row timestamp (inclusive)
      • + *
    • {@code revDuration = 10m, fwdDuration = 10m} - contains rows from 10m earlier through 10m following the + * current row timestamp (inclusive)
      • + *
    • {@code revDuration = 10m, fwdDuration = -5m} - contains rows from 10m earlier through 5m before the current + * row timestamp (inclusive), this is a purely backwards looking window
      • + *
    • {@code revDuration = -5m, fwdDuration = 10m} - contains rows from 5m following through 10m following the + * current row timestamp (inclusive), this is a purely forwards looking window
      • + *
    + * + *

    + * The provided {@code formula} should specify the output column name. Some examples of formula are: + * + * 

    +     * sum_AB = sum(col_A) + sum(col_B)
+     * max_AB = max(col_A) + max(col_B)
+     * values = col_A + col_B
+     *
    + * + * @param timestampCol the name of the timestamp column + * @param revDuration the look-behind window size (in Duration) + * @param fwdDuration the look-ahead window size (in Duration) + * @param formula the user-defined {@link RollingFormulaSpec#formula() formula} to apply to each group. This formula + * can contain any combination of the following: + *
      + *
    • Built-in functions such as min, max, etc.
      • + *
    • Mathematical arithmetic such as *, +, /, etc.
      • + *
    • User-defined functions
      • + *
    + * @return The aggregation + */ + static UpdateByOperation RollingFormula(String timestampCol, Duration revDuration, Duration fwdDuration, + String formula) { + return RollingFormulaSpec.ofTime(timestampCol, revDuration, fwdDuration, formula).clause(); + } + + /** + * Create a {@link RollingFormulaSpec rolling formula} using time as the windowing unit. This function accepts + * {@code nanoseconds} as the reverse window parameters. A row containing a {@code null} in the timestamp column + * belongs to no window and will not have a value computed or be considered in the windows of other rows. + * + *

    + * The provided {@code formula} should specify the output column name. Some examples of formula are: + * + * 

    +     * sum_AB = sum(col_A) + sum(col_B)
+     * max_AB = max(col_A) + max(col_B)
+     * values = col_A + col_B
+     *
    + * + * @param timestampCol the name of the timestamp column + * @param revTime the look-behind window size (in nanoseconds) + * @param formula the user-defined formula to apply to each group. This formula can contain any combination of the + * following: + *
      + *
    • Built-in functions such as min, max, etc.
      • + *
    • Mathematical arithmetic such as *, +, /, etc.
      • + *
    • User-defined functions
      • + *
    + * @return The aggregation + */ + static UpdateByOperation RollingFormula(String timestampCol, long revTime, String formula) { + return RollingFormulaSpec.ofTime(timestampCol, revTime, formula).clause(); + } + + /** + * Create a {@link RollingFormulaSpec rolling formula} using time as the windowing unit. This function accepts + * {@code nanoseconds} as the reverse and forward window parameters. Negative values are allowed and can be used to + * generate completely forward or completely reverse windows. A row containing a {@code null} in the timestamp + * column belongs to no window and will not have a value computed or be considered in the windows of other rows. + * + *

    + * The provided {@code formula} should specify the output column name. Some examples of formula are: + * + * 

    +     * sum_AB = sum(col_A) + sum(col_B)
+     * max_AB = max(col_A) + max(col_B)
+     * values = col_A + col_B
+     *
    + * + * @param timestampCol the name of the timestamp column + * @param revTime the look-behind window size (in nanoseconds) + * @param fwdTime the look-ahead window size (in nanoseconds) + * @param formula the user-defined formula to apply to each group. This formula can contain any combination of the + * following: + *
      + *
    • Built-in functions such as min, max, etc.
      • + *
    • Mathematical arithmetic such as *, +, /, etc.
      • + *
    • User-defined functions
      • + *
    + * @return The aggregation + */ + static UpdateByOperation RollingFormula(String timestampCol, long revTime, long fwdTime, String formula) { + return RollingFormulaSpec.ofTime(timestampCol, revTime, fwdTime, formula).clause(); + } T walk(Visitor visitor); diff --git a/table-api/src/main/java/io/deephaven/api/updateby/spec/RollingFormulaSpec.java b/table-api/src/main/java/io/deephaven/api/updateby/spec/RollingFormulaSpec.java index 55b57c48159..b4576d3eb51 100644 --- a/table-api/src/main/java/io/deephaven/api/updateby/spec/RollingFormulaSpec.java +++ b/table-api/src/main/java/io/deephaven/api/updateby/spec/RollingFormulaSpec.java @@ -4,10 +4,12 @@ package io.deephaven.api.updateby.spec; import io.deephaven.annotations.BuildableStyle; +import io.deephaven.api.Selectable; import org.immutables.value.Value; import org.immutables.value.Value.Immutable; import java.time.Duration; +import java.util.Optional; /** * An {@link UpdateBySpec} for performing a windowed rolling formula operation. @@ -15,10 +17,24 @@ @Immutable @BuildableStyle public abstract class RollingFormulaSpec extends RollingOpSpec { - + /** + * The formula to use to calculate output values. The formula is similar to + * {@link io.deephaven.api.TableOperations#update} and {@link io.deephaven.api.TableOperations#updateView} in + * specifying the output column name and the expression to compute in terms of the input columns. (e.g. + * {@code "outputCol = sum(inputColA + inputColB)"}). + *

    + * The alternative (and deprecated) form for {@link #formula()} is active when {@link #paramToken()} is provided. In + * this case the formula should only contain the expression in terms of the token. (e.g. {@code sum(x)} where x is + * the {@link #paramToken()}). NOTE: This form is deprecated and will be removed in a future release. + */ public abstract String formula(); - public abstract String paramToken(); + /** + * (Deprecated) The token to use in {@link #formula()} to represent the input column. If this parameter is provided, + * then only a single input column can be provided in the formula. + */ + @Deprecated + public abstract Optional paramToken(); public static RollingFormulaSpec ofTicks(long revTicks, String formula, String paramToken) { return of(WindowScale.ofTicks(revTicks), formula, paramToken); @@ -53,6 +69,41 @@ public static RollingFormulaSpec ofTime(final String timestampCol, long revDurat formula, paramToken); } + // New methods for supporting the non-tokenized version (expects valid column names in the formula) + + public static RollingFormulaSpec ofTicks(long revTicks, String formula) { + return of(WindowScale.ofTicks(revTicks), formula); + } + + public static RollingFormulaSpec ofTicks(long revTicks, long fwdTicks, String formula) { + return of(WindowScale.ofTicks(revTicks), WindowScale.ofTicks(fwdTicks), formula); + } + + public static RollingFormulaSpec ofTime(final String timestampCol, Duration revDuration, String formula) { + return of(WindowScale.ofTime(timestampCol, revDuration), formula); + } + + public static RollingFormulaSpec ofTime(final String timestampCol, Duration revDuration, Duration fwdDuration, + String formula) { + return of(WindowScale.ofTime(timestampCol, revDuration), + WindowScale.ofTime(timestampCol, fwdDuration), + formula); + } + + public static RollingFormulaSpec ofTime(final String timestampCol, long revDuration, String formula) { + return of(WindowScale.ofTime(timestampCol, revDuration), + formula); + } + + public static RollingFormulaSpec ofTime(final String timestampCol, long revDuration, long fwdDuration, + String formula) { + return of(WindowScale.ofTime(timestampCol, revDuration), + WindowScale.ofTime(timestampCol, fwdDuration), + formula); + } + + // Base methods for creating the RollingFormulaSpec + public static RollingFormulaSpec of(WindowScale revWindowScale, String formula, String paramToken) { return ImmutableRollingFormulaSpec.builder() .revWindowScale(revWindowScale) @@ -71,6 +122,33 @@ public static RollingFormulaSpec of(WindowScale revWindowScale, WindowScale fwdW .build(); } + public static RollingFormulaSpec of(WindowScale revWindowScale, String formula) { + return ImmutableRollingFormulaSpec.builder() + .revWindowScale(revWindowScale) + .formula(formula) + .build(); + } + + public static RollingFormulaSpec of(WindowScale revWindowScale, WindowScale fwdWindowScale, String formula) { + return ImmutableRollingFormulaSpec.builder() + .revWindowScale(revWindowScale) + .fwdWindowScale(fwdWindowScale) + .formula(formula) + .build(); + } + + /** + * If {@link #paramToken()} is not supplied, this will contain a {@link Selectable} of the parsed + * {@link #formula()}. + */ + @Value.Lazy + public Selectable selectable() { + if (paramToken().isPresent()) { + throw new UnsupportedOperationException("selectable() is not supported when paramToken() is present"); + } + return Selectable.parse(formula()); + } + @Override public final boolean applicableTo(Class inputType) { return true; @@ -86,12 +164,9 @@ final void checkFormula() { if (formula().isEmpty()) { throw new IllegalArgumentException("formula must not be empty"); } - } - - @Value.Check - final void checkParamToken() { - if (paramToken().isEmpty()) { - throw new IllegalArgumentException("paramToken must not be empty"); + if (!paramToken().isPresent()) { + // Call the selectable method to parse now and throw on invalid input. + selectable(); } } } diff --git a/table-api/src/main/java/io/deephaven/api/updateby/spec/RollingOpSpec.java b/table-api/src/main/java/io/deephaven/api/updateby/spec/RollingOpSpec.java index c90bdc457f7..e6da6a2364f 100644 --- a/table-api/src/main/java/io/deephaven/api/updateby/spec/RollingOpSpec.java +++ b/table-api/src/main/java/io/deephaven/api/updateby/spec/RollingOpSpec.java @@ -10,24 +10,32 @@ */ public abstract class RollingOpSpec extends UpdateBySpecBase { - // We would like to use jdk.internal.util.ArraysSupport.MAX_ARRAY_LENGTH, but it is not exported + /** + * We would like to use jdk.internal.util.ArraysSupport.MAX_ARRAY_LENGTH, but it is not exported + */ final static int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8; - // Provide a default reverse-looking timescale + /** + * Provide a default reverse-looking timescale + */ @Value.Default public WindowScale revWindowScale() { return WindowScale.ofTicks(0); } - // Provide a default forward-looking timescale + /** + * Provide a default forward-looking timescale + */ @Value.Default public WindowScale fwdWindowScale() { return WindowScale.ofTicks(0); } + /** + * Assert some rational constraints on window sizes (leq MAX_SIZE and geq 0) + */ @Value.Check final void checkWindowSizes() { - // assert some rational constraints on window sizes (leq MAX_SIZE and geq 0) final double size = revWindowScale().getFractionalTimeScaleUnits() + fwdWindowScale().getFractionalTimeScaleUnits(); if (size < 0) { @@ -37,4 +45,16 @@ final void checkWindowSizes() { "UpdateBy rolling window size may not exceed MAX_ARRAY_SIZE (" + MAX_ARRAY_SIZE + ")"); } } + + /** + * Assert forward and reverse time columns match each other. + */ + @Value.Check + final void checkCompatibleTimeColumns() { + if (revWindowScale().timestampCol() != null && fwdWindowScale().timestampCol() != null + && !revWindowScale().timestampCol().equals(fwdWindowScale().timestampCol())) { + throw new IllegalArgumentException( + "UpdateBy rolling window forward and reverse timestamp columns must match"); + } + } } diff --git a/table-api/src/main/java/io/deephaven/api/updateby/spec/UpdateBySpec.java b/table-api/src/main/java/io/deephaven/api/updateby/spec/UpdateBySpec.java index 0c8813edde3..1536c80f551 100644 --- a/table-api/src/main/java/io/deephaven/api/updateby/spec/UpdateBySpec.java +++ b/table-api/src/main/java/io/deephaven/api/updateby/spec/UpdateBySpec.java @@ -61,6 +61,13 @@ public interface UpdateBySpec { */ ColumnUpdateOperation clause(Collection pairs); + /** + * Build a {@link ColumnUpdateOperation} clause for this UpdateBySpec with no input/output column name specified. + * + * @return The aggregation + */ + ColumnUpdateOperation clause(); + // region Visitor T walk(Visitor visitor); diff --git a/table-api/src/main/java/io/deephaven/api/updateby/spec/UpdateBySpecBase.java b/table-api/src/main/java/io/deephaven/api/updateby/spec/UpdateBySpecBase.java index 366f792a568..efd9cacbb84 100644 --- a/table-api/src/main/java/io/deephaven/api/updateby/spec/UpdateBySpecBase.java +++ b/table-api/src/main/java/io/deephaven/api/updateby/spec/UpdateBySpecBase.java @@ -48,6 +48,13 @@ public final ColumnUpdateOperation clause(Collection pairs) { .build(); } + @Override + public final ColumnUpdateOperation clause() { + return ColumnUpdateOperation.builder() + .spec(this) + .build(); + } + /** * Returns {@code true} if the input class is a primitive or boxed numeric type *