Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add additional counting aggregations to AggBy #6358

Open
wants to merge 13 commits into
base: main
Choose a base branch
from
Open

feat: add additional counting aggregations to AggBy #6358

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
1e21206
Initial commit for AggCount additional operators.
lbooker42 Nov 8, 2024
94eddb2
Added python server and GRPC client support.
lbooker42 Nov 13, 2024
67e49ee
Correct build failure.
lbooker42 Nov 14, 2024
4db4e5d
Merge branch 'main' into lab-agg-count
lbooker42 Nov 15, 2024
ca3cad2
Added `agg_by` `count_all` alias to `count_`.
lbooker42 Nov 15, 2024
03d01be
Relocated AggCountType to be useful for multiple count operations.
lbooker42 Nov 19, 2024
3b61be1
Unifying the GRPC among the count types.
lbooker42 Nov 20, 2024
85f08dd
Merged with main.
lbooker42 Nov 20, 2024
776f1cd
Correct TestAggBy incorrect column generators for 'charCol'.
lbooker42 Nov 20, 2024
890e474
Added CountNonNZero, CountNonNegative, CountNonPositive AggCountType …
lbooker42 Nov 20, 2024
63530da
Merged with main.
lbooker42 Nov 20, 2024
3d0cde8
Doc changes and additional tests.
lbooker42 Nov 20, 2024
68bfa32
Updated CountOperator modify chunk to assert non-negative count are m…
lbooker42 Nov 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion engine/table/src/main/java/io/deephaven/engine/table/impl/by/BaseChunkedCountOperator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
//
package io.deephaven.engine.table.impl.by;

import io.deephaven.base.verify.Assert;
import io.deephaven.chunk.*;
import io.deephaven.chunk.attributes.ChunkLengths;
import io.deephaven.chunk.attributes.ChunkPositions;
Expand Down Expand Up @@ -104,7 +105,9 @@ private boolean removeChunk(
int chunkSize) {
final int count = doCount(chunkStart, chunkSize, values);
if (count > 0) {
resultColumnSource.set(destination, plusLong(resultColumnSource.getUnsafe(destination), -count));
final long newCount = plusLong(resultColumnSource.getUnsafe(destination), -count);
Assert.geqZero(newCount, "newCount");
resultColumnSource.set(destination, newCount);
lbooker42 marked this conversation as resolved.
Show resolved Hide resolved
return true;
}
return false;
Expand Down
49 changes: 47 additions & 2 deletions engine/table/src/test/java/io/deephaven/engine/table/impl/TestAggBy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public void testBy() {
ColumnHolder<?> bHolder = col("B", 1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
ColumnHolder<?> cHolder = col("C", 1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
ColumnHolder<?> dHolder = col("D", -5, NULL_INT, -3, NULL_INT, -1, 0, 1, NULL_INT, 3, NULL_INT);
ColumnHolder<?> eHolder = col("E", Double.NEGATIVE_INFINITY, -4.0, NULL_DOUBLE, NULL_DOUBLE, -1.0, 0.0, 1.0,
ColumnHolder<?> eHolder = col("E", Double.NEGATIVE_INFINITY, -4.0, NULL_DOUBLE, NULL_DOUBLE, -1.0, -0.0, 1.0,
Double.NaN, 3.0, Double.POSITIVE_INFINITY);
ColumnHolder<?> fHolder =
col("F", BigDecimal.valueOf(-5), null, BigDecimal.valueOf(-3), null, BigDecimal.valueOf(-1),
Expand Down Expand Up @@ -141,7 +141,10 @@ public void testBy() {
AggCountZero("zeroD=D", "zeroE=E", "zeroF=F", "zeroG=G"),
AggCountNaN("nanD=D", "nanE=E", "nanF=F", "nanG=G"),
AggCountInfinite("infD=D", "infE=E", "infF=F", "infG=G"),
AggCountFinite("finiteD=D", "finiteE=E", "finiteF=F", "finiteG=G")),
AggCountFinite("finiteD=D", "finiteE=E", "finiteF=F", "finiteG=G"),
AggCountNonZero("nonZeroD=D", "nonZeroE=E", "nonZeroF=F", "nonZeroG=G"),
AggCountNonNegative("nonNegD=D", "nonNegE=E", "nonNegF=F", "nonNegG=G"),
AggCountNonPositive("nonPosD=D", "nonPosE=E", "nonPosF=F", "nonPosG=G")),
"A");
show(doubleCounted);
assertEquals(2, doubleCounted.size());
Expand Down Expand Up @@ -265,6 +268,45 @@ public void testBy() {
counts = ColumnVectors.ofLong(doubleCounted, "finiteG");
assertEquals(4L, counts.get(0));
assertEquals(2L, counts.get(1));
//////////////////////////////
counts = ColumnVectors.ofLong(doubleCounted, "nonZeroD");
assertEquals(3L, counts.get(0));
assertEquals(2L, counts.get(1));
counts = ColumnVectors.ofLong(doubleCounted, "nonZeroE");
assertEquals(5L, counts.get(0));
assertEquals(2L, counts.get(1));
counts = ColumnVectors.ofLong(doubleCounted, "nonZeroF");
assertEquals(3L, counts.get(0));
assertEquals(2L, counts.get(1));
counts = ColumnVectors.ofLong(doubleCounted, "nonZeroG");
assertEquals(3L, counts.get(0));
assertEquals(2L, counts.get(1));
//////////////////////////////
counts = ColumnVectors.ofLong(doubleCounted, "nonNegD");
assertEquals(2L, counts.get(0));
assertEquals(1L, counts.get(1));
counts = ColumnVectors.ofLong(doubleCounted, "nonNegE");
assertEquals(3L, counts.get(0));
assertEquals(1L, counts.get(1));
counts = ColumnVectors.ofLong(doubleCounted, "nonNegF");
assertEquals(2L, counts.get(0));
assertEquals(1L, counts.get(1));
counts = ColumnVectors.ofLong(doubleCounted, "nonNegG");
assertEquals(2L, counts.get(0));
assertEquals(1L, counts.get(1));
//////////////////////////////
counts = ColumnVectors.ofLong(doubleCounted, "nonPosD");
assertEquals(3L, counts.get(0));
assertEquals(1L, counts.get(1));
counts = ColumnVectors.ofLong(doubleCounted, "nonPosE");
assertEquals(4L, counts.get(0));
assertEquals(0L, counts.get(1));
counts = ColumnVectors.ofLong(doubleCounted, "nonPosF");
assertEquals(3L, counts.get(0));
assertEquals(1L, counts.get(1));
counts = ColumnVectors.ofLong(doubleCounted, "nonPosG");
assertEquals(3L, counts.get(0));
assertEquals(1L, counts.get(1));

// Lets do some interesting incremental computations, as this is the use case that I'm really aiming at. For
// example, getting the count, and average on each update.
Expand Down Expand Up @@ -390,6 +432,9 @@ public void testComboByCountTypes() {
EvalNugget.from(() -> queryTable.aggBy(AggCountNaN(supportedColumns), "Sym").sort("Sym")),
EvalNugget.from(() -> queryTable.aggBy(AggCountInfinite(supportedColumns), "Sym").sort("Sym")),
EvalNugget.from(() -> queryTable.aggBy(AggCountFinite(supportedColumns), "Sym").sort("Sym")),
EvalNugget.from(() -> queryTable.aggBy(AggCountNonZero(supportedColumns), "Sym").sort("Sym")),
EvalNugget.from(() -> queryTable.aggBy(AggCountNonNegative(supportedColumns), "Sym").sort("Sym")),
EvalNugget.from(() -> queryTable.aggBy(AggCountNonPositive(supportedColumns), "Sym").sort("Sym")),
};
final int steps = 100; // 8;
for (int step = 0; step < steps; step++) {
Expand Down
Loading