diff --git a/engine/api/src/main/java/io/deephaven/engine/exceptions/IncompatibleTableDefinitionException.java b/engine/api/src/main/java/io/deephaven/engine/exceptions/IncompatibleTableDefinitionException.java new file mode 100644 index 00000000000..479154bfe5d --- /dev/null +++ b/engine/api/src/main/java/io/deephaven/engine/exceptions/IncompatibleTableDefinitionException.java @@ -0,0 +1,26 @@ +package io.deephaven.engine.exceptions; + +import io.deephaven.UncheckedDeephavenException; + +/** + * Runtime exception representing an incompatibility between table definitions. + */ +@SuppressWarnings({"WeakerAccess", "unused"}) +public class IncompatibleTableDefinitionException extends UncheckedDeephavenException { + + public IncompatibleTableDefinitionException() { + super(); + } + + public IncompatibleTableDefinitionException(String message) { + super(message); + } + + public IncompatibleTableDefinitionException(String message, Throwable cause) { + super(message, cause); + } + + public IncompatibleTableDefinitionException(Throwable cause) { + super(cause); + } +} diff --git a/engine/api/src/main/java/io/deephaven/engine/table/ColumnSource.java b/engine/api/src/main/java/io/deephaven/engine/table/ColumnSource.java index 9299fcfca7b..f3486672106 100644 --- a/engine/api/src/main/java/io/deephaven/engine/table/ColumnSource.java +++ b/engine/api/src/main/java/io/deephaven/engine/table/ColumnSource.java @@ -50,7 +50,7 @@ WritableRowSet match( * values until this method is called. This is an option, not an obligation: some simple ColumnSource * implementations (like TSingleValueSource for various T) always track previous values; other implementations (like * PrevColumnSource) never do; some (like TArrayColumnSource) only start tracking once this method is called. - * + *

* An immutable column source can not have distinct prev values; therefore it is implemented as a no-op. */ default void startTrackingPrevValues() { @@ -202,7 +202,7 @@ default ColumnSource cast(Class clazz, @Nullable Cl /** * Most column sources will return the same value for a given row without respect to the order that the rows are * read. Those columns sources are considered "stateless" and should return true. - * + *

* Some column sources, however may be dependent on evaluation order. For example, a formula that updates a Map must * be evaluated from the first row to the last row. A column source that has the potential to depend on the order of * evaluation must return false. diff --git a/engine/api/src/main/java/io/deephaven/engine/table/TableDefinition.java b/engine/api/src/main/java/io/deephaven/engine/table/TableDefinition.java index 45fe5a33a37..a2903fc4f2f 100644 --- a/engine/api/src/main/java/io/deephaven/engine/table/TableDefinition.java +++ b/engine/api/src/main/java/io/deephaven/engine/table/TableDefinition.java @@ -3,13 +3,13 @@ */ package io.deephaven.engine.table; -import io.deephaven.UncheckedDeephavenException; import io.deephaven.api.ColumnName; import io.deephaven.base.cache.OpenAddressedCanonicalizationCache; import io.deephaven.base.log.LogOutput; import io.deephaven.base.log.LogOutputAppendable; import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.impl.NoSuchColumnException; +import io.deephaven.engine.exceptions.IncompatibleTableDefinitionException; import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.qst.column.header.ColumnHeader; import org.jetbrains.annotations.NotNull; @@ -654,29 +654,4 @@ private List> getWritableColumns(final boolean partitioningT } return columns; } - - /** - * Runtime exception representing an incompatibility between table definitions. - */ - @SuppressWarnings({"WeakerAccess", "unused"}) - public static class IncompatibleTableDefinitionException extends UncheckedDeephavenException { - - private static final long serialVersionUID = 7668080323885707687L; - - public IncompatibleTableDefinitionException() { - super(); - } - - public IncompatibleTableDefinitionException(String message) { - super(message); - } - - public IncompatibleTableDefinitionException(String message, Throwable cause) { - super(message, cause); - } - - public IncompatibleTableDefinitionException(Throwable cause) { - super(cause); - } - } } diff --git a/engine/base/src/main/java/io/deephaven/engine/table/impl/DefaultChunkSource.java b/engine/base/src/main/java/io/deephaven/engine/table/impl/DefaultChunkSource.java index fbfdff00865..ef46a04fcb7 100644 --- a/engine/base/src/main/java/io/deephaven/engine/table/impl/DefaultChunkSource.java +++ b/engine/base/src/main/java/io/deephaven/engine/table/impl/DefaultChunkSource.java @@ -72,7 +72,7 @@ default Chunk getPrevChunkByFilling(@NotNull final GetContext context, default ChunkSource getPrevSource() { final ChunkSource.WithPrev chunkSource = this; - return new ChunkSource() { + return new ChunkSource<>() { @Override public ChunkType getChunkType() { return chunkSource.getChunkType(); diff --git a/engine/table/build.gradle b/engine/table/build.gradle index 261678a3c06..3732a1dad07 100644 --- a/engine/table/build.gradle +++ b/engine/table/build.gradle @@ -37,6 +37,7 @@ dependencies { implementation project(':plugin') implementation depCommonsLang3 + implementation 'org.testng:testng:7.1.0' Classpaths.inheritCommonsText(project, 'implementation') Classpaths.inheritGroovy(project, 'groovy', 'implementation') diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractColumnSource.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractColumnSource.java index 889ab074f06..e057aca5c16 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractColumnSource.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/AbstractColumnSource.java @@ -140,6 +140,7 @@ public WritableRowSet match( final boolean caseInsensitive, @NotNull final RowSet mapper, final Object... keys) { + // TODO (deephaven-core#3851): port this to new grouping API final Map groupToRange = (isImmutable() || !usePrev) ? getGroupToRange(mapper) : null; if (groupToRange != null) { RowSetBuilderRandom allInMatchingGroups = RowSetFactory.builderRandom(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/SourcePartitionedTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/SourcePartitionedTable.java index 4abda3d3786..db7a72cb8e4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/SourcePartitionedTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/SourcePartitionedTable.java @@ -203,7 +203,7 @@ private Table makeConstituentTable(@NotNull final TableLocation tableLocation) { final PartitionAwareSourceTable constituent = new PartitionAwareSourceTable( constituentDefinition, "SingleLocationSourceTable-" + tableLocation, - RegionedTableComponentFactoryImpl.INSTANCE, + RegionedTableComponentFactoryImpl.make(), new SingleTableLocationProvider(tableLocation), refreshSizes ? refreshCombiner : null); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/chunkfilter/ChunkFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/chunkfilter/ChunkFilter.java index 6fb73115598..25996bab3d2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/chunkfilter/ChunkFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/chunkfilter/ChunkFilter.java @@ -7,10 +7,12 @@ import io.deephaven.engine.exceptions.CancellationException; import io.deephaven.engine.rowset.*; import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.table.ChunkSource; import io.deephaven.engine.table.ColumnSource; import io.deephaven.chunk.*; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; import io.deephaven.chunk.attributes.Values; +import org.jetbrains.annotations.NotNull; public interface ChunkFilter { /** @@ -138,14 +140,17 @@ default void filter(Chunk values, LongChunk ke * filter. * * @param selection the RowSet to filter - * @param columnSource the column source to filter + * @param chunkSource the chunk source to filter * @param usePrev should we use previous values from the column source? * @param chunkFilter the chunk filter to apply * * @return A new WritableRowSet representing the filtered values, owned by the caller */ - static WritableRowSet applyChunkFilter(RowSet selection, ColumnSource columnSource, boolean usePrev, - ChunkFilter chunkFilter) { + static WritableRowSet applyChunkFilter( + @NotNull final RowSet selection, + @NotNull final ChunkSource chunkSource, + final boolean usePrev, + @NotNull final ChunkFilter chunkFilter) { final RowSetBuilderSequential builder = RowSetFactory.builderSequential(); final int contextSize = (int) Math.min(FILTER_CHUNK_SIZE, selection.size()); @@ -153,7 +158,7 @@ static WritableRowSet applyChunkFilter(RowSet selection, ColumnSource columnS long filteredChunks = 0; long lastInterruptCheck = System.currentTimeMillis(); - try (final ColumnSource.GetContext getContext = columnSource.makeGetContext(contextSize); + try (final ColumnSource.GetContext getContext = chunkSource.makeGetContext(contextSize); final WritableLongChunk longChunk = WritableLongChunk.makeWritableChunk(contextSize); final RowSequence.Iterator rsIt = selection.getRowSequenceIterator()) { while (rsIt.hasMore()) { @@ -176,9 +181,11 @@ static WritableRowSet applyChunkFilter(RowSet selection, ColumnSource columnS final Chunk dataChunk; if (usePrev) { - dataChunk = columnSource.getPrevChunk(getContext, okChunk); + // noinspection unchecked + dataChunk = ((ChunkSource.WithPrev) chunkSource) + .getPrevChunk(getContext, okChunk); } else { - dataChunk = columnSource.getChunk(getContext, okChunk); + dataChunk = chunkSource.getChunk(getContext, okChunk); } chunkFilter.filter(dataChunk, keyChunk, longChunk); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/ColumnLocation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/ColumnLocation.java index fe6073508bf..6bf38ede4f6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/ColumnLocation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/ColumnLocation.java @@ -41,6 +41,16 @@ public interface ColumnLocation extends StringUtils.StringKeyedObject, NamedImpl */ boolean exists(); + /** + * Get this column location cast to the specified type + * + * @return {@code this}, with the appropriate cast applied + */ + default CL cast() { + // noinspection unchecked + return (CL) this; + } + /** *

* Get the metadata object stored with this column, or null if no such data exists. diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/KeyRangeGroupingProvider.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/KeyRangeGroupingProvider.java index fbf2d98f2c4..a8695e8d286 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/KeyRangeGroupingProvider.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/KeyRangeGroupingProvider.java @@ -4,7 +4,6 @@ package io.deephaven.engine.table.impl.locations; import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.rowset.TrackingWritableRowSet; import org.jetbrains.annotations.NotNull; /** diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/TableLocation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/TableLocation.java index a4fb0085240..b086f0d6d9b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/TableLocation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/TableLocation.java @@ -3,12 +3,17 @@ */ package io.deephaven.engine.table.impl.locations; +import io.deephaven.api.SortColumn; import io.deephaven.base.log.LogOutput; import io.deephaven.base.log.LogOutputAppendable; +import io.deephaven.engine.table.Table; import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.util.annotations.FinalDefault; import io.deephaven.util.type.NamedImplementation; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; /** * Building block for Deephaven "source" tables, with helper methods for discovering locations and their sizes. A @@ -36,6 +41,12 @@ interface Listener extends BasicTableDataListener { @NotNull ImmutableTableKey getTableKey(); + // TODO: NATE NOCOMMIT IF UNUSED ELSE MAKE CAST + default T as(Class otherType) { + // noinspection unchecked + return (T) this; + } + /** * @return An {@link ImmutableTableLocationKey} instance for this location */ @@ -77,6 +88,31 @@ interface Listener extends BasicTableDataListener { */ void refresh(); + /** + * Get an ordered list of columns this location is sorted by. + * + * @return a non-null ordered list of {@link SortColumn}s + */ + @NotNull + List getSortedColumns(); + + /** + * Check if this location has a data index for the specified columns. + * + * @param columns the set of columns to check for. + * @return true if the table has a Data Index for the specified columns + */ + boolean hasDataIndexFor(@NotNull String... columns); + + /** + * Get the data index table for the specified set of columns. Note that the order of columns does not matter here. + * + * @param columns the key columns for the index + * @return the index table or null if one does not exist. + */ + @Nullable + Table getDataIndex(@NotNull String... columns); + /** * @param name The column name * @return The ColumnLocation for the defined column under this table location diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/AbstractTableLocation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/AbstractTableLocation.java index 85ddd813832..04e1c226904 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/AbstractTableLocation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/AbstractTableLocation.java @@ -4,11 +4,20 @@ package io.deephaven.engine.table.impl.locations.impl; import io.deephaven.base.verify.Require; +import io.deephaven.engine.table.Table; import io.deephaven.engine.util.string.StringUtils; import io.deephaven.engine.table.impl.locations.*; import io.deephaven.engine.rowset.RowSet; import io.deephaven.hash.KeyedObjectHashMap; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.lang.ref.SoftReference; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; /** * Partial TableLocation implementation for use by TableDataService implementations. @@ -16,6 +25,7 @@ public abstract class AbstractTableLocation extends SubscriptionAggregator implements TableLocation { + protected static final SoftReference NO_GROUPING_SENTINEL = new SoftReference<>(null); private final ImmutableTableKey tableKey; private final ImmutableTableLocationKey tableLocationKey; @@ -24,6 +34,11 @@ public abstract class AbstractTableLocation private final KeyedObjectHashMap columnLocations = new KeyedObjectHashMap<>(StringUtils.charSequenceKey()); + /** + * A map of grouping (or data index) columns to the materialized + */ + protected volatile Map, SoftReference
> cachedGroupings; + /** * @param tableKey Table key for the table this location belongs to * @param tableLocationKey Table location key that identifies this location @@ -138,4 +153,63 @@ public final ColumnLocation getColumnLocation(@NotNull final CharSequence name) protected final void clearColumnLocations() { columnLocations.clear(); } + + @Nullable + @Override + public final Table getDataIndex(@NotNull final String... columns) { + final List colNames = Arrays.asList(columns); + Table grouping = null; + if (cachedGroupings != null) { + final SoftReference
cachedGrouping = cachedGroupings.get(colNames); + if (cachedGrouping == NO_GROUPING_SENTINEL) { + return null; + } + + if (cachedGrouping != null) { + grouping = cachedGrouping.get(); + if (grouping != null) { + // System.out.println("HAD CACHE"); + return grouping; + } + } + } + + synchronized (this) { + if (cachedGroupings == null) { + cachedGroupings = new HashMap<>(); + } + + final SoftReference
cachedGrouping = cachedGroupings.get(colNames); + if (cachedGrouping == NO_GROUPING_SENTINEL) { + return null; + } + + if (cachedGrouping != null) { + grouping = cachedGrouping.get(); + } + + if (grouping == null) { + grouping = getDataIndexImpl(columns); + + if (grouping == null || grouping.isEmpty()) { + cachedGroupings.put(colNames, NO_GROUPING_SENTINEL); + } else { + // System.out.println("NO CACHE"); + cachedGroupings.put(colNames, new SoftReference<>(grouping)); + } + } + + return grouping; + } + } + + /** + * Load the data index from the location implementation. Implementations of this method should not perform any + * result caching. + * + * @param columns the columns to load an index for. + * @return the data index table, or an empty table or null if none existed. + */ + @Nullable + protected abstract Table getDataIndexImpl(@NotNull final String... columns); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/NonexistentTableLocation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/NonexistentTableLocation.java index 9bca3b3087f..15ca82055ce 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/NonexistentTableLocation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/locations/impl/NonexistentTableLocation.java @@ -3,12 +3,17 @@ */ package io.deephaven.engine.table.impl.locations.impl; +import io.deephaven.api.SortColumn; +import io.deephaven.engine.table.Table; import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.locations.TableKey; import io.deephaven.engine.table.impl.locations.TableLocation; import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.rowset.RowSetFactory; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; /** * {@link TableLocation} implementation for locations that are found to not actually exist when accessed. @@ -36,4 +41,20 @@ public void refresh() {} protected ColumnLocation makeColumnLocation(@NotNull String name) { throw new UnsupportedOperationException(); } + + @Override + public @NotNull List getSortedColumns() { + // TODO NATE NOCOMMIT: what is ideal here? + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasDataIndexFor(@NotNull String... columns) { + return false; + } + + @Override + protected @Nullable Table getDataIndexImpl(@NotNull String... columns) { + return null; + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 74fb3080b2b..51b470c5744 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -13,6 +13,8 @@ import io.deephaven.engine.table.impl.preview.DisplayWrapper; import io.deephaven.engine.context.QueryScope; import io.deephaven.time.DateTimeUtils; +import io.deephaven.util.BooleanUtils; +import io.deephaven.util.QueryConstants; import io.deephaven.util.type.ArrayTypeUtils; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.rowset.RowSet; @@ -206,6 +208,9 @@ public static ColumnTypeConvertor getConvertor(final Class cls, final String return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_BYTE".equals(str)) { + return QueryConstants.NULL_BYTE; + } return Byte.parseByte(str); } }; @@ -214,6 +219,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_SHORT".equals(str)) { + return QueryConstants.NULL_SHORT; + } return Short.parseShort(str); } }; @@ -222,6 +230,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_INT".equals(str)) { + return QueryConstants.NULL_INT; + } return Integer.parseInt(str); } }; @@ -230,6 +241,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_LONG".equals(str)) { + return QueryConstants.NULL_LONG; + } return Long.parseLong(str); } }; @@ -238,6 +252,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_FLOAT".equals(str)) { + return QueryConstants.NULL_FLOAT; + } return Float.parseFloat(str); } }; @@ -246,6 +263,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_DOUBLE".equals(str)) { + return QueryConstants.NULL_DOUBLE; + } return Double.parseDouble(str); } }; @@ -261,6 +281,9 @@ Object convertStringLiteral(String str) { if (str.equalsIgnoreCase("false")) { return Boolean.FALSE; } + if ("null".equals(str) || "NULL_BOOLEAN".equals(str)) { + return BooleanUtils.NULL_BOOLEAN_AS_BYTE; + } throw new IllegalArgumentException("String " + str + " isn't a valid boolean value (!str.equalsIgnoreCase(\"true\") && !str.equalsIgnoreCase(\"false\"))"); } @@ -271,6 +294,9 @@ Object convertStringLiteral(String str) { @Override Object convertStringLiteral(String str) { if (str.length() > 1) { + if ("null".equals(str) || "NULL_CHAR".equals(str)) { + return QueryConstants.NULL_CHAR; + } // TODO: #1517 Allow escaping of chars if (str.length() == 3 && ((str.charAt(0) == '\'' && str.charAt(2) == '\'') || (str.charAt(0) == '"' && str.charAt(2) == '"'))) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ReverseLookupColumnSource.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ReverseLookupColumnSource.java deleted file mode 100644 index 4c6c7668bf6..00000000000 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/ReverseLookupColumnSource.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.engine.table.impl.sources; - -import io.deephaven.stringset.LongBitmapStringSet; -import io.deephaven.engine.table.ColumnSource; - -import java.util.function.ToIntFunction; - -/** - * Common interface for column sources that provide a reverse-lookup function (value to int key). Note that int keys are - * used because this is intended for column sources with a small, contiguous key range starting from 0 and well shorter - * than Integer.MAX_VALUE. - */ -public interface ReverseLookupColumnSource extends ColumnSource, - LongBitmapStringSet.ReversibleLookup { - /** - * Get a reverse-lookup function for all non-null values stored in this column source at - * {@code keys <= highestKeyNeeded}. - * - * @param highestKeyNeeded The highest key needed in the result map - * @return A reverse-lookup function that has all values defined for keys in [0, highestKeyNeeded] - */ - ToIntFunction getReverseLookup(final int highestKeyNeeded); - - /** - * Get an implementation-defined "extra value" associated with this column source. - */ - EXTRA_VALUE_TYPE getExtra(); - - /** - * Perform a reverse lookup - * - * @param highestIndex The highest key needed for the lookup - * @param value The value we are looking up - * @return The key, between 0 and highestIndex, for the value. A value outside this range if the value has no - * mapping in the range. - */ - - default int rget(int highestIndex, DATA_TYPE value) { - return getReverseLookup(highestIndex).applyAsInt(value); - } -} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegion.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegion.java index 718b9056bfd..34cfe1b6685 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegion.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegion.java @@ -3,15 +3,25 @@ */ package io.deephaven.engine.table.impl.sources.regioned; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.Releasable; import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.WritableChunk; import io.deephaven.engine.page.Page; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; public interface ColumnRegion extends Page, Releasable { + // TODO NATE NOCOMMIT MAYBE THIS SHOULD BE NULLABLE to better support null regions (and make them singletons again) + // javadoc why / when nullable + @Nullable + ColumnLocation getLocation(); + @Override @FinalDefault default long firstRowOffset() { @@ -23,6 +33,36 @@ default long firstRowOffset() { */ void invalidate(); + /** + * Check if this region has matching features. + * + * @return true if this has matching features + */ + default boolean supportsMatching() { + return false; + } + + // Should this really return a writable row set/ + /** + * If {@link #supportsMatching()} is {@code true} this method will perform matching in lieu of the standard + * {@link RegionedColumnSourceBase#match(boolean, boolean, boolean, RowSet, Object...)}. + * + * @param invertMatch if the match should be inverted + * @param usePrev if the match should use the previous value + * @param caseInsensitive if the match is case insensitive + * @param rowSequence relevant rows to search + * @param sortedKeys the keys to search for in sorted order + * @return an index of matched rows + */ + default WritableRowSet match( + boolean invertMatch, + boolean usePrev, + boolean caseInsensitive, + @NotNull RowSequence rowSequence, + Object... sortedKeys) { + throw new UnsupportedOperationException("This region does not support matching"); + } + abstract class Null extends GenericColumnRegionBase implements ColumnRegion, WithDefaultsForRepeatingValues { @@ -39,5 +79,23 @@ public void fillChunkAppend(@NotNull final FillContext context, destination.fillWithNullValue(offset, length); destination.setSize(offset + length); } + + @Override + public final ColumnLocation getLocation() { + return null; + } + + @Override + public final boolean supportsMatching() { + return true; + } + + @Override + public abstract WritableRowSet match( + boolean invertMatch, + boolean usePrev, + boolean caseInsensitive, + @NotNull RowSequence rowSequence, + Object... sortedKeys); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionByte.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionByte.java index a8f00446713..0cbe37b4297 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionByte.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionByte.java @@ -3,9 +3,14 @@ */ package io.deephaven.engine.table.impl.sources.regioned; -import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; @@ -82,6 +87,23 @@ public byte[] getBytes(final long firstElementIndex, @NotNull final byte[] desti Arrays.fill(destination, destinationOffset, destinationOffset + length, QueryConstants.NULL_BYTE); return destination; } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + final boolean nullMatched = sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0].equals(QueryConstants.NULL_BYTE_BOXED)); + if (nullMatched && !invertMatch || !nullMatched && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + return RowSetFactory.empty(); + } } final class Constant @@ -112,14 +134,52 @@ public byte[] getBytes(final long firstElementIndex, @NotNull final byte[] desti Arrays.fill(destination, destinationOffset, destinationOffset + length, value); return destination; } + + @Override + public ColumnLocation getLocation() { + return null; + } + + @Override + public boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + boolean valueMatches = arrayContainsValue(sortedKeys); + if (valueMatches && !invertMatch || !valueMatches && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + + return RowSetFactory.empty(); + } + + private boolean arrayContainsValue(final Object[] sortedKeys) { + if (value == QueryConstants.NULL_BYTE && sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0] == QueryConstants.NULL_BYTE_BOXED)) { + return true; + } + return Arrays.binarySearch(sortedKeys, value) >= 0; + } } final class StaticPageStore extends RegionedPageStore.Static> implements ColumnRegionByte { - public StaticPageStore(@NotNull final Parameters parameters, @NotNull final ColumnRegionByte[] regions) { - super(parameters, regions); + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionByte[] regions, + @NotNull final ColumnLocation location) { + super(parameters, regions, location); } @Override @@ -143,5 +203,15 @@ public byte getByte(@NotNull final FillContext context, final long elementIndex) public byte[] getBytes(final long firstElementIndex, @NotNull final byte[] destination, final int destinationOffset, final int length) { return lookupRegion(firstElementIndex).getBytes(firstElementIndex, destination, destinationOffset, length); } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionChar.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionChar.java index f990541652e..70f255923be 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionChar.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionChar.java @@ -3,13 +3,20 @@ */ package io.deephaven.engine.table.impl.sources.regioned; -import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; +import java.util.Arrays; + /** * Column region interface for regions that support fetching primitive chars. */ @@ -58,6 +65,23 @@ private Null(final long pageMask) { public char getChar(final long elementIndex) { return QueryConstants.NULL_CHAR; } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + final boolean nullMatched = sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0].equals(QueryConstants.NULL_CHAR_BOXED)); + if (nullMatched && !invertMatch || !nullMatched && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + return RowSetFactory.empty(); + } } final class Constant @@ -82,14 +106,52 @@ public void fillChunkAppend(@NotNull final FillContext context, @NotNull final W destination.asWritableCharChunk().fillWithValue(offset, length, value); destination.setSize(offset + length); } + + @Override + public ColumnLocation getLocation() { + return null; + } + + @Override + public boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + boolean valueMatches = arrayContainsValue(sortedKeys); + if (valueMatches && !invertMatch || !valueMatches && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + + return RowSetFactory.empty(); + } + + private boolean arrayContainsValue(final Object[] sortedKeys) { + if (value == QueryConstants.NULL_CHAR && sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0] == QueryConstants.NULL_CHAR_BOXED)) { + return true; + } + return Arrays.binarySearch(sortedKeys, value) >= 0; + } } final class StaticPageStore extends RegionedPageStore.Static> implements ColumnRegionChar { - public StaticPageStore(@NotNull final Parameters parameters, @NotNull final ColumnRegionChar[] regions) { - super(parameters, regions); + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionChar[] regions, + @NotNull final ColumnLocation location) { + super(parameters, regions, location); } @Override @@ -108,5 +170,15 @@ public char getChar(final long elementIndex) { public char getChar(@NotNull final FillContext context, final long elementIndex) { return lookupRegion(elementIndex).getChar(context, elementIndex); } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionChunkDictionary.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionChunkDictionary.java index d31b33db4c1..ab2fc32961b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionChunkDictionary.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionChunkDictionary.java @@ -4,9 +4,15 @@ package io.deephaven.engine.table.impl.sources.regioned; import io.deephaven.base.string.cache.StringCache; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.ChunkSource; import io.deephaven.engine.table.impl.DefaultChunkSource; +import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; +import io.deephaven.engine.table.impl.chunkfilter.ChunkMatchFilterFactory; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.util.string.StringUtils; -import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.*; import io.deephaven.engine.page.Page; import io.deephaven.engine.rowset.RowSequence; @@ -28,11 +34,14 @@ public class ColumnRegionChunkDictionary private final Supplier> dictionaryChunkSupplier; private final Function conversion; + @NotNull + private final ColumnLocation location; public static ColumnRegionObject create( final long pageMask, @NotNull final Class dataType, - @NotNull final Supplier> dictionaryChunkSupplier) { + @NotNull final Supplier> dictionaryChunkSupplier, + @NotNull final ColumnLocation location) { if (CharSequence.class.isAssignableFrom(dataType)) { // noinspection unchecked final StringCache stringCache = @@ -40,18 +49,21 @@ public static ColumnRegionObject // noinspection unchecked final Function conversion = (final String dictValue) -> (DATA_TYPE) stringCache.getCachedString(dictValue); - return new ColumnRegionChunkDictionary<>(pageMask, dictionaryChunkSupplier, conversion); + return new ColumnRegionChunkDictionary<>(pageMask, dictionaryChunkSupplier, conversion, location); } return new ColumnRegionChunkDictionary<>(pageMask, dictionaryChunkSupplier, - Function.identity()); + Function.identity(), location); } - private ColumnRegionChunkDictionary(final long pageMask, + private ColumnRegionChunkDictionary( + final long pageMask, @NotNull final Supplier> dictionaryChunkSupplier, - @NotNull final Function conversion) { + @NotNull final Function conversion, + @NotNull final ColumnLocation location) { super(pageMask); this.dictionaryChunkSupplier = dictionaryChunkSupplier; this.conversion = conversion; + this.location = location; } private ObjectChunk getDictionaryChunk() { @@ -102,4 +114,29 @@ public boolean gatherDictionaryValuesRowSet( } return advanceToNextPage(keysToVisit); } + + @Override + public ColumnLocation getLocation() { + return location; + } + + @Override + public boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // TODO NATE NOCOMMIT - needs getNativeType (maybe?) pushdown instead? + try (RowSet selection = rowSequence.asRowSet()) { + // noinspection unchecked + return ChunkFilter.applyChunkFilter(selection, (ChunkSource) this, false, + ChunkMatchFilterFactory.getChunkFilter(Object.class, caseInsensitive, invertMatch, sortedKeys)); + } + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionDouble.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionDouble.java index a3445cf2a89..0476bb79ade 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionDouble.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionDouble.java @@ -8,13 +8,20 @@ */ package io.deephaven.engine.table.impl.sources.regioned; -import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; +import java.util.Arrays; + /** * Column region interface for regions that support fetching primitive doubles. */ @@ -63,6 +70,23 @@ private Null(final long pageMask) { public double getDouble(final long elementIndex) { return QueryConstants.NULL_DOUBLE; } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + final boolean nullMatched = sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0].equals(QueryConstants.NULL_DOUBLE_BOXED)); + if (nullMatched && !invertMatch || !nullMatched && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + return RowSetFactory.empty(); + } } final class Constant @@ -87,14 +111,52 @@ public void fillChunkAppend(@NotNull final FillContext context, @NotNull final W destination.asWritableDoubleChunk().fillWithValue(offset, length, value); destination.setSize(offset + length); } + + @Override + public ColumnLocation getLocation() { + return null; + } + + @Override + public boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + boolean valueMatches = arrayContainsValue(sortedKeys); + if (valueMatches && !invertMatch || !valueMatches && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + + return RowSetFactory.empty(); + } + + private boolean arrayContainsValue(final Object[] sortedKeys) { + if (value == QueryConstants.NULL_DOUBLE && sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0] == QueryConstants.NULL_DOUBLE_BOXED)) { + return true; + } + return Arrays.binarySearch(sortedKeys, value) >= 0; + } } final class StaticPageStore extends RegionedPageStore.Static> implements ColumnRegionDouble { - public StaticPageStore(@NotNull final Parameters parameters, @NotNull final ColumnRegionDouble[] regions) { - super(parameters, regions); + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionDouble[] regions, + @NotNull final ColumnLocation location) { + super(parameters, regions, location); } @Override @@ -113,5 +175,15 @@ public double getDouble(final long elementIndex) { public double getDouble(@NotNull final FillContext context, final long elementIndex) { return lookupRegion(elementIndex).getDouble(context, elementIndex); } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionFloat.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionFloat.java index b8cc6489044..e0eb02b4a9a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionFloat.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionFloat.java @@ -8,13 +8,20 @@ */ package io.deephaven.engine.table.impl.sources.regioned; -import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; +import java.util.Arrays; + /** * Column region interface for regions that support fetching primitive floats. */ @@ -63,6 +70,23 @@ private Null(final long pageMask) { public float getFloat(final long elementIndex) { return QueryConstants.NULL_FLOAT; } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + final boolean nullMatched = sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0].equals(QueryConstants.NULL_FLOAT_BOXED)); + if (nullMatched && !invertMatch || !nullMatched && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + return RowSetFactory.empty(); + } } final class Constant @@ -87,14 +111,52 @@ public void fillChunkAppend(@NotNull final FillContext context, @NotNull final W destination.asWritableFloatChunk().fillWithValue(offset, length, value); destination.setSize(offset + length); } + + @Override + public ColumnLocation getLocation() { + return null; + } + + @Override + public boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + boolean valueMatches = arrayContainsValue(sortedKeys); + if (valueMatches && !invertMatch || !valueMatches && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + + return RowSetFactory.empty(); + } + + private boolean arrayContainsValue(final Object[] sortedKeys) { + if (value == QueryConstants.NULL_FLOAT && sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0] == QueryConstants.NULL_FLOAT_BOXED)) { + return true; + } + return Arrays.binarySearch(sortedKeys, value) >= 0; + } } final class StaticPageStore extends RegionedPageStore.Static> implements ColumnRegionFloat { - public StaticPageStore(@NotNull final Parameters parameters, @NotNull final ColumnRegionFloat[] regions) { - super(parameters, regions); + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionFloat[] regions, + @NotNull final ColumnLocation location) { + super(parameters, regions, location); } @Override @@ -113,5 +175,15 @@ public float getFloat(final long elementIndex) { public float getFloat(@NotNull final FillContext context, final long elementIndex) { return lookupRegion(elementIndex).getFloat(context, elementIndex); } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionInt.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionInt.java index 44e7863f418..b0d440a3bbb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionInt.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionInt.java @@ -8,13 +8,20 @@ */ package io.deephaven.engine.table.impl.sources.regioned; -import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; +import java.util.Arrays; + /** * Column region interface for regions that support fetching primitive ints. */ @@ -63,6 +70,23 @@ private Null(final long pageMask) { public int getInt(final long elementIndex) { return QueryConstants.NULL_INT; } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + final boolean nullMatched = sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0].equals(QueryConstants.NULL_INT_BOXED)); + if (nullMatched && !invertMatch || !nullMatched && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + return RowSetFactory.empty(); + } } final class Constant @@ -87,14 +111,52 @@ public void fillChunkAppend(@NotNull final FillContext context, @NotNull final W destination.asWritableIntChunk().fillWithValue(offset, length, value); destination.setSize(offset + length); } + + @Override + public ColumnLocation getLocation() { + return null; + } + + @Override + public boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + boolean valueMatches = arrayContainsValue(sortedKeys); + if (valueMatches && !invertMatch || !valueMatches && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + + return RowSetFactory.empty(); + } + + private boolean arrayContainsValue(final Object[] sortedKeys) { + if (value == QueryConstants.NULL_INT && sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0] == QueryConstants.NULL_INT_BOXED)) { + return true; + } + return Arrays.binarySearch(sortedKeys, value) >= 0; + } } final class StaticPageStore extends RegionedPageStore.Static> implements ColumnRegionInt { - public StaticPageStore(@NotNull final Parameters parameters, @NotNull final ColumnRegionInt[] regions) { - super(parameters, regions); + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionInt[] regions, + @NotNull final ColumnLocation location) { + super(parameters, regions, location); } @Override @@ -113,5 +175,15 @@ public int getInt(final long elementIndex) { public int getInt(@NotNull final FillContext context, final long elementIndex) { return lookupRegion(elementIndex).getInt(context, elementIndex); } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionLong.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionLong.java index a1cf5ee8856..3ca560ade8a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionLong.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionLong.java @@ -8,13 +8,20 @@ */ package io.deephaven.engine.table.impl.sources.regioned; -import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; +import java.util.Arrays; + /** * Column region interface for regions that support fetching primitive longs. */ @@ -63,6 +70,23 @@ private Null(final long pageMask) { public long getLong(final long elementIndex) { return QueryConstants.NULL_LONG; } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + final boolean nullMatched = sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0].equals(QueryConstants.NULL_LONG_BOXED)); + if (nullMatched && !invertMatch || !nullMatched && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + return RowSetFactory.empty(); + } } final class Constant @@ -87,14 +111,52 @@ public void fillChunkAppend(@NotNull final FillContext context, @NotNull final W destination.asWritableLongChunk().fillWithValue(offset, length, value); destination.setSize(offset + length); } + + @Override + public ColumnLocation getLocation() { + return null; + } + + @Override + public boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + boolean valueMatches = arrayContainsValue(sortedKeys); + if (valueMatches && !invertMatch || !valueMatches && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + + return RowSetFactory.empty(); + } + + private boolean arrayContainsValue(final Object[] sortedKeys) { + if (value == QueryConstants.NULL_LONG && sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0] == QueryConstants.NULL_LONG_BOXED)) { + return true; + } + return Arrays.binarySearch(sortedKeys, value) >= 0; + } } final class StaticPageStore extends RegionedPageStore.Static> implements ColumnRegionLong { - public StaticPageStore(@NotNull final Parameters parameters, @NotNull final ColumnRegionLong[] regions) { - super(parameters, regions); + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionLong[] regions, + @NotNull final ColumnLocation location) { + super(parameters, regions, location); } @Override @@ -113,5 +175,15 @@ public long getLong(final long elementIndex) { public long getLong(@NotNull final FillContext context, final long elementIndex) { return lookupRegion(elementIndex).getLong(context, elementIndex); } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionObject.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionObject.java index 606431ae788..083ed278b2c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionObject.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionObject.java @@ -4,17 +4,17 @@ package io.deephaven.engine.table.impl.sources.regioned; import io.deephaven.chunk.attributes.Any; +import io.deephaven.engine.rowset.*; import io.deephaven.engine.table.impl.chunkattributes.DictionaryKeys; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.WritableLongChunk; import io.deephaven.engine.page.Page; -import io.deephaven.engine.rowset.RowSequence; -import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.rowset.RowSetBuilderSequential; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; +import java.util.Arrays; import java.util.stream.IntStream; import static io.deephaven.util.QueryConstants.NULL_LONG; @@ -161,6 +161,22 @@ public boolean gatherDictionaryValuesRowSet(@NotNull final RowSet.SearchIterator public ColumnRegionLong getDictionaryKeysRegion() { return dictionaryKeysRegion == null ? dictionaryKeysRegion = ColumnRegionLong.createNull(mask()) : dictionaryKeysRegion; } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + final boolean nullMatched = sortedKeys.length > 0 && sortedKeys[0] == null; + if (nullMatched && !invertMatch || !nullMatched && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + return RowSetFactory.empty(); + } } static ColumnRegionLong createConstantDictionaryKeysRegion(final long pageMask) { @@ -212,6 +228,40 @@ public boolean gatherDictionaryValuesRowSet(@NotNull final RowSet.SearchIterator public ColumnRegionLong getDictionaryKeysRegion() { return dictionaryKeysRegion == null ? dictionaryKeysRegion = createConstantDictionaryKeysRegion(mask()) : dictionaryKeysRegion; } + + @Override + public ColumnLocation getLocation() { + return null; + } + + @Override + public boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + final boolean valueMatches = arrayContainsValue(sortedKeys); + if (valueMatches && !invertMatch || !valueMatches && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + + return RowSetFactory.empty(); + } + + private boolean arrayContainsValue(final Object[] sortedKeys) { + if (value == null && sortedKeys.length > 0 && sortedKeys[0] == null) { + return true; + } + return Arrays.asList(sortedKeys).contains(value); + } } final class StaticPageStore @@ -221,8 +271,11 @@ final class StaticPageStore private ColumnRegionLong dictionaryKeysRegion; private ColumnRegionObject dictionaryValuesRegion; - public StaticPageStore(@NotNull final Parameters parameters, @NotNull final ColumnRegionObject[] regions) { - super(parameters, regions); + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionObject[] regions, + @NotNull final ColumnLocation location) { + super(parameters, regions, location); } @Override @@ -267,14 +320,16 @@ public boolean gatherDictionaryValuesRowSet(@NotNull final RowSet.SearchIterator @Override public ColumnRegionLong getDictionaryKeysRegion() { return dictionaryKeysRegion == null - ? dictionaryKeysRegion = new ColumnRegionLong.StaticPageStore<>(parameters(), mapRegionsToDictionaryKeys()) + ? dictionaryKeysRegion = new ColumnRegionLong.StaticPageStore<>( + parameters(), mapRegionsToDictionaryKeys(), getLocation()) : dictionaryKeysRegion; } @Override public ColumnRegionObject getDictionaryValuesRegion() { return dictionaryValuesRegion == null - ? dictionaryValuesRegion = new ColumnRegionObject.StaticPageStore<>(parameters(), mapRegionsToDictionaryValues()) + ? dictionaryValuesRegion = new ColumnRegionObject.StaticPageStore<>( + parameters(), mapRegionsToDictionaryValues(), getLocation()) : dictionaryValuesRegion; } @@ -291,6 +346,16 @@ private ColumnRegionObject[] mapRegionsToDictionaryValues() { .mapToObj(ri -> getRegion(ri).getDictionaryValuesRegion()) .toArray(ColumnRegionObject[]::new); } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } final class DictionaryKeysWrapper implements ColumnRegionLong, Page.WithDefaults { @@ -350,5 +415,25 @@ public void fillChunkAppend(@NotNull final FillContext context, @NotNull final W } } } + + @Override + public boolean supportsMatching() { + return wrapped.supportsMatching(); + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + return wrapped.match(invertMatch, usePrev, caseInsensitive, rowSequence, sortedKeys); + } + + @Override + public ColumnLocation getLocation() { + return wrapped.getLocation(); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionReferencing.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionReferencing.java index 688b52ca829..9b2063f6eca 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionReferencing.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionReferencing.java @@ -8,6 +8,7 @@ import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; @@ -33,6 +34,15 @@ void convertRegion( @NotNull WritableChunk destination, @NotNull Chunk source, @NotNull RowSequence rowSequence); + + /** + * Convert an array of values in the region type to an array of values in boxed native format. + * + * @param values the values to convert + * @return a new array of converted values + */ + @NotNull + Object[] convertArray(@NotNull Object[] values); } class Null> @@ -51,5 +61,15 @@ public Null(REFERENCED_COLUMN_REGION nullReferencedColumnRegion) { public REFERENCED_COLUMN_REGION getReferencedRegion() { return nullReferencedColumnRegion; } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + return nullReferencedColumnRegion.match(invertMatch, usePrev, caseInsensitive, rowSequence, sortedKeys); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionReferencingImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionReferencingImpl.java index 4b8b10bbbcd..ba927e31ee7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionReferencingImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionReferencingImpl.java @@ -4,13 +4,20 @@ package io.deephaven.engine.table.impl.sources.regioned; import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ChunkSource; import io.deephaven.engine.table.GetContextMaker; import io.deephaven.engine.table.SharedContext; import io.deephaven.chunk.WritableChunk; import io.deephaven.engine.page.Page; import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; +import io.deephaven.engine.table.impl.chunkfilter.ChunkMatchFilterFactory; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import javax.annotation.OverridingMethodsMustInvokeSuper; @@ -18,9 +25,14 @@ public class ColumnRegionReferencingImpl, Page.WithDefaults { private final REFERENCED_COLUMN_REGION referencedColumnRegion; + @NotNull + private final Converter converter; - public ColumnRegionReferencingImpl(@NotNull final REFERENCED_COLUMN_REGION referencedColumnRegion) { + public ColumnRegionReferencingImpl( + @NotNull final REFERENCED_COLUMN_REGION referencedColumnRegion, + @NotNull final Converter converter) { this.referencedColumnRegion = referencedColumnRegion; + this.converter = converter; } @Override @@ -42,10 +54,16 @@ public long mask() { @Override public void fillChunkAppend(@NotNull ChunkSource.FillContext context, @NotNull WritableChunk destination, @NotNull RowSequence rowSequence) { - FillContext.converter(context).convertRegion(destination, + converter.convertRegion(destination, referencedColumnRegion.getChunk(FillContext.nativeGetContext(context), rowSequence), rowSequence); } + @Override + public ChunkSource.FillContext makeFillContext( + final int chunkCapacity, @Nullable final SharedContext sharedContext) { + return new FillContext(referencedColumnRegion, chunkCapacity, sharedContext); + } + @Override @OverridingMethodsMustInvokeSuper public void releaseCachedResources() { @@ -53,25 +71,48 @@ public void releaseCachedResources() { referencedColumnRegion.releaseCachedResources(); } + @Override + public ColumnLocation getLocation() { + return referencedColumnRegion.getLocation(); + } + + @Override + public boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + final Object[] nativeKeys = converter.convertArray(sortedKeys); + try (final RowSet rows = rowSequence.asRowSet()) { + // TODO NATE NOCOMMIT: getNativeType? or pushdown? + // noinspection unchecked + return ChunkFilter.applyChunkFilter(rows, (ChunkSource) referencedColumnRegion, false, + ChunkMatchFilterFactory.getChunkFilter( + null, caseInsensitive, invertMatch, nativeKeys)); + // referencedColumnRegion.getNativeType(), caseInsensitive, invertMatch, nativeKeys)); + } + } + static class FillContext implements ChunkSource.FillContext { private final ChunkSource.GetContext nativeGetContext; - private final Converter converter; - FillContext(GetContextMaker getContextMaker, Converter converter, int chunkCapacity, - SharedContext sharedContext) { - this.converter = converter; + FillContext( + @NotNull final GetContextMaker getContextMaker, + int chunkCapacity, + @Nullable final SharedContext sharedContext) { this.nativeGetContext = getContextMaker.makeGetContext(chunkCapacity, sharedContext); } - static ChunkSource.GetContext nativeGetContext(ChunkSource.FillContext context) { + static ChunkSource.GetContext nativeGetContext(@NotNull final ChunkSource.FillContext context) { return ((FillContext) context).nativeGetContext; } - static Converter converter(ChunkSource.FillContext context) { - // noinspection unchecked - return ((FillContext) context).converter; - } - @Override public void close() { nativeGetContext.close(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionShort.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionShort.java index 588f34eeb7c..fa634b528df 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionShort.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/ColumnRegionShort.java @@ -8,13 +8,20 @@ */ package io.deephaven.engine.table.impl.sources.regioned; -import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; +import java.util.Arrays; + /** * Column region interface for regions that support fetching primitive shorts. */ @@ -63,6 +70,23 @@ private Null(final long pageMask) { public short getShort(final long elementIndex) { return QueryConstants.NULL_SHORT; } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + final boolean nullMatched = sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0].equals(QueryConstants.NULL_SHORT_BOXED)); + if (nullMatched && !invertMatch || !nullMatched && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + return RowSetFactory.empty(); + } } final class Constant @@ -87,14 +111,52 @@ public void fillChunkAppend(@NotNull final FillContext context, @NotNull final W destination.asWritableShortChunk().fillWithValue(offset, length, value); destination.setSize(offset + length); } + + @Override + public ColumnLocation getLocation() { + return null; + } + + @Override + public boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + boolean valueMatches = arrayContainsValue(sortedKeys); + if (valueMatches && !invertMatch || !valueMatches && invertMatch) { + try (final RowSet rowSet = rowSequence.asRowSet()) { + return rowSet.copy(); + } + } + + return RowSetFactory.empty(); + } + + private boolean arrayContainsValue(final Object[] sortedKeys) { + if (value == QueryConstants.NULL_SHORT && sortedKeys.length > 0 + && (sortedKeys[0] == null || sortedKeys[0] == QueryConstants.NULL_SHORT_BOXED)) { + return true; + } + return Arrays.binarySearch(sortedKeys, value) >= 0; + } } final class StaticPageStore extends RegionedPageStore.Static> implements ColumnRegionShort { - public StaticPageStore(@NotNull final Parameters parameters, @NotNull final ColumnRegionShort[] regions) { - super(parameters, regions); + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionShort[] regions, + @NotNull final ColumnLocation location) { + super(parameters, regions, location); } @Override @@ -113,5 +175,15 @@ public short getShort(final long elementIndex) { public short getShort(@NotNull final FillContext context, final long elementIndex) { return lookupRegion(elementIndex).getShort(context, elementIndex); } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/DeferredColumnRegionBase.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/DeferredColumnRegionBase.java index 37258661c0e..cd35f0900df 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/DeferredColumnRegionBase.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/DeferredColumnRegionBase.java @@ -5,9 +5,11 @@ import io.deephaven.base.verify.Require; import io.deephaven.chunk.attributes.Any; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.SharedContext; import io.deephaven.chunk.*; import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import org.jetbrains.annotations.NotNull; import javax.annotation.OverridingMethodsMustInvokeSuper; @@ -77,8 +79,10 @@ public ChunkType getChunkType() { } @Override - public void fillChunk(@NotNull FillContext context, @NotNull WritableChunk destination, - @NotNull RowSequence rowSequence) { + public void fillChunk( + @NotNull final FillContext context, + @NotNull final WritableChunk destination, + @NotNull final RowSequence rowSequence) { getResultRegion().fillChunk(context, destination, rowSequence); } @@ -91,22 +95,42 @@ public void fillChunkAppend( } @Override - public Chunk getChunk(@NotNull GetContext context, @NotNull RowSequence rowSequence) { + public Chunk getChunk(@NotNull final GetContext context, @NotNull final RowSequence rowSequence) { return getResultRegion().getChunk(context, rowSequence); } @Override - public Chunk getChunk(@NotNull GetContext context, long firstKey, long lastKey) { + public Chunk getChunk(@NotNull final GetContext context, final long firstKey, final long lastKey) { return getResultRegion().getChunk(context, firstKey, lastKey); } @Override - public FillContext makeFillContext(int chunkCapacity, SharedContext sharedContext) { + public FillContext makeFillContext(final int chunkCapacity, final SharedContext sharedContext) { return getResultRegion().makeFillContext(chunkCapacity, sharedContext); } @Override - public GetContext makeGetContext(int chunkCapacity, SharedContext sharedContext) { + public GetContext makeGetContext(final int chunkCapacity, final SharedContext sharedContext) { return getResultRegion().makeGetContext(chunkCapacity, sharedContext); } + + @Override + public ColumnLocation getLocation() { + return getResultRegion().getLocation(); + } + + @Override + public boolean supportsMatching() { + return getResultRegion().supportsMatching(); + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + return getResultRegion().match(invertMatch, usePrev, caseInsensitive, rowSequence, sortedKeys); + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/DeferredColumnRegionReferencing.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/DeferredColumnRegionReferencing.java deleted file mode 100644 index 0ea34389d48..00000000000 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/DeferredColumnRegionReferencing.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.engine.table.impl.sources.regioned; - -import io.deephaven.chunk.attributes.Any; -import org.jetbrains.annotations.NotNull; - -import java.util.function.Supplier; - -/** - * {@link ColumnRegionReferencing} implementation for deferred regions, i.e. regions that will be properly constructed - * on first access. - */ -public class DeferredColumnRegionReferencing> - extends DeferredColumnRegionBase> - implements ColumnRegionReferencing { - - DeferredColumnRegionReferencing(final long pageMask, - @NotNull Supplier> resultRegionFactory) { - super(pageMask, resultRegionFactory); - } - - @NotNull - @Override - public REFERENCED_COLUMN_REGION getReferencedRegion() { - return getResultRegion().getReferencedRegion(); - } - -} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/MakeRegion.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/MakeRegion.java index bf7683325f4..45a64edb5f3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/MakeRegion.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/MakeRegion.java @@ -6,6 +6,7 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.impl.locations.ColumnLocation; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -17,6 +18,7 @@ public interface MakeRegion columnDefinition, + REGION_TYPE makeRegion( + @NotNull ColumnDefinition columnDefinition, @NotNull ColumnLocation columnLocation, + @NotNull SourceTableColumnInstructions instructions, int regionIndex); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSource.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSource.java index ce0afa78d5f..f76b73715a7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSource.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSource.java @@ -9,6 +9,8 @@ import io.deephaven.engine.table.impl.sources.DeferredGroupingColumnSource; import io.deephaven.engine.table.impl.ImmutableColumnSource; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.TrackingWritableRowSet; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import io.deephaven.util.annotations.VisibleForTesting; import org.jetbrains.annotations.NotNull; @@ -106,10 +108,13 @@ static long getRowKey(final int regionIndex, final long regionOffset) { * * @param columnDefinition The column definition for this column source (potentially varies by region) * @param columnLocation The column location for the region being added + * @param instructions The instructions for the region being added * @return The index assigned to the added region */ - int addRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation); + int addRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions); /** * Invalidate the specified region. An invalidated region will throw an exception on any read attempt if it cannot @@ -118,4 +123,9 @@ int addRegion(@NotNull final ColumnDefinition columnDefinition, * @param regionIndex the region to invalidate */ void invalidateRegion(int regionIndex); + + /** + * Disable the use of grouping for any operation of this source. + */ + void disableGrouping(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceArray.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceArray.java index b43d4fdbfb6..cbbaf71112c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceArray.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceArray.java @@ -10,6 +10,7 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.impl.locations.ColumnLocation; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import io.deephaven.util.annotations.TestUseOnly; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -80,19 +81,21 @@ private static > REG @Override @OverridingMethodsMustInvokeSuper - public synchronized int addRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation) { + public synchronized int addRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions) { maybeExtendRegions(); final int regionIndex = regionCount; - regions[regionIndex] = makeDeferred.make(PARAMETERS.regionMask, - () -> updateRegion(regionIndex, makeRegion(columnDefinition, columnLocation, regionIndex))); + regions[regionIndex] = makeDeferred.make(PARAMETERS.regionMask, () -> updateRegion(regionIndex, + makeRegion(columnDefinition, columnLocation, instructions, regionIndex))); return regionCount++; } /** *

* Add a pre-constructed region without going through the abstract factory method - * {@link #makeRegion(ColumnDefinition, ColumnLocation, int)}. + * {@link #makeRegion(ColumnDefinition, ColumnLocation, SourceTableColumnInstructions, int)}. *

* This method is for unit testing purposes only! * diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceBase.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceBase.java index 7ac1e19e170..a9205fc9e7c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceBase.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceBase.java @@ -5,19 +5,43 @@ import io.deephaven.base.verify.Assert; import io.deephaven.chunk.attributes.Values; +import io.deephaven.configuration.Configuration; +import io.deephaven.engine.rowset.*; import io.deephaven.engine.table.impl.sources.AbstractDeferredGroupingColumnSource; import io.deephaven.chunk.WritableChunk; -import io.deephaven.engine.rowset.RowSequence; import io.deephaven.util.annotations.TestUseOnly; +import io.deephaven.util.compare.ObjectComparisons; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + /** * Partial implementation of {@link RegionedColumnSource} for array-backed and delegating implementations to extend. */ -abstract class RegionedColumnSourceBase> +public abstract class RegionedColumnSourceBase> extends AbstractDeferredGroupingColumnSource implements RegionedPageStore, RegionedColumnSource { + public static boolean USE_PARALLEL = Configuration.getInstance().getBooleanForClassWithDefault( + RegionedColumnSourceBase.class, "useParallelMatchFilter", true); + public static boolean USE_PUSHDOWN = Configuration.getInstance().getBooleanForClassWithDefault( + RegionedColumnSourceBase.class, "pushDownMatchFilter", true); + + private static class MatchParams { + final ColumnRegion region; + final RowSequence relevantKeys; + final long regionFirstkey; + + private MatchParams(ColumnRegion region, RowSequence relevantKeys, long regionFirstKey) { + this.region = region; + this.relevantKeys = relevantKeys; + this.regionFirstkey = regionFirstKey; + } + } + + private volatile boolean groupingEnabled = true; static final Parameters PARAMETERS; static { @@ -42,6 +66,11 @@ public void invalidateRegion(final int regionIndex) { getRegion(regionIndex).invalidate(); } + @Override + public void disableGrouping() { + groupingEnabled = false; + } + @Override public final Parameters parameters() { return PARAMETERS; @@ -83,4 +112,108 @@ public void fillPrevChunk(@NotNull FillContext context, */ @NotNull abstract REGION_TYPE getNullRegion(); + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSet rowSet, + final Object... keys) { + if (!USE_PUSHDOWN) { + return super.match(invertMatch, usePrev, caseInsensitive, rowSet, keys); + } + + Arrays.sort(keys, ObjectComparisons::compare); + WritableRowSet result; + if (!USE_PARALLEL) { + result = doSequentialMatch(invertMatch, caseInsensitive, rowSet, keys); + } else { + result = doParallelMatch(invertMatch, caseInsensitive, rowSet, keys); + } + + result.retain(rowSet); + return result; + } + + private WritableRowSet doParallelMatch( + final boolean invertMatch, + final boolean caseInsensitive, + @NotNull final RowSet rowSet, + final Object... keys) { + // First find all the relevant regions for this set + final List relevantRegions = new ArrayList<>(); + try (final RowSequence.Iterator rowSequenceIterator = rowSet.getRowSequenceIterator()) { + for (int regionIdx = 0; regionIdx < getRegionCount(); regionIdx++) { + final ColumnRegion region = getRegion(regionIdx); + final long firstRegionIndex = region.firstRow(regionIdx); + final long lastRegionIndex = region.maxRow(regionIdx); + if (rowSequenceIterator.advance(firstRegionIndex) + && rowSequenceIterator.peekNextKey() <= lastRegionIndex) { + final RowSequence nextOrderedKeysThrough = + rowSequenceIterator.getNextRowSequenceThrough(lastRegionIndex); + relevantRegions.add(new MatchParams(region, nextOrderedKeysThrough.asRowSet(), firstRegionIndex)); + } + } + + // Now filter them all in parallel + final WritableRowSet result; + if (relevantRegions.size() == 1) { + final MatchParams p = relevantRegions.get(0); + result = matchWithinRegion(p.region, + p.relevantKeys, + caseInsensitive, + invertMatch, + keys).copy(); + } else { + // TODO (deephaven-core#3851): parallelize; use disk backed deferred grouping pool if ported + final RowSetBuilderRandom resultBuilder = RowSetFactory.builderRandom(); + relevantRegions.stream() + .map(p -> matchWithinRegion(p.region, + p.relevantKeys, + caseInsensitive, + invertMatch, + keys)) + .forEach(resultBuilder::addRowSet); + result = resultBuilder.build(); + } + return result; + } + } + + private WritableRowSet doSequentialMatch( + final boolean invertMatch, + final boolean caseInsensitive, + @NotNull final RowSet startingRowSet, + final Object... keys) { + RowSetBuilderSequential resultBuilder = RowSetFactory.builderSequential(); + try (final RowSequence.Iterator okIt = startingRowSet.getRowSequenceIterator()) { + for (int regionIdx = 0; regionIdx < getRegionCount(); regionIdx++) { + final ColumnRegion region = getRegion(regionIdx); + final long firstRegionIndex = region.firstRow(regionIdx); + final long lastRegionIndex = region.maxRow(regionIdx); + if (okIt.advance(firstRegionIndex) && okIt.peekNextKey() <= lastRegionIndex) { + final RowSequence relevantRegionRows = okIt.getNextRowSequenceThrough(lastRegionIndex); + RowSet regionMatch = matchWithinRegion(region, + relevantRegionRows, + caseInsensitive, + invertMatch, + keys); + resultBuilder.appendRowSequence(regionMatch); + } + } + } + + return resultBuilder.build(); + } + + private RowSet matchWithinRegion( + @NotNull final ColumnRegion region, + @NotNull final RowSequence rowsToFilter, + final boolean caseInsensitive, + final boolean invertMatch, + @NotNull final Object[] keys) { + // TODO: ensure keys are sorted before parllelizing + return region.match(invertMatch, false, caseInsensitive, rowsToFilter, keys); + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceBoolean.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceBoolean.java index 7aea30c3742..3ad345523b5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceBoolean.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceBoolean.java @@ -47,4 +47,16 @@ public Boolean get(long rowKey) { return rowKey == RowSequence.NULL_ROW_KEY ? null : BooleanUtils.byteAsBoolean(getNativeSource().lookupRegion(rowKey).getByte(rowKey)); } + + @Override + @NotNull + public Object[] convertArray(@NotNull final Object[] keys) { + final Object[] convertedValues = new Object[keys.length]; + for(int ii = 0; ii < keys.length; ii++) { + if(keys[ii] == null || keys[ii] instanceof Boolean) { + convertedValues[ii] = BooleanUtils.booleanAsByte((Boolean)keys[ii]); + } + } + return convertedValues; + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceByte.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceByte.java index d617073347a..e60c594afa0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceByte.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceByte.java @@ -17,6 +17,7 @@ import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.ColumnSourceGetDefaults; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.jetbrains.annotations.NotNull; import static io.deephaven.util.type.TypeUtils.unbox; @@ -40,9 +41,11 @@ public byte getByte(final long rowKey) { interface MakeRegionDefault extends MakeRegion> { @Override - default ColumnRegionByte makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + default ColumnRegionByte makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { if (columnLocation.exists()) { return columnLocation.makeColumnRegionByte(columnDefinition); } @@ -78,9 +81,11 @@ static final class Partitioning extends RegionedColumnSourceByte { } @Override - public ColumnRegionByte makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + public ColumnRegionByte makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { final TableLocationKey locationKey = columnLocation.getTableLocation().getKey(); final Object partitioningColumnValue = locationKey.getPartitionValue(columnDefinition.getName()); if (partitioningColumnValue != null && !Byte.class.isAssignableFrom(partitioningColumnValue.getClass())) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceChar.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceChar.java index 0316daefc43..e16bb491d74 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceChar.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceChar.java @@ -10,6 +10,7 @@ import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.ColumnSourceGetDefaults; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.jetbrains.annotations.NotNull; import static io.deephaven.util.type.TypeUtils.unbox; @@ -33,9 +34,11 @@ public char getChar(final long rowKey) { interface MakeRegionDefault extends MakeRegion> { @Override - default ColumnRegionChar makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + default ColumnRegionChar makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { if (columnLocation.exists()) { return columnLocation.makeColumnRegionChar(columnDefinition); } @@ -61,9 +64,11 @@ static final class Partitioning extends RegionedColumnSourceChar { } @Override - public ColumnRegionChar makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + public ColumnRegionChar makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { final TableLocationKey locationKey = columnLocation.getTableLocation().getKey(); final Object partitioningColumnValue = locationKey.getPartitionValue(columnDefinition.getName()); if (partitioningColumnValue != null && !Character.class.isAssignableFrom(partitioningColumnValue.getClass())) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceDouble.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceDouble.java index 6c50a7cc3a0..6d42301e982 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceDouble.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceDouble.java @@ -15,6 +15,7 @@ import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.ColumnSourceGetDefaults; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.jetbrains.annotations.NotNull; import static io.deephaven.util.type.TypeUtils.unbox; @@ -38,9 +39,11 @@ public double getDouble(final long rowKey) { interface MakeRegionDefault extends MakeRegion> { @Override - default ColumnRegionDouble makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + default ColumnRegionDouble makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { if (columnLocation.exists()) { return columnLocation.makeColumnRegionDouble(columnDefinition); } @@ -66,9 +69,11 @@ static final class Partitioning extends RegionedColumnSourceDouble { } @Override - public ColumnRegionDouble makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + public ColumnRegionDouble makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { final TableLocationKey locationKey = columnLocation.getTableLocation().getKey(); final Object partitioningColumnValue = locationKey.getPartitionValue(columnDefinition.getName()); if (partitioningColumnValue != null && !Double.class.isAssignableFrom(partitioningColumnValue.getClass())) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceFloat.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceFloat.java index ae8bb9184dc..d9ce04968c2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceFloat.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceFloat.java @@ -15,6 +15,7 @@ import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.ColumnSourceGetDefaults; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.jetbrains.annotations.NotNull; import static io.deephaven.util.type.TypeUtils.unbox; @@ -38,9 +39,11 @@ public float getFloat(final long rowKey) { interface MakeRegionDefault extends MakeRegion> { @Override - default ColumnRegionFloat makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + default ColumnRegionFloat makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { if (columnLocation.exists()) { return columnLocation.makeColumnRegionFloat(columnDefinition); } @@ -66,9 +69,11 @@ static final class Partitioning extends RegionedColumnSourceFloat { } @Override - public ColumnRegionFloat makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + public ColumnRegionFloat makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { final TableLocationKey locationKey = columnLocation.getTableLocation().getKey(); final Object partitioningColumnValue = locationKey.getPartitionValue(columnDefinition.getName()); if (partitioningColumnValue != null && !Float.class.isAssignableFrom(partitioningColumnValue.getClass())) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceInstant.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceInstant.java index d90c58bb209..d0126fd41f9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceInstant.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceInstant.java @@ -87,4 +87,16 @@ public ColumnSource toLocalTime(ZoneId zone) { public ColumnSource toEpochNano() { return getNativeSource(); } + + @Override + @NotNull + public Object[] convertArray(@NotNull final Object[] keys) { + final Object[] newValues = new Object[keys.length]; + for (int ii = 0; ii < keys.length; ii++) { + if (keys[ii] == null || keys[ii] instanceof Instant) { + newValues[ii] = DateTimeUtils.epochNanos((Instant) keys[ii]); + } + } + return newValues; + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceInt.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceInt.java index f07e2d05733..d11255995e5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceInt.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceInt.java @@ -15,6 +15,7 @@ import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.ColumnSourceGetDefaults; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.jetbrains.annotations.NotNull; import static io.deephaven.util.type.TypeUtils.unbox; @@ -38,9 +39,11 @@ public int getInt(final long rowKey) { interface MakeRegionDefault extends MakeRegion> { @Override - default ColumnRegionInt makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + default ColumnRegionInt makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { if (columnLocation.exists()) { return columnLocation.makeColumnRegionInt(columnDefinition); } @@ -66,9 +69,11 @@ static final class Partitioning extends RegionedColumnSourceInt { } @Override - public ColumnRegionInt makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + public ColumnRegionInt makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { final TableLocationKey locationKey = columnLocation.getTableLocation().getKey(); final Object partitioningColumnValue = locationKey.getPartitionValue(columnDefinition.getName()); if (partitioningColumnValue != null && !Integer.class.isAssignableFrom(partitioningColumnValue.getClass())) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceLong.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceLong.java index 9009869c1bb..619753914bb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceLong.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceLong.java @@ -26,6 +26,7 @@ import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.ColumnSourceGetDefaults; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.jetbrains.annotations.NotNull; import static io.deephaven.util.type.TypeUtils.unbox; @@ -49,9 +50,11 @@ public long getLong(final long rowKey) { interface MakeRegionDefault extends MakeRegion> { @Override - default ColumnRegionLong makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + default ColumnRegionLong makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { if (columnLocation.exists()) { return columnLocation.makeColumnRegionLong(columnDefinition); } @@ -126,9 +129,11 @@ static final class Partitioning extends RegionedColumnSourceLong { } @Override - public ColumnRegionLong makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + public ColumnRegionLong makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { final TableLocationKey locationKey = columnLocation.getTableLocation().getKey(); final Object partitioningColumnValue = locationKey.getPartitionValue(columnDefinition.getName()); if (partitioningColumnValue != null && !Long.class.isAssignableFrom(partitioningColumnValue.getClass())) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceManager.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceManager.java index 572e36a0313..0812cfc8fb3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceManager.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceManager.java @@ -8,16 +8,18 @@ import io.deephaven.engine.rowset.RowSetBuilderSequential; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.WritableRowSet; -import io.deephaven.engine.table.ColumnDefinition; -import io.deephaven.engine.table.impl.ColumnSourceManager; import io.deephaven.engine.table.impl.ColumnToCodecMappings; -import io.deephaven.engine.table.impl.locations.*; import io.deephaven.engine.table.impl.locations.impl.TableLocationUpdateSubscriptionBuffer; -import io.deephaven.engine.table.impl.sources.DeferredGroupingColumnSource; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableInstructions; import io.deephaven.hash.KeyedObjectHashMap; import io.deephaven.hash.KeyedObjectKey; -import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; +import io.deephaven.engine.table.ColumnDefinition; +import io.deephaven.engine.table.impl.ColumnSourceManager; +import io.deephaven.engine.table.impl.locations.*; +import io.deephaven.engine.table.impl.sources.DeferredGroupingColumnSource; +import io.deephaven.internal.log.LoggerFactory; import org.jetbrains.annotations.NotNull; import java.util.*; @@ -49,7 +51,7 @@ public class RegionedColumnSourceManager implements ColumnSourceManager { /** * An unmodifiable view of columnSources. */ - private final Map> sharedColumnSources = + private final Map> sharedColumnSources = Collections.unmodifiableMap(columnSources); /** @@ -69,6 +71,9 @@ public class RegionedColumnSourceManager implements ColumnSourceManager { */ private final List orderedIncludedTableLocations = new ArrayList<>(); + /** A control class for modifying the behavior of the table and its column sources */ + private final SourceTableInstructions instructions; + /** * Whether grouping is enabled. */ @@ -82,16 +87,38 @@ public class RegionedColumnSourceManager implements ColumnSourceManager { * @param componentFactory The component factory * @param columnDefinitions The column definitions */ - RegionedColumnSourceManager(final boolean isRefreshing, + RegionedColumnSourceManager( + final boolean isRefreshing, + @NotNull final RegionedTableComponentFactory componentFactory, + @NotNull final ColumnToCodecMappings codecMappings, + @NotNull final List> columnDefinitions) { + this(isRefreshing, componentFactory, codecMappings, SourceTableInstructions.EMPTY, columnDefinitions); + } + + /** + * Construct a column manager with the specified component factory and definitions. + * + * @param isRefreshing Whether the table using this column source manager is refreshing + * @param componentFactory The component factory + * @param instructions A control class for modifying the behavior of the table and its column sources + * @param columnDefinitions The column definitions + */ + RegionedColumnSourceManager( + final boolean isRefreshing, @NotNull final RegionedTableComponentFactory componentFactory, @NotNull final ColumnToCodecMappings codecMappings, + @NotNull final SourceTableInstructions instructions, @NotNull final List> columnDefinitions) { this.isRefreshing = isRefreshing; + this.instructions = instructions; this.columnDefinitions = columnDefinitions; for (final ColumnDefinition columnDefinition : columnDefinitions) { - columnSources.put( - columnDefinition.getName(), - componentFactory.createRegionedColumnSource(columnDefinition, codecMappings)); + final RegionedColumnSource regionedColumnSource = + componentFactory.createRegionedColumnSource(columnDefinition, codecMappings); + if (instructions.groupingDisabled()) { + regionedColumnSource.disableGrouping(); + } + columnSources.put(columnDefinition.getName(), regionedColumnSource); } } @@ -199,7 +226,7 @@ public final synchronized boolean isEmpty() { } @Override - public final Map> getColumnSources() { + public final Map> getColumnSources() { return sharedColumnSources; } @@ -211,7 +238,8 @@ public final synchronized void disableGrouping() { isGroupingEnabled = false; for (ColumnDefinition columnDefinition : columnDefinitions) { if (columnDefinition.isGrouping()) { - DeferredGroupingColumnSource columnSource = getColumnSources().get(columnDefinition.getName()); + final RegionedColumnSource columnSource = getColumnSources().get(columnDefinition.getName()); + columnSource.disableGrouping(); columnSource.setGroupingProvider(null); columnSource.setGroupToRange(null); } @@ -304,12 +332,13 @@ private void processInitial(final RowSetBuilderSequential addedRowSetBuilder, fi .appendRange(regionFirstKey + subRegionFirstKey, regionFirstKey + subRegionLastKey)); RowSet addRowSetInTable = null; try { - for (final ColumnDefinition columnDefinition : columnDefinitions) { - // noinspection unchecked + for (final ColumnDefinition columnDefinition : columnDefinitions) { + // noinspection unchecked,rawtypes final ColumnLocationState state = new ColumnLocationState( columnDefinition, columnSources.get(columnDefinition.getName()), - location.getColumnLocation(columnDefinition.getName())); + location.getColumnLocation(columnDefinition.getName()), + instructions.getInstructions(columnDefinition.getName())); columnLocationStates.add(state); state.regionAllocated(regionIndex); if (state.needToUpdateGrouping()) { @@ -430,18 +459,23 @@ private class ColumnLocationState { protected final ColumnDefinition definition; protected final RegionedColumnSource source; protected final ColumnLocation location; - - private ColumnLocationState(ColumnDefinition definition, - RegionedColumnSource source, - ColumnLocation location) { + @NotNull + private final SourceTableColumnInstructions instructions; + + private ColumnLocationState( + @NotNull final ColumnDefinition definition, + @NotNull final RegionedColumnSource source, + @NotNull final ColumnLocation location, + @NotNull final SourceTableColumnInstructions instructions) { this.definition = definition; this.source = source; this.location = location; + this.instructions = instructions; } private void regionAllocated(final int regionIndex) { - Assert.eq(regionIndex, "regionIndex", source.addRegion(definition, location), - "source.addRegion((definition, location)"); + Assert.eq(regionIndex, "regionIndex", source.addRegion(definition, location, instructions), + "source.addRegion((definition, location, instructions)"); } private boolean needToUpdateGrouping() { @@ -456,6 +490,7 @@ private boolean needToUpdateGrouping() { private void updateGrouping(@NotNull final RowSet locationAddedRowSetInTable) { if (definition.isGrouping()) { Assert.eqTrue(isGroupingEnabled, "isGroupingEnabled"); + // noinspection rawtypes GroupingProvider groupingProvider = source.getGroupingProvider(); if (groupingProvider == null) { groupingProvider = GroupingProvider.makeGroupingProvider(definition); @@ -463,7 +498,7 @@ private void updateGrouping(@NotNull final RowSet locationAddedRowSetInTable) { source.setGroupingProvider(groupingProvider); } if (groupingProvider instanceof KeyRangeGroupingProvider) { - ((KeyRangeGroupingProvider) groupingProvider).addSource(location, locationAddedRowSetInTable); + ((KeyRangeGroupingProvider) groupingProvider).addSource(location, locationAddedRowSetInTable); } } else if (definition.isPartitioning()) { final DeferredGroupingColumnSource partitioningColumnSource = source; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceObject.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceObject.java index b73e5bdf8a6..e5a1e34b40f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceObject.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceObject.java @@ -10,6 +10,7 @@ import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.ColumnSourceGetDefaults; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -39,9 +40,12 @@ public AsValues(@NotNull final Class dataType, @Nullable final Class< super(ColumnRegionObject.createNull(PARAMETERS.regionMask), dataType, componentType, DeferredColumnRegionObject::new); } - public ColumnRegionObject makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + public ColumnRegionObject makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { + // TODO NATE NOCOMMIT: propagate instructions to region creation if (columnLocation.exists()) { //noinspection unchecked return (ColumnRegionObject) columnLocation.makeColumnRegionObject(columnDefinition); @@ -59,9 +63,11 @@ static final class Partitioning extends RegionedColumnSourceObject makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + public ColumnRegionObject makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { final TableLocationKey locationKey = columnLocation.getTableLocation().getKey(); final Object partitioningColumnValue = locationKey.getPartitionValue(columnDefinition.getName()); if (partitioningColumnValue != null && !getType().isAssignableFrom(partitioningColumnValue.getClass())) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceReferencing.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceReferencing.java index b963c238557..80059d626d4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceReferencing.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceReferencing.java @@ -8,7 +8,9 @@ import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.SharedContext; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import javax.annotation.OverridingMethodsMustInvokeSuper; @@ -18,7 +20,7 @@ */ abstract class RegionedColumnSourceReferencing> extends RegionedColumnSourceBase> - implements ColumnRegionReferencingImpl.Converter { + implements ColumnRegionReferencing.Converter { @NotNull private final ColumnRegionReferencing.Null nullRegion; @@ -57,7 +59,7 @@ public ColumnRegionReferencing getRegion(int regionInd return nullRegion; } - return new ColumnRegionReferencingImpl<>(nativeRegion); + return new ColumnRegionReferencingImpl<>(nativeRegion, this); } @Override @@ -66,8 +68,11 @@ public int getRegionCount() { } @Override - public int addRegion(@NotNull ColumnDefinition columnDefinition, @NotNull ColumnLocation columnLocation) { - return nativeSource.addRegion(columnDefinition, columnLocation); + public int addRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions) { + return nativeSource.addRegion(columnDefinition, columnLocation, instructions); } @Override @@ -76,8 +81,10 @@ int addRegionForUnitTests(OTHER_REGION_TYPE region) { } @Override - public FillContext makeFillContext(int chunkCapacity, SharedContext sharedContext) { - return new ColumnRegionReferencingImpl.FillContext<>(nativeSource, this, chunkCapacity, sharedContext); + public FillContext makeFillContext( + final int chunkCapacity, + @Nullable final SharedContext sharedContext) { + return new ColumnRegionReferencingImpl.FillContext<>(nativeSource, chunkCapacity, sharedContext); } @NotNull diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceShort.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceShort.java index 181984c3ea3..78b7446eea8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceShort.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceShort.java @@ -15,6 +15,7 @@ import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.ColumnSourceGetDefaults; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.jetbrains.annotations.NotNull; import static io.deephaven.util.type.TypeUtils.unbox; @@ -38,9 +39,11 @@ public short getShort(final long rowKey) { interface MakeRegionDefault extends MakeRegion> { @Override - default ColumnRegionShort makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + default ColumnRegionShort makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { if (columnLocation.exists()) { return columnLocation.makeColumnRegionShort(columnDefinition); } @@ -66,9 +69,11 @@ static final class Partitioning extends RegionedColumnSourceShort { } @Override - public ColumnRegionShort makeRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation, - final int regionIndex) { + public ColumnRegionShort makeRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions, + final int regionIndex) { final TableLocationKey locationKey = columnLocation.getTableLocation().getKey(); final Object partitioningColumnValue = locationKey.getPartitionValue(columnDefinition.getName()); if (partitioningColumnValue != null && !Short.class.isAssignableFrom(partitioningColumnValue.getClass())) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceWithDictionary.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceWithDictionary.java index c9bd05ddbee..5072313ef75 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceWithDictionary.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceWithDictionary.java @@ -15,6 +15,7 @@ import io.deephaven.engine.table.impl.sources.RowIdSource; import io.deephaven.engine.table.impl.chunkattributes.DictionaryKeys; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.NotNull; @@ -61,12 +62,10 @@ private final class AsLong extends RegionedColumnSourceBase> implements ColumnSourceGetDefaults.ForLong { - private final ColumnRegionLong nullRegion; private volatile ColumnRegionLong[] wrapperRegions; private AsLong() { super(long.class); - nullRegion = ColumnRegionLong.createNull(PARAMETERS.regionMask); // noinspection unchecked wrapperRegions = new ColumnRegionLong[0]; } @@ -78,9 +77,11 @@ public long getLong(final long rowKey) { } @Override - public int addRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation) { - return RegionedColumnSourceWithDictionary.this.addRegion(columnDefinition, columnLocation); + public int addRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions) { + return RegionedColumnSourceWithDictionary.this.addRegion(columnDefinition, columnLocation, instructions); } @Override @@ -91,7 +92,7 @@ int addRegionForUnitTests(@NotNull final OTHER_REGION_TYPE r @NotNull @Override ColumnRegionLong getNullRegion() { - return nullRegion; + return ColumnRegionLong.createNull(PARAMETERS.regionMask); } @Override @@ -104,7 +105,7 @@ public ColumnRegionLong getRegion(final int regionIndex) { final ColumnRegionObject sourceRegion = RegionedColumnSourceWithDictionary.this.getRegion(regionIndex); if (sourceRegion instanceof ColumnRegion.Null) { - return nullRegion; + return getNullRegion(); } ColumnRegionLong[] localWrappers; ColumnRegionLong wrapper; @@ -168,9 +169,11 @@ public DATA_TYPE get(final long rowKey) { } @Override - public int addRegion(@NotNull final ColumnDefinition columnDefinition, - @NotNull final ColumnLocation columnLocation) { - return RegionedColumnSourceWithDictionary.this.addRegion(columnDefinition, columnLocation); + public int addRegion( + @NotNull final ColumnDefinition columnDefinition, + @NotNull final ColumnLocation columnLocation, + @NotNull final SourceTableColumnInstructions instructions) { + return RegionedColumnSourceWithDictionary.this.addRegion(columnDefinition, columnLocation, instructions); } @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceZonedDateTime.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceZonedDateTime.java index 333366aaee5..3484b2c5092 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceZonedDateTime.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedColumnSourceZonedDateTime.java @@ -105,4 +105,16 @@ public ColumnSource toEpochNano() { public ZoneId getZone() { return zone; } + + @Override + @NotNull + public Object[] convertArray(@NotNull final Object[] keys) { + final Object[] convertedValues = new Object[keys.length]; + for (int ii = 0; ii < keys.length; ii++) { + if (keys[ii] == null || keys[ii] instanceof ZonedDateTime) { + convertedValues[ii] = DateTimeUtils.epochNanos((ZonedDateTime) keys[ii]); + } + } + return convertedValues; + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedPageStore.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedPageStore.java index b429158ba07..22b201fe45b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedPageStore.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedPageStore.java @@ -10,9 +10,12 @@ import io.deephaven.engine.page.Page; import io.deephaven.engine.page.PageStore; import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; +import java.util.Arrays; + public interface RegionedPageStore> extends PageStore { @@ -133,18 +136,22 @@ private static long validateMask(final long mask, final String name) { /** * A regioned page store for use when the full set of regions and their sizes are known. */ - abstract class Static> + abstract class Static> implements RegionedPageStore { private final Parameters parameters; private final REGION_TYPE[] regions; + @NotNull + private final ColumnLocation columnLocation; /** * @param parameters Mask and shift parameters * @param regions Array of all regions in this page store. Array becomes property of the page store. */ - public Static(@NotNull final Parameters parameters, - @NotNull final REGION_TYPE[] regions) { + public Static( + @NotNull final Parameters parameters, + @NotNull final REGION_TYPE[] regions, + @NotNull final ColumnLocation columnLocation) { this.parameters = parameters; this.regions = Require.elementsNeqNull(regions, "regions"); Require.leq(regions.length, "regions.length", parameters.maximumRegionCount, @@ -152,6 +159,7 @@ public Static(@NotNull final Parameters parameters, for (final REGION_TYPE region : regions) { Require.eq(region.mask(), "region.mask()", parameters.regionMask, "parameters.regionMask"); } + this.columnLocation = columnLocation; } @Override @@ -168,5 +176,9 @@ public final int getRegionCount() { public final REGION_TYPE getRegion(final int regionIndex) { return regions[regionIndex]; } + + public ColumnLocation getLocation() { + return columnLocation; + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedTableComponentFactoryImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedTableComponentFactoryImpl.java index 70a7154f799..106598722de 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedTableComponentFactoryImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/RegionedTableComponentFactoryImpl.java @@ -7,6 +7,7 @@ import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.ColumnSourceManager; import io.deephaven.engine.table.impl.ColumnToCodecMappings; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableInstructions; import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; @@ -24,6 +25,11 @@ public class RegionedTableComponentFactoryImpl implements RegionedTableComponent private static final Map, Supplier>> SIMPLE_DATA_TYPE_TO_REGIONED_COLUMN_SOURCE_SUPPLIER; + private static final RegionedTableComponentFactory INSTANCE = + new RegionedTableComponentFactoryImpl(SourceTableInstructions.EMPTY); + + private final SourceTableInstructions instructions; + static { Map, Supplier>> typeToSupplier = new HashMap<>(); typeToSupplier.put(Byte.class, RegionedColumnSourceByte.AsValues::new); @@ -38,16 +44,35 @@ public class RegionedTableComponentFactoryImpl implements RegionedTableComponent SIMPLE_DATA_TYPE_TO_REGIONED_COLUMN_SOURCE_SUPPLIER = Collections.unmodifiableMap(typeToSupplier); } - public static final RegionedTableComponentFactory INSTANCE = new RegionedTableComponentFactoryImpl(); + /** + * Create a default instance of the factory, using no special table or column instructions. + * + * @return a {RegionedTableComponentFactory} + */ + public static RegionedTableComponentFactory make() { + return INSTANCE; + } - private RegionedTableComponentFactoryImpl() {} + /** + * Create a factory that uses the input {@link SourceTableInstructions} for column and region creation. + * + * @param instructions the instructions + * @return a new {@link RegionedTableComponentFactory} + */ + public static RegionedTableComponentFactory make(@NotNull final SourceTableInstructions instructions) { + return new RegionedTableComponentFactoryImpl(instructions); + } + + private RegionedTableComponentFactoryImpl(@NotNull final SourceTableInstructions instructions) { + this.instructions = instructions; + } @Override public ColumnSourceManager createColumnSourceManager( final boolean isRefreshing, @NotNull final ColumnToCodecMappings codecMappings, @NotNull final List> columnDefinitions) { - return new RegionedColumnSourceManager(isRefreshing, this, codecMappings, columnDefinitions); + return new RegionedColumnSourceManager(isRefreshing, this, codecMappings, instructions, columnDefinitions); } /** diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/CacheBehavior.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/CacheBehavior.java new file mode 100644 index 00000000000..25986a4fd81 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/CacheBehavior.java @@ -0,0 +1,15 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.table.impl.sources.regioned.instructions; + +public enum CacheBehavior { + /** Use Deephaven's normal caching behavior */ + Default, + + /** Always cache results */ + Always, + + /** Never cache results */ + Never +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/CacheType.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/CacheType.java new file mode 100644 index 00000000000..64abdcb0fb9 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/CacheType.java @@ -0,0 +1,13 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.table.impl.sources.regioned.instructions; + +public enum CacheType { + + /** Create caches using soft references */ + Soft, + + /** Create caches using hard references */ + Hard +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/SourceTableColumnInstructions.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/SourceTableColumnInstructions.java new file mode 100644 index 00000000000..def0a10b8e6 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/SourceTableColumnInstructions.java @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.table.impl.sources.regioned.instructions; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value; +import org.jetbrains.annotations.NotNull; + + +@Value.Immutable +@BuildableStyle +public abstract class SourceTableColumnInstructions { + public static final SourceTableColumnInstructions DEFAULT = builder().build(); + + /** + * @return the desired caching behavior + */ + @Value.Default + public CacheBehavior cacheBehavior() { + return CacheBehavior.Default; + } + + @Value.Default + public CacheType cacheType() { + return CacheType.Soft; + } + + public static Builder builder() { + return ImmutableSourceTableColumnInstructions.builder(); + } + + public interface Builder { + Builder cacheBehavior(@NotNull CacheBehavior cacheBehavior); + + Builder cacheType(@NotNull CacheType cacheType); + + SourceTableColumnInstructions build(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/SourceTableInstructions.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/SourceTableInstructions.java new file mode 100644 index 00000000000..1ccea67588e --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/instructions/SourceTableInstructions.java @@ -0,0 +1,44 @@ +package io.deephaven.engine.table.impl.sources.regioned.instructions; + +import io.deephaven.annotations.BuildableStyle; +import io.deephaven.engine.table.impl.SourceTable; +import org.immutables.value.Value; +import org.jetbrains.annotations.NotNull; + +import java.util.Map; + +/** + * A contextual class to control the underlying behaviors of {@link SourceTable source tables}. This can be used to + * control the behaviors of different region types, for example Parquet and Deephaven regions. It can also be used to + * control behaviors of the source table itself. + */ + +@Value.Immutable +@BuildableStyle +public abstract class SourceTableInstructions { + public static final SourceTableInstructions EMPTY = builder().build(); + + @Value.Default + public boolean groupingDisabled() { + return false; + } + + abstract Map columnInstructions(); + + public static Builder builder() { + return ImmutableSourceTableInstructions.builder(); + } + + public final SourceTableColumnInstructions getInstructions(@NotNull final String column) { + return columnInstructions().getOrDefault(column, SourceTableColumnInstructions.DEFAULT); + } + + public interface Builder { + Builder groupingDisabled(boolean disabled); + + Builder putColumnInstructions(@NotNull final String colName, + @NotNull final SourceTableColumnInstructions instructions); + + SourceTableInstructions build(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/util/config/MutableInputTable.java b/engine/table/src/main/java/io/deephaven/engine/util/config/MutableInputTable.java index 202256ca7ea..14e81500f9b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/config/MutableInputTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/config/MutableInputTable.java @@ -4,6 +4,7 @@ package io.deephaven.engine.util.config; import io.deephaven.engine.exceptions.ArgumentException; +import io.deephaven.engine.exceptions.IncompatibleTableDefinitionException; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; @@ -52,7 +53,7 @@ default List getValueNames() { * Helper to check if a table is compatible with this table, so that it could be added as contents. * * @param tableToApply the table to check if it can used to add or modify this input table - * @throws TableDefinition.IncompatibleTableDefinitionException if the definitions are not compatible + * @throws IncompatibleTableDefinitionException if the definitions are not compatible */ default void validateAddOrModify(final Table tableToApply) { getTableDefinition().checkMutualCompatibility(tableToApply.getDefinition()); diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/TestPartitioningColumns.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/TestPartitioningColumns.java index 06ac1240ee7..a05af1979a0 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/TestPartitioningColumns.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/TestPartitioningColumns.java @@ -4,25 +4,18 @@ package io.deephaven.engine.table.impl; import io.deephaven.api.filter.Filter; -import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.testutil.TstUtils; import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.time.DateTimeUtils; -import io.deephaven.engine.table.impl.locations.ColumnLocation; -import io.deephaven.engine.table.impl.locations.TableKey; -import io.deephaven.engine.table.impl.locations.TableLocation; -import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.engine.table.impl.locations.impl.*; import io.deephaven.engine.table.impl.select.MatchFilter; import io.deephaven.engine.table.impl.select.WhereFilter; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.impl.sources.regioned.*; import io.deephaven.engine.rowset.RowSetFactory; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.junit.Rule; import org.junit.Test; @@ -74,7 +67,7 @@ public void testEverything() { final TableDefinition resultDefinition = TableDefinition.of(input.getDefinition().getColumnStream() .map(ColumnDefinition::withPartitioning).collect(Collectors.toList())); final Table result = new PartitionAwareSourceTable(resultDefinition, "TestPartitioningColumns", - RegionedTableComponentFactoryImpl.INSTANCE, + RegionedTableComponentFactoryImpl.make(), new PollingTableLocationProvider<>( StandaloneTableKey.getInstance(), recordingLocationKeyFinder, @@ -96,95 +89,4 @@ public void testEverything() { TstUtils.assertTableEquals(expected.selectDistinct(), result.selectDistinct()); } - - private static final class DummyTableLocation extends AbstractTableLocation { - - protected DummyTableLocation(@NotNull final TableKey tableKey, - @NotNull final TableLocationKey tableLocationKey) { - super(tableKey, tableLocationKey, false); - } - - @Override - public void refresh() { - - } - - @NotNull - @Override - protected ColumnLocation makeColumnLocation(@NotNull String name) { - return new ColumnLocation() { - @NotNull - @Override - public TableLocation getTableLocation() { - return DummyTableLocation.this; - } - - @NotNull - @Override - public String getName() { - return name; - } - - @Override - public boolean exists() { - throw new UnsupportedOperationException(); - } - - @Nullable - @Override - public METADATA_TYPE getMetadata(@NotNull ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionChar makeColumnRegionChar( - @NotNull ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionByte makeColumnRegionByte( - @NotNull ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionShort makeColumnRegionShort( - @NotNull ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionInt makeColumnRegionInt( - @NotNull ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionLong makeColumnRegionLong( - @NotNull ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionFloat makeColumnRegionFloat( - @NotNull ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionDouble makeColumnRegionDouble( - @NotNull ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - @Override - public ColumnRegionObject makeColumnRegionObject( - @NotNull ColumnDefinition columnDefinition) { - throw new UnsupportedOperationException(); - } - - }; - } - } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/locations/impl/DummyColumnLocation.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/locations/impl/DummyColumnLocation.java new file mode 100644 index 00000000000..02baff1562b --- /dev/null +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/locations/impl/DummyColumnLocation.java @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.table.impl.locations.impl; + +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.table.ColumnDefinition; +import io.deephaven.engine.table.impl.locations.ColumnLocation; +import io.deephaven.engine.table.impl.locations.TableLocation; +import io.deephaven.engine.table.impl.sources.regioned.*; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class DummyColumnLocation implements ColumnLocation { + private final DummyTableLocation dummyTableLocation; + private final @NotNull String name; + + public DummyColumnLocation(DummyTableLocation dummyTableLocation, @NotNull String name) { + this.dummyTableLocation = dummyTableLocation; + this.name = name; + } + + @NotNull + @Override + public TableLocation getTableLocation() { + return dummyTableLocation; + } + + @NotNull + @Override + public String getName() { + return name; + } + + @Override + public boolean exists() { + throw new UnsupportedOperationException(); + } + + @Nullable + @Override + public METADATA_TYPE getMetadata(@NotNull ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionChar makeColumnRegionChar( + @NotNull ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionByte makeColumnRegionByte( + @NotNull ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionShort makeColumnRegionShort( + @NotNull ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionInt makeColumnRegionInt( + @NotNull ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionLong makeColumnRegionLong( + @NotNull ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionFloat makeColumnRegionFloat( + @NotNull ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionDouble makeColumnRegionDouble( + @NotNull ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + + @Override + public ColumnRegionObject makeColumnRegionObject( + @NotNull ColumnDefinition columnDefinition) { + throw new UnsupportedOperationException(); + } + +} diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/locations/impl/DummyTableLocation.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/locations/impl/DummyTableLocation.java new file mode 100644 index 00000000000..f0d7ee8c82a --- /dev/null +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/locations/impl/DummyTableLocation.java @@ -0,0 +1,50 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.table.impl.locations.impl; + +import io.deephaven.api.SortColumn; +import io.deephaven.engine.table.Table; +import io.deephaven.engine.table.impl.locations.ColumnLocation; +import io.deephaven.engine.table.impl.locations.TableKey; +import io.deephaven.engine.table.impl.locations.TableLocationKey; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; + +public final class DummyTableLocation extends AbstractTableLocation { + + public DummyTableLocation( + @NotNull final TableKey tableKey, + @NotNull final TableLocationKey tableLocationKey) { + super(tableKey, tableLocationKey, false); + } + + @Override + public void refresh() { + + } + + @Override + public @NotNull List getSortedColumns() { + // TODO NATE NOCOMMIT? + return List.of(); + } + + @Override + public boolean hasDataIndexFor(@NotNull String... columns) { + return false; + } + + @Override + protected @Nullable Table getDataIndexImpl(@NotNull String... columns) { + return null; + } + + @NotNull + @Override + protected ColumnLocation makeColumnLocation(@NotNull String name) { + return new DummyColumnLocation(this, name); + } +} diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TestRegionedColumnSourceManager.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TestRegionedColumnSourceManager.java index 1ad647f0532..be82e2917d1 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TestRegionedColumnSourceManager.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TestRegionedColumnSourceManager.java @@ -7,6 +7,7 @@ import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.impl.ColumnToCodecMappings; +import io.deephaven.engine.table.impl.sources.regioned.instructions.SourceTableColumnInstructions; import io.deephaven.engine.testutil.testcase.RefreshingTableTestCase; import io.deephaven.engine.table.impl.locations.*; import io.deephaven.engine.table.impl.locations.impl.SimpleTableLocationKey; @@ -325,8 +326,10 @@ public Object invoke(Invocation invocation) { locationIndexToRegionIndex.put(li, regionIndex); IntStream.range(0, NUM_COLUMNS).forEach(ci -> checking(new Expectations() { { - oneOf(columnSources[ci]).addRegion(with(columnDefinitions.get(ci)), - with(columnLocations[li][ci])); + oneOf(columnSources[ci]).addRegion( + with(columnDefinitions.get(ci)), + with(columnLocations[li][ci]), + SourceTableColumnInstructions.DEFAULT); will(returnValue(regionIndex)); } })); diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionByte.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionByte.java index 9e4c4344414..3d75a02549d 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionByte.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionByte.java @@ -4,6 +4,8 @@ package io.deephaven.engine.table.impl.sources.regioned; import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.chunk.WritableByteChunk; import io.deephaven.chunk.WritableChunk; @@ -62,6 +64,21 @@ public void fillChunkAppend(@NotNull FillContext context, @NotNull WritableChunk charDestination.setSize(size + length); } + + @Override + public ColumnLocation getLocation() { + throw new UnsupportedOperationException(); + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + throw new UnsupportedOperationException("This test region does not support matching"); + } } public static class TestNull extends TstColumnRegionPrimative> { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionChar.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionChar.java index a0e2cd9784a..5f184565913 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionChar.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionChar.java @@ -5,6 +5,8 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.chunk.WritableCharChunk; import io.deephaven.chunk.WritableChunk; @@ -53,6 +55,21 @@ public void fillChunkAppend(@NotNull FillContext context, @NotNull WritableChunk charDestination.setSize(size + length); } + + @Override + public ColumnLocation getLocation() { + throw new UnsupportedOperationException(); + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + throw new UnsupportedOperationException("This test region does not support matching"); + } } public static class TestNull extends TstColumnRegionPrimative> { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionDouble.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionDouble.java index 5ab8dc37aa6..d65bb654fa7 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionDouble.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionDouble.java @@ -10,6 +10,8 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.chunk.WritableDoubleChunk; import io.deephaven.chunk.WritableChunk; @@ -58,6 +60,21 @@ public void fillChunkAppend(@NotNull FillContext context, @NotNull WritableChunk doubleDestination.setSize(size + length); } + + @Override + public ColumnLocation getLocation() { + throw new UnsupportedOperationException(); + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + throw new UnsupportedOperationException("This test region does not support matching"); + } } public static class TestNull extends TstColumnRegionPrimative> { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionFloat.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionFloat.java index 799630bc19a..338c5d3f1b2 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionFloat.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionFloat.java @@ -10,6 +10,8 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.chunk.WritableFloatChunk; import io.deephaven.chunk.WritableChunk; @@ -58,6 +60,21 @@ public void fillChunkAppend(@NotNull FillContext context, @NotNull WritableChunk floatDestination.setSize(size + length); } + + @Override + public ColumnLocation getLocation() { + throw new UnsupportedOperationException(); + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + throw new UnsupportedOperationException("This test region does not support matching"); + } } public static class TestNull extends TstColumnRegionPrimative> { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionInt.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionInt.java index 8fc8d76ef30..25da27ab3d7 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionInt.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionInt.java @@ -10,6 +10,8 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.chunk.WritableIntChunk; import io.deephaven.chunk.WritableChunk; @@ -58,6 +60,21 @@ public void fillChunkAppend(@NotNull FillContext context, @NotNull WritableChunk intDestination.setSize(size + length); } + + @Override + public ColumnLocation getLocation() { + throw new UnsupportedOperationException(); + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + throw new UnsupportedOperationException("This test region does not support matching"); + } } public static class TestNull extends TstColumnRegionPrimative> { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionLong.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionLong.java index 92728fda34d..4086b947f39 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionLong.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionLong.java @@ -10,6 +10,8 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.chunk.WritableLongChunk; import io.deephaven.chunk.WritableChunk; @@ -58,6 +60,21 @@ public void fillChunkAppend(@NotNull FillContext context, @NotNull WritableChunk longDestination.setSize(size + length); } + + @Override + public ColumnLocation getLocation() { + throw new UnsupportedOperationException(); + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + throw new UnsupportedOperationException("This test region does not support matching"); + } } public static class TestNull extends TstColumnRegionPrimative> { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionShort.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionShort.java index 4f0e90e3b05..64bd60409f9 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionShort.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/TstColumnRegionShort.java @@ -10,6 +10,8 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.util.QueryConstants; import io.deephaven.chunk.WritableShortChunk; import io.deephaven.chunk.WritableChunk; @@ -58,6 +60,21 @@ public void fillChunkAppend(@NotNull FillContext context, @NotNull WritableChunk shortDestination.setSize(size + length); } + + @Override + public ColumnLocation getLocation() { + throw new UnsupportedOperationException(); + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + throw new UnsupportedOperationException("This test region does not support matching"); + } } public static class TestNull extends TstColumnRegionPrimative> { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/ByteRegionBinarySearchKernelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/ByteRegionBinarySearchKernelTest.java index e87d5ac7ecd..2cd1f6c1b7a 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/ByteRegionBinarySearchKernelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/ByteRegionBinarySearchKernelTest.java @@ -13,6 +13,10 @@ import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.locations.impl.DummyColumnLocation; +import io.deephaven.engine.table.impl.locations.impl.DummyTableLocation; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableKey; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableLocationKey; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionByte; import io.deephaven.engine.table.impl.sources.regioned.RegionedColumnSource; import io.deephaven.engine.testutil.junit4.EngineCleanup; @@ -224,6 +228,11 @@ public void testInvertedRowIsRange() { } private static final int PAGE_SIZE = 1 << 16; + private static final DummyTableLocation dummyTableLocation = new DummyTableLocation( + StandaloneTableKey.getInstance(), StandaloneTableLocationKey.getInstance()); + private static final DummyColumnLocation dummyColumnLocation = new DummyColumnLocation( + dummyTableLocation, "dummyColumn"); + private static ColumnRegionByte makeColumnRegionByte(@NotNull final List values) { return new AppendOnlyFixedSizePageRegionByte<>( RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, new AppendOnlyRegionAccessor<>() { @@ -240,6 +249,6 @@ public void readChunkPage(long firstRowPosition, int minimumSize, @NotNull Writa public long size() { return values.size(); } - }); + }, dummyColumnLocation); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/CharRegionBinarySearchKernelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/CharRegionBinarySearchKernelTest.java index 27d7d8777b3..fedf064fac8 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/CharRegionBinarySearchKernelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/CharRegionBinarySearchKernelTest.java @@ -8,6 +8,10 @@ import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.locations.impl.DummyColumnLocation; +import io.deephaven.engine.table.impl.locations.impl.DummyTableLocation; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableKey; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableLocationKey; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionChar; import io.deephaven.engine.table.impl.sources.regioned.RegionedColumnSource; import io.deephaven.engine.testutil.junit4.EngineCleanup; @@ -219,6 +223,11 @@ public void testInvertedRowIsRange() { } private static final int PAGE_SIZE = 1 << 16; + private static final DummyTableLocation dummyTableLocation = new DummyTableLocation( + StandaloneTableKey.getInstance(), StandaloneTableLocationKey.getInstance()); + private static final DummyColumnLocation dummyColumnLocation = new DummyColumnLocation( + dummyTableLocation, "dummyColumn"); + private static ColumnRegionChar makeColumnRegionChar(@NotNull final List values) { return new AppendOnlyFixedSizePageRegionChar<>( RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, new AppendOnlyRegionAccessor<>() { @@ -235,6 +244,6 @@ public void readChunkPage(long firstRowPosition, int minimumSize, @NotNull Writa public long size() { return values.size(); } - }); + }, dummyColumnLocation); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/DoubleRegionBinarySearchKernelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/DoubleRegionBinarySearchKernelTest.java index b39b60d7e88..80150e6c134 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/DoubleRegionBinarySearchKernelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/DoubleRegionBinarySearchKernelTest.java @@ -13,6 +13,10 @@ import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.locations.impl.DummyColumnLocation; +import io.deephaven.engine.table.impl.locations.impl.DummyTableLocation; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableKey; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableLocationKey; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionDouble; import io.deephaven.engine.table.impl.sources.regioned.RegionedColumnSource; import io.deephaven.engine.testutil.junit4.EngineCleanup; @@ -224,6 +228,11 @@ public void testInvertedRowIsRange() { } private static final int PAGE_SIZE = 1 << 16; + private static final DummyTableLocation dummyTableLocation = new DummyTableLocation( + StandaloneTableKey.getInstance(), StandaloneTableLocationKey.getInstance()); + private static final DummyColumnLocation dummyColumnLocation = new DummyColumnLocation( + dummyTableLocation, "dummyColumn"); + private static ColumnRegionDouble makeColumnRegionDouble(@NotNull final List values) { return new AppendOnlyFixedSizePageRegionDouble<>( RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, new AppendOnlyRegionAccessor<>() { @@ -240,6 +249,6 @@ public void readChunkPage(long firstRowPosition, int minimumSize, @NotNull Writa public long size() { return values.size(); } - }); + }, dummyColumnLocation); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/FloatRegionBinarySearchKernelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/FloatRegionBinarySearchKernelTest.java index 446a53c7dac..924062f9971 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/FloatRegionBinarySearchKernelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/FloatRegionBinarySearchKernelTest.java @@ -13,6 +13,10 @@ import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.locations.impl.DummyColumnLocation; +import io.deephaven.engine.table.impl.locations.impl.DummyTableLocation; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableKey; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableLocationKey; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionFloat; import io.deephaven.engine.table.impl.sources.regioned.RegionedColumnSource; import io.deephaven.engine.testutil.junit4.EngineCleanup; @@ -224,6 +228,11 @@ public void testInvertedRowIsRange() { } private static final int PAGE_SIZE = 1 << 16; + private static final DummyTableLocation dummyTableLocation = new DummyTableLocation( + StandaloneTableKey.getInstance(), StandaloneTableLocationKey.getInstance()); + private static final DummyColumnLocation dummyColumnLocation = new DummyColumnLocation( + dummyTableLocation, "dummyColumn"); + private static ColumnRegionFloat makeColumnRegionFloat(@NotNull final List values) { return new AppendOnlyFixedSizePageRegionFloat<>( RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, new AppendOnlyRegionAccessor<>() { @@ -240,6 +249,6 @@ public void readChunkPage(long firstRowPosition, int minimumSize, @NotNull Writa public long size() { return values.size(); } - }); + }, dummyColumnLocation); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/IntRegionBinarySearchKernelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/IntRegionBinarySearchKernelTest.java index 2b40a9c9f89..bab6fa20a0a 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/IntRegionBinarySearchKernelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/IntRegionBinarySearchKernelTest.java @@ -13,6 +13,10 @@ import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.locations.impl.DummyColumnLocation; +import io.deephaven.engine.table.impl.locations.impl.DummyTableLocation; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableKey; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableLocationKey; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionInt; import io.deephaven.engine.table.impl.sources.regioned.RegionedColumnSource; import io.deephaven.engine.testutil.junit4.EngineCleanup; @@ -224,6 +228,11 @@ public void testInvertedRowIsRange() { } private static final int PAGE_SIZE = 1 << 16; + private static final DummyTableLocation dummyTableLocation = new DummyTableLocation( + StandaloneTableKey.getInstance(), StandaloneTableLocationKey.getInstance()); + private static final DummyColumnLocation dummyColumnLocation = new DummyColumnLocation( + dummyTableLocation, "dummyColumn"); + private static ColumnRegionInt makeColumnRegionInt(@NotNull final List values) { return new AppendOnlyFixedSizePageRegionInt<>( RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, new AppendOnlyRegionAccessor<>() { @@ -240,6 +249,6 @@ public void readChunkPage(long firstRowPosition, int minimumSize, @NotNull Writa public long size() { return values.size(); } - }); + }, dummyColumnLocation); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/LongRegionBinarySearchKernelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/LongRegionBinarySearchKernelTest.java index 12dcaac307a..908d3659fe0 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/LongRegionBinarySearchKernelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/LongRegionBinarySearchKernelTest.java @@ -13,6 +13,10 @@ import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.locations.impl.DummyColumnLocation; +import io.deephaven.engine.table.impl.locations.impl.DummyTableLocation; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableKey; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableLocationKey; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionLong; import io.deephaven.engine.table.impl.sources.regioned.RegionedColumnSource; import io.deephaven.engine.testutil.junit4.EngineCleanup; @@ -224,6 +228,11 @@ public void testInvertedRowIsRange() { } private static final int PAGE_SIZE = 1 << 16; + private static final DummyTableLocation dummyTableLocation = new DummyTableLocation( + StandaloneTableKey.getInstance(), StandaloneTableLocationKey.getInstance()); + private static final DummyColumnLocation dummyColumnLocation = new DummyColumnLocation( + dummyTableLocation, "dummyColumn"); + private static ColumnRegionLong makeColumnRegionLong(@NotNull final List values) { return new AppendOnlyFixedSizePageRegionLong<>( RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, new AppendOnlyRegionAccessor<>() { @@ -240,6 +249,6 @@ public void readChunkPage(long firstRowPosition, int minimumSize, @NotNull Writa public long size() { return values.size(); } - }); + }, dummyColumnLocation); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/ShortRegionBinarySearchKernelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/ShortRegionBinarySearchKernelTest.java index c54e715d24a..caf2344454f 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/ShortRegionBinarySearchKernelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/sources/regioned/kernel/ShortRegionBinarySearchKernelTest.java @@ -13,6 +13,10 @@ import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.locations.impl.DummyColumnLocation; +import io.deephaven.engine.table.impl.locations.impl.DummyTableLocation; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableKey; +import io.deephaven.engine.table.impl.locations.impl.StandaloneTableLocationKey; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionShort; import io.deephaven.engine.table.impl.sources.regioned.RegionedColumnSource; import io.deephaven.engine.testutil.junit4.EngineCleanup; @@ -224,6 +228,11 @@ public void testInvertedRowIsRange() { } private static final int PAGE_SIZE = 1 << 16; + private static final DummyTableLocation dummyTableLocation = new DummyTableLocation( + StandaloneTableKey.getInstance(), StandaloneTableLocationKey.getInstance()); + private static final DummyColumnLocation dummyColumnLocation = new DummyColumnLocation( + dummyTableLocation, "dummyColumn"); + private static ColumnRegionShort makeColumnRegionShort(@NotNull final List values) { return new AppendOnlyFixedSizePageRegionShort<>( RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, new AppendOnlyRegionAccessor<>() { @@ -240,6 +249,6 @@ public void readChunkPage(long firstRowPosition, int minimumSize, @NotNull Writa public long size() { return values.size(); } - }); + }, dummyColumnLocation); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/util/TestTableTools.java b/engine/table/src/test/java/io/deephaven/engine/util/TestTableTools.java index df34e4b5a33..4e76b0b7f02 100644 --- a/engine/table/src/test/java/io/deephaven/engine/util/TestTableTools.java +++ b/engine/table/src/test/java/io/deephaven/engine/util/TestTableTools.java @@ -7,6 +7,7 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.*; +import io.deephaven.engine.exceptions.IncompatibleTableDefinitionException; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.RowSetShiftData; @@ -99,31 +100,31 @@ public void testMergeOfMismatchedTables() { try { TableTools.merge(table1, table2); TestCase.fail("Expected exception"); - } catch (TableDefinition.IncompatibleTableDefinitionException expected) { + } catch (IncompatibleTableDefinitionException expected) { } try { TableTools.merge(table2, table1); TestCase.fail("Expected exception"); - } catch (TableDefinition.IncompatibleTableDefinitionException expected) { + } catch (IncompatibleTableDefinitionException expected) { } try { TableTools.merge(table2, emptyTable); TestCase.fail("Expected exception"); - } catch (TableDefinition.IncompatibleTableDefinitionException expected) { + } catch (IncompatibleTableDefinitionException expected) { } try { TableTools.merge(table2, table2.updateView("S2=StringKeys1")); TestCase.fail("Expected exception"); - } catch (TableDefinition.IncompatibleTableDefinitionException expected) { + } catch (IncompatibleTableDefinitionException expected) { } try { TableTools.merge(table2, table2.dropColumns("StringKeys1")); TestCase.fail("Expected exception"); - } catch (TableDefinition.IncompatibleTableDefinitionException expected) { + } catch (IncompatibleTableDefinitionException expected) { } } diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/locations/TableBackedColumnLocation.java b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/locations/TableBackedColumnLocation.java index a0cce6fe00b..30d6acaea9f 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/locations/TableBackedColumnLocation.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/locations/TableBackedColumnLocation.java @@ -45,50 +45,50 @@ public boolean exists() { @Override public ColumnRegionChar makeColumnRegionChar(@NotNull final ColumnDefinition columnDefinition) { return new AppendOnlyFixedSizePageRegionChar<>( - RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this); + RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this, this); } @Override public ColumnRegionByte makeColumnRegionByte(@NotNull final ColumnDefinition columnDefinition) { return new AppendOnlyFixedSizePageRegionByte<>( - RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this); + RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this, this); } @Override public ColumnRegionShort makeColumnRegionShort(@NotNull final ColumnDefinition columnDefinition) { return new AppendOnlyFixedSizePageRegionShort<>( - RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this); + RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this, this); } @Override public ColumnRegionInt makeColumnRegionInt(@NotNull final ColumnDefinition columnDefinition) { return new AppendOnlyFixedSizePageRegionInt<>( - RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this); + RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this, this); } @Override public ColumnRegionLong makeColumnRegionLong(@NotNull final ColumnDefinition columnDefinition) { return new AppendOnlyFixedSizePageRegionLong<>( - RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this); + RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this, this); } @Override public ColumnRegionFloat makeColumnRegionFloat(@NotNull final ColumnDefinition columnDefinition) { return new AppendOnlyFixedSizePageRegionFloat<>( - RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this); + RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this, this); } @Override public ColumnRegionDouble makeColumnRegionDouble(@NotNull final ColumnDefinition columnDefinition) { return new AppendOnlyFixedSizePageRegionDouble<>( - RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this); + RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this, this); } @Override public ColumnRegionObject makeColumnRegionObject( @NotNull final ColumnDefinition columnDefinition) { return new AppendOnlyFixedSizePageRegionObject<>( - RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this); + RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK, PAGE_SIZE, this, this); } @Override diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/locations/TableBackedTableLocation.java b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/locations/TableBackedTableLocation.java index b5f40945795..a564c826339 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/locations/TableBackedTableLocation.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/locations/TableBackedTableLocation.java @@ -1,5 +1,7 @@ package io.deephaven.engine.testutil.locations; +import io.deephaven.api.SortColumn; +import io.deephaven.engine.table.Table; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.locations.TableDataException; @@ -7,6 +9,9 @@ import io.deephaven.engine.table.impl.locations.impl.StandaloneTableKey; import io.deephaven.engine.updategraph.UpdateSourceRegistrar; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; public final class TableBackedTableLocation extends AbstractTableLocation { @@ -60,4 +65,20 @@ public void refresh() { protected @NotNull ColumnLocation makeColumnLocation(@NotNull final String name) { return new TableBackedColumnLocation(this, name); } + + @Override + public @NotNull List getSortedColumns() { + // TODO NATE NOCOMMIT + return List.of(); + } + + @Override + public boolean hasDataIndexFor(@NotNull String... columns) { + return false; + } + + @Override + protected @Nullable Table getDataIndexImpl(@NotNull String... columns) { + return null; + } } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/AbstractBulkValuesWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/AbstractBulkValuesWriter.java index 9277a23d77b..6370164b517 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/AbstractBulkValuesWriter.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/AbstractBulkValuesWriter.java @@ -85,7 +85,7 @@ public int writeBulkVector(@NotNull final BUFFER_TYPE bulkValues, *

* *

- * Definition levels are used to encode how many preceeding optional fields are defined at a given point. For + * Definition levels are used to encode how many preceding optional fields are defined at a given point. For * Vectors, as defined above, this gives us three different possible values for the definition value of a single * item: *

    diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java index b9290e96407..3c6ccf9f6df 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReader.java @@ -5,6 +5,7 @@ import org.apache.parquet.column.Dictionary; import org.apache.parquet.internal.column.columnindex.OffsetIndex; +import org.apache.parquet.column.statistics.Statistics; import org.apache.parquet.schema.PrimitiveType; import org.jetbrains.annotations.Nullable; @@ -63,6 +64,10 @@ interface ColumnPageDirectAccessor { */ Supplier getDictionarySupplier(); + boolean hasStatistics(); + + , S extends Statistics> S getStatistics(); + Dictionary NULL_DICTIONARY = new NullDictionary(); final class NullDictionary extends Dictionary { diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java index 8de3d8b9281..6d5fc00a942 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java @@ -13,13 +13,15 @@ import org.apache.parquet.column.Dictionary; import org.apache.parquet.column.Encoding; import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.statistics.Statistics; import org.apache.parquet.format.*; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.internal.column.columnindex.OffsetIndex; -import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.io.InputStream; @@ -39,6 +41,7 @@ public class ColumnChunkReaderImpl implements ColumnChunkReader { private final ColumnChunk columnChunk; + private final ColumnChunkMetaData columnChunkMetaData; private final SeekableChannelsProvider channelsProvider; private final Path rootPath; private final CompressorAdapter decompressor; @@ -58,14 +61,21 @@ public class ColumnChunkReaderImpl implements ColumnChunkReader { */ private final String version; - ColumnChunkReaderImpl(ColumnChunk columnChunk, SeekableChannelsProvider channelsProvider, Path rootPath, - MessageType type, OffsetIndex offsetIndex, List fieldTypes, final long numRows, - final String version) { - this.channelsProvider = channelsProvider; + ColumnChunkReaderImpl( + @NotNull final ColumnChunk columnChunk, + @NotNull final ColumnChunkMetaData columnChunkMetaData, + @NotNull final SeekableChannelsProvider channelsProvider, + @NotNull final Path rootPath, + @NotNull final MessageType type, + @NotNull final OffsetIndex offsetIndex, + @NotNull final List fieldTypes, + final long numRows, + @Nullable final String version) { this.columnChunk = columnChunk; + this.columnChunkMetaData = columnChunkMetaData; + this.channelsProvider = channelsProvider; this.rootPath = rootPath; - this.path = type - .getColumnDescription(columnChunk.meta_data.getPath_in_schema().toArray(new String[0])); + this.path = type.getColumnDescription(columnChunk.meta_data.getPath_in_schema().toArray(String[]::new)); if (columnChunk.getMeta_data().isSetCodec()) { decompressor = DeephavenCompressorAdapterFactory.getInstance() .getByName(columnChunk.getMeta_data().getCodec().name()); @@ -150,6 +160,17 @@ public boolean usesDictionaryOnEveryPage() { return true; } + @Override + public boolean hasStatistics() { + return columnChunk.getMeta_data().isSetStatistics(); + } + + @Override + public , S extends Statistics> S getStatistics() { + // noinspection unchecked + return (S) columnChunkMetaData.getStatistics(); + } + @Override public Supplier getDictionarySupplier() { return dictionarySupplier; diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnWriter.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnWriter.java index 8725a9a1424..52961b58f64 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnWriter.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnWriter.java @@ -4,33 +4,56 @@ package io.deephaven.parquet.base; import io.deephaven.util.SafeCloseable; +import org.apache.parquet.column.statistics.Statistics; import org.jetbrains.annotations.NotNull; import org.apache.parquet.column.statistics.Statistics; import java.io.IOException; import java.nio.IntBuffer; +/** + * Implementations of this are responsible for encoding pages of data into Parquet file columns + */ public interface ColumnWriter extends SafeCloseable { /** - * Add a page with no nulls to the file. + * Add a page of data that does not contain null values. + * + * @param pageData the raw page data + * @param valuesCount the count of values in the page. + * @param statistics the + * @throws IOException if there is an error writing values. */ void addPageNoNulls(@NotNull Object pageData, int valuesCount, @NotNull Statistics statistics) throws IOException; /** - * Add a dictionary page to the file. + * Add a page of dictionary data to the column. + * + * @param dictionaryValues the values in the dictionary + * @param valuesCount the count of values + * @throws IOException if there is an error writing dictionary values. */ void addDictionaryPage(@NotNull Object dictionaryValues, int valuesCount) throws IOException; /** - * Add a page (potentially containing nulls) to the file. + * Add a page of values which may contain null values. + * + * @param pageData the raw page data. + * @param valuesCount the total count of items + * @throws IOException if there is an error writing values. */ - void addPage(Object pageData, int valuesCount, Statistics statistics) throws IOException; + void addPage(@NotNull Object pageData, int valuesCount, Statistics statistics) throws IOException; /** - * Add a vector page to the file. + * Encode a page of vector values. + * + * @param pageData the raw flattened page data + * @param repeatCount an int buffer where each int is the length of a single source vector. + * @param valuesCount the total number of vectors to encode + * @throws IOException if there is an error writing values. */ - void addVectorPage(@NotNull Object pageData, + void addVectorPage( + @NotNull Object pageData, @NotNull IntBuffer repeatCount, int valuesCount, @NotNull Statistics statistics) @@ -42,7 +65,10 @@ void addVectorPage(@NotNull Object pageData, void resetStats(); /** - * Return the current statistics. + * Get the {@link Statistics} object into which to encode min/max/null count information for the footer. + * + * @return the statistics object */ + @NotNull Statistics getStats(); } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnWriterImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnWriterImpl.java index 1aea3081a3b..422eb6d70cc 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnWriterImpl.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnWriterImpl.java @@ -286,6 +286,7 @@ public void writePageV2( final int uncompressedDataSize = data.remaining(); final int uncompressedSize = (int) (uncompressedDataSize + repetitionLevels.size() + definitionLevels.size()); + compressorAdapter.reset(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (final WritableByteChannel channel = Channels.newChannel(compressorAdapter.compress(baos))) { channel.write(data); @@ -447,6 +448,7 @@ public void resetStats() { statistics = Statistics.createStats(column.getPrimitiveType()); } + @NotNull @Override public Statistics getStats() { return statistics; diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java index 4545c52b855..6bdadcc933c 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java @@ -8,6 +8,8 @@ import org.apache.parquet.format.*; import org.apache.parquet.format.ColumnOrder; import org.apache.parquet.format.Type; +import org.apache.parquet.format.converter.ParquetMetadataConverter; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.*; import java.io.ByteArrayInputStream; @@ -28,14 +30,22 @@ public class ParquetFileReader { private static final String MAGIC_STR = "PAR1"; static final byte[] MAGIC = MAGIC_STR.getBytes(StandardCharsets.US_ASCII); - public final FileMetaData fileMetaData; + private final FileMetaData fileMetaData; private final SeekableChannelsProvider channelsProvider; private final Path rootPath; private final MessageType type; + private final ParquetMetadata parquetMetadata; public ParquetFileReader(final String filePath, final SeekableChannelsProvider channelsProvider) throws IOException { this.channelsProvider = channelsProvider; + + // TODO (AB): This whole method is scary. It has a bunch of code stolen from Apache's ParquetFileReader + // but is returning a parquet.format.FileMetaData instead of a ParquetMetaData, which itself + // has a DIFFERENT FileMetaData (org.apache.parquet.hadoop.FileMetaData) Which has different fields + // I am leaving it for now, but I'm concerned that we are not reading this properly (even though + // we are reading the correct data) + // Root path should be this file if a single file, else the parent directory for a metadata // file rootPath = @@ -71,6 +81,7 @@ public ParquetFileReader(final String filePath, final SeekableChannelsProvider c Helpers.readBytes(readChannel, footer); } fileMetaData = Util.readFileMetaData(new ByteArrayInputStream(footer)); + parquetMetadata = new ParquetMetadataConverter().fromParquetMetadata(fileMetaData); type = fromParquetSchema(fileMetaData.schema, fileMetaData.column_orders); } @@ -180,6 +191,7 @@ private int readIntLittleEndian(SeekableByteChannel f) throws IOException { */ public RowGroupReader getRowGroup(final int groupNumber, final String version) { return new RowGroupReaderImpl( + parquetMetadata.getBlocks().get(groupNumber), fileMetaData.getRow_groups().get(groupNumber), channelsProvider, rootPath, @@ -188,6 +200,14 @@ public RowGroupReader getRowGroup(final int groupNumber, final String version) { version); } + public ParquetMetadata getParquetMetadata() { + return parquetMetadata; + } + + public FileMetaData getFileMetaData() { + return fileMetaData; + } + private static MessageType fromParquetSchema(List schema, List columnOrders) throws ParquetFileReaderException { final Iterator iterator = schema.iterator(); diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java index 90d4e20add3..29a4a6c28fb 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/RowGroupReaderImpl.java @@ -8,6 +8,8 @@ import org.apache.parquet.format.RowGroup; import org.apache.parquet.format.Util; import org.apache.parquet.format.converter.ParquetMetadataConverter; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.internal.column.columnindex.OffsetIndex; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; @@ -33,11 +35,13 @@ public class RowGroupReaderImpl implements RowGroupReader { private final MessageType type; private final Map> schemaMap = new HashMap<>(); private final Map chunkMap = new HashMap<>(); + private final Map metadataMap = new HashMap<>(); private final Path rootPath; private final String version; RowGroupReaderImpl( + @NotNull final BlockMetaData blockMetaData, @NotNull final RowGroup rowGroup, @NotNull final SeekableChannelsProvider channelsProvider, @NotNull final Path rootPath, @@ -48,19 +52,25 @@ public class RowGroupReaderImpl implements RowGroupReader { this.rowGroup = rowGroup; this.rootPath = rootPath; this.type = type; - for (ColumnChunk column : rowGroup.columns) { - List path_in_schema = column.getMeta_data().path_in_schema; - String key = path_in_schema.toString(); - chunkMap.put(key, column); - List nonRequiredFields = new ArrayList<>(); + + // TODO: Use BlockMetaData instead of column.getMetaData + final List columns = rowGroup.columns; + for (int ii = 0; ii < columns.size(); ii++) { + final ColumnChunk column = columns.get(ii); + final List path_in_schema = column.getMeta_data().path_in_schema; + final String key = path_in_schema.toString(); + final List nonRequiredFields = new ArrayList<>(); for (int indexInPath = 0; indexInPath < path_in_schema.size(); indexInPath++) { - Type fieldType = schema - .getType(path_in_schema.subList(0, indexInPath + 1).toArray(new String[0])); + final Type fieldType = schema.getType(path_in_schema.subList(0, indexInPath + 1) + .toArray(String[]::new)); if (fieldType.getRepetition() != Type.Repetition.REQUIRED) { nonRequiredFields.add(fieldType); } } + + chunkMap.put(key, column); schemaMap.put(key, nonRequiredFields); + metadataMap.put(key, blockMetaData.getColumns().get(ii)); } this.version = version; } @@ -84,8 +94,15 @@ public ColumnChunkReaderImpl getColumnChunk(@NotNull final List path) { throw new UncheckedIOException(e); } } - return new ColumnChunkReaderImpl(columnChunk, channelsProvider, rootPath, type, offsetIndex, fieldTypes, - numRows(), version); + return new ColumnChunkReaderImpl(columnChunk, + metadataMap.get(key), + channelsProvider, + rootPath, + type, + offsetIndex, + fieldTypes, + numRows(), + version); } @Override diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java index b9648636066..c50ba3fe609 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java @@ -4,13 +4,13 @@ package io.deephaven.parquet.table; import io.deephaven.UncheckedDeephavenException; -import io.deephaven.stringset.StringSet; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.parquet.base.ParquetFileReader; import io.deephaven.parquet.table.metadata.CodecInfo; import io.deephaven.parquet.table.metadata.ColumnTypeInfo; import io.deephaven.parquet.table.metadata.TableInfo; -import io.deephaven.parquet.base.ParquetFileReader; import org.apache.parquet.format.converter.ParquetMetadataConverter; +import io.deephaven.stringset.StringSet; import io.deephaven.util.codec.SimpleByteArrayCodec; import io.deephaven.util.codec.UTF8StringAsByteArrayCodec; import org.apache.commons.lang3.mutable.MutableObject; @@ -100,7 +100,7 @@ public static ParquetInstructions readParquetSchema( @NotNull final BiFunction, String> legalizeColumnNameFunc) throws IOException { final ParquetFileReader parquetFileReader = ParquetTools.getParquetFileReaderChecked(new File(filePath)); final ParquetMetadata parquetMetadata = - new ParquetMetadataConverter().fromParquetMetadata(parquetFileReader.fileMetaData); + new ParquetMetadataConverter().fromParquetMetadata(parquetFileReader.getFileMetaData()); return readParquetSchema(parquetFileReader.getSchema(), parquetMetadata.getFileMetaData().getKeyValueMetaData(), readInstructions, consumer, legalizeColumnNameFunc); } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java index c477d65bf69..1149bd4d47e 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java @@ -702,7 +702,7 @@ public static Table readSingleFileTable( null); return new SimpleSourceTable(tableDefinition.getWritable(), "Read single parquet file from " + tableLocationKey.getFile(), - RegionedTableComponentFactoryImpl.INSTANCE, locationProvider, null); + RegionedTableComponentFactoryImpl.make(), locationProvider, null); } /** @@ -756,7 +756,7 @@ public static Table readPartitionedTable( return new PartitionAwareSourceTable( tableDefinition, description, - RegionedTableComponentFactoryImpl.INSTANCE, + RegionedTableComponentFactoryImpl.make(), new PollingTableLocationProvider<>( StandaloneTableKey.getInstance(), keyFinder, diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java index b1720cdbdce..2091f42f519 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java @@ -139,7 +139,7 @@ public ParquetMetadataFileLayout( ColumnDefinition::getName, cd -> PartitionParser.lookupSupported(cd.getDataType(), cd.getComponentType()))); final Map fileNameToRowGroupIndices = new LinkedHashMap<>(); - final List rowGroups = metadataFileReader.fileMetaData.getRow_groups(); + final List rowGroups = metadataFileReader.getFileMetaData().getRow_groups(); final int numRowGroups = rowGroups.size(); for (int rgi = 0; rgi < numRowGroups; ++rgi) { fileNameToRowGroupIndices @@ -212,7 +212,7 @@ private static ParquetMetadata convertMetadata(@NotNull final File file, @NotNull final ParquetFileReader fileReader, @NotNull final ParquetMetadataConverter converter) { try { - return converter.fromParquetMetadata(fileReader.fileMetaData); + return converter.fromParquetMetadata(fileReader.getFileMetaData()); } catch (IOException e) { throw new TableDataException("Error while converting file metadata from " + file); } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java index 6c61c9276e5..d696f3d036a 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetColumnLocation.java @@ -51,6 +51,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.*; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongFunction; import java.util.function.Supplier; @@ -60,7 +61,7 @@ import static io.deephaven.engine.table.impl.sources.regioned.RegionedColumnSource.ROW_KEY_TO_SUB_REGION_ROW_INDEX_MASK; import static io.deephaven.parquet.table.ParquetTableWriter.*; -final class ParquetColumnLocation extends AbstractColumnLocation { +final public class ParquetColumnLocation extends AbstractColumnLocation { private static final String IMPLEMENTATION_NAME = ParquetColumnLocation.class.getSimpleName(); @@ -198,7 +199,7 @@ public METADATA_TYPE getMetadata(@NotNull final ColumnDefinition } } final Optional tableInfo = ParquetSchemaReader.parseMetadata( - new ParquetMetadataConverter().fromParquetMetadata(parquetFileReader.fileMetaData) + new ParquetMetadataConverter().fromParquetMetadata(parquetFileReader.getFileMetaData()) .getFileMetaData().getKeyValueMetaData()); final Map columnTypes = tableInfo.map(TableInfo::columnTypeMap).orElse(Collections.emptyMap()); @@ -251,7 +252,7 @@ private REGION_TYPE makeColumnRegion( @NotNull final Function, SOURCE[]> sourceArrayFactory, @NotNull final ColumnDefinition columnDefinition, @NotNull final LongFunction nullRegionFactory, - @NotNull final Function singleRegionFactory, + @NotNull final BiFunction, REGION_TYPE> singleRegionFactory, @NotNull final Function, REGION_TYPE> multiRegionFactory) { final SOURCE[] sources = sourceArrayFactory.apply(columnDefinition); return sources.length == 1 @@ -260,11 +261,12 @@ private REGION_TYPE makeColumnRegion( source -> makeSingleColumnRegion(source, nullRegionFactory, singleRegionFactory))); } - private REGION_TYPE makeSingleColumnRegion(final SOURCE source, + private REGION_TYPE makeSingleColumnRegion( + final SOURCE source, @NotNull final LongFunction nullRegionFactory, - @NotNull final Function singleRegionFactory) { + @NotNull final BiFunction, REGION_TYPE> singleRegionFactory) { return source == null ? nullRegionFactory.apply(tl().getRegionParameters().regionMask) - : singleRegionFactory.apply(source); + : singleRegionFactory.apply(source, this.cast()); } @Override @@ -273,8 +275,8 @@ public ColumnRegionChar makeColumnRegionChar( // noinspection unchecked return (ColumnRegionChar) makeColumnRegion(this::getPageStores, columnDefinition, ColumnRegionChar::createNull, ParquetColumnRegionChar::new, - rs -> new ColumnRegionChar.StaticPageStore<>(tl().getRegionParameters(), - rs.toArray(ColumnRegionChar[]::new))); + rs -> new ColumnRegionChar.StaticPageStore(tl().getRegionParameters(), + rs.toArray(ColumnRegionChar[]::new), this.cast())); } @Override @@ -284,7 +286,7 @@ public ColumnRegionByte makeColumnRegionByte( return (ColumnRegionByte) makeColumnRegion(this::getPageStores, columnDefinition, ColumnRegionByte::createNull, ParquetColumnRegionByte::new, rs -> new ColumnRegionByte.StaticPageStore<>(tl().getRegionParameters(), - rs.toArray(ColumnRegionByte[]::new))); + rs.toArray(ColumnRegionByte[]::new), this)); } @Override @@ -294,7 +296,7 @@ public ColumnRegionShort makeColumnRegionShort( return (ColumnRegionShort) makeColumnRegion(this::getPageStores, columnDefinition, ColumnRegionShort::createNull, ParquetColumnRegionShort::new, rs -> new ColumnRegionShort.StaticPageStore<>(tl().getRegionParameters(), - rs.toArray(ColumnRegionShort[]::new))); + rs.toArray(ColumnRegionShort[]::new), this)); } @Override @@ -304,7 +306,7 @@ public ColumnRegionInt makeColumnRegionInt( return (ColumnRegionInt) makeColumnRegion(this::getPageStores, columnDefinition, ColumnRegionInt::createNull, ParquetColumnRegionInt::new, rs -> new ColumnRegionInt.StaticPageStore<>(tl().getRegionParameters(), - rs.toArray(ColumnRegionInt[]::new))); + rs.toArray(ColumnRegionInt[]::new), this)); } @Override @@ -314,7 +316,7 @@ public ColumnRegionLong makeColumnRegionLong( return (ColumnRegionLong) makeColumnRegion(this::getPageStores, columnDefinition, ColumnRegionLong::createNull, ParquetColumnRegionLong::new, rs -> new ColumnRegionLong.StaticPageStore<>(tl().getRegionParameters(), - rs.toArray(ColumnRegionLong[]::new))); + rs.toArray(ColumnRegionLong[]::new), this)); } @Override @@ -324,7 +326,7 @@ public ColumnRegionFloat makeColumnRegionFloat( return (ColumnRegionFloat) makeColumnRegion(this::getPageStores, columnDefinition, ColumnRegionFloat::createNull, ParquetColumnRegionFloat::new, rs -> new ColumnRegionFloat.StaticPageStore<>(tl().getRegionParameters(), - rs.toArray(ColumnRegionFloat[]::new))); + rs.toArray(ColumnRegionFloat[]::new), this)); } @Override @@ -334,7 +336,7 @@ public ColumnRegionDouble makeColumnRegionDouble( return (ColumnRegionDouble) makeColumnRegion(this::getPageStores, columnDefinition, ColumnRegionDouble::createNull, ParquetColumnRegionDouble::new, rs -> new ColumnRegionDouble.StaticPageStore<>(tl().getRegionParameters(), - rs.toArray(ColumnRegionDouble[]::new))); + rs.toArray(ColumnRegionDouble[]::new), this)); } @Override @@ -357,7 +359,8 @@ public ColumnRegionObject makeColumnRegionObject( IntStream.range(0, sources.length) .mapToObj(ri -> makeSingleColumnRegionObject(dataType, sources[ri], dictKeySources[ri], dictionaryChunkSuppliers[ri])) - .toArray(ColumnRegionObject[]::new)); + .toArray(ColumnRegionObject[]::new), + this); } private ColumnRegionObject makeSingleColumnRegionObject( @@ -369,9 +372,11 @@ private ColumnRegionObject makeSingleColumnRegionObject( return ColumnRegionObject.createNull(tl().getRegionParameters().regionMask); } return new ParquetColumnRegionObject<>(source, - () -> new ParquetColumnRegionLong<>(Require.neqNull(dictKeySource, "dictKeySource")), + () -> new ParquetColumnRegionLong<>(Require.neqNull(dictKeySource, "dictKeySource"), + this.cast()), () -> ColumnRegionChunkDictionary.create(tl().getRegionParameters().regionMask, - dataType, Require.neqNull(dictValuesSupplier, "dictValuesSupplier"))); + dataType, Require.neqNull(dictValuesSupplier, "dictValuesSupplier"), this), + this.cast()); } /** diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java index 88d2ae87acc..975d5bd65be 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java @@ -3,12 +3,15 @@ */ package io.deephaven.parquet.table.location; +import io.deephaven.api.SortColumn; +import io.deephaven.engine.table.Table; import io.deephaven.engine.table.impl.locations.TableKey; import io.deephaven.engine.table.impl.locations.impl.AbstractTableLocation; import io.deephaven.parquet.table.ParquetInstructions; import io.deephaven.parquet.table.ParquetSchemaReader; import io.deephaven.parquet.table.metadata.ColumnTypeInfo; import io.deephaven.parquet.table.metadata.GroupingColumnInfo; +import io.deephaven.parquet.table.metadata.SortColumnInfo; import io.deephaven.parquet.table.metadata.TableInfo; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.table.impl.sources.regioned.RegionedColumnSource; @@ -24,9 +27,11 @@ import org.apache.parquet.format.RowGroup; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.io.File; import java.util.*; +import java.util.stream.Collectors; import java.util.stream.IntStream; public class ParquetTableLocation extends AbstractTableLocation { @@ -42,6 +47,8 @@ public class ParquetTableLocation extends AbstractTableLocation { private final Map parquetColumnNameToPath; private final Map groupingColumns; private final Map columnTypes; + private final List sortingColumns; + private final TableInfo tableInfo; private final String version; private volatile RowGroupReader[] rowGroupReaders; @@ -61,7 +68,7 @@ public ParquetTableLocation(@NotNull final TableKey tableKey, final int rowGroupCount = rowGroupIndices.length; rowGroups = IntStream.of(rowGroupIndices) - .mapToObj(rgi -> parquetFileReader.fileMetaData.getRow_groups().get(rgi)) + .mapToObj(rgi -> parquetFileReader.getFileMetaData().getRow_groups().get(rgi)) .sorted(Comparator.comparingInt(RowGroup::getOrdinal)) .toArray(RowGroup[]::new); final long maxRowCount = Arrays.stream(rowGroups).mapToLong(RowGroup::getNum_rows).max().orElse(0L); @@ -81,11 +88,14 @@ public ParquetTableLocation(@NotNull final TableKey tableKey, // in order to read *this* file's metadata, rather than inheriting file metadata from the _metadata file. // Obvious issues included grouping table paths, codecs, etc. // Presumably, we could store per-file instances of the metadata in the _metadata file's map. - final Optional tableInfo = - ParquetSchemaReader.parseMetadata(parquetMetadata.getFileMetaData().getKeyValueMetaData()); - groupingColumns = tableInfo.map(TableInfo::groupingColumnMap).orElse(Collections.emptyMap()); - columnTypes = tableInfo.map(TableInfo::columnTypeMap).orElse(Collections.emptyMap()); - version = tableInfo.map(TableInfo::version).orElse(null); + tableInfo = ParquetSchemaReader.parseMetadata(parquetMetadata.getFileMetaData().getKeyValueMetaData()) + .orElse(TableInfo.builder().build()); + groupingColumns = tableInfo.groupingColumnMap(); + columnTypes = tableInfo.columnTypeMap(); + version = tableInfo.version(); + sortingColumns = tableInfo.sortingColumns().stream() + .map(SortColumnInfo::toSortColumn) + .collect(Collectors.toList()); handleUpdate(computeIndex(), tableLocationKey.getFile().lastModified()); } @@ -115,11 +125,11 @@ RegionedPageStore.Parameters getRegionParameters() { } public Map getGroupingColumns() { - return groupingColumns; + return tableInfo.groupingColumnMap(); } public Map getColumnTypes() { - return columnTypes; + return tableInfo.columnTypeMap(); } private RowGroupReader[] getRowGroupReaders() { @@ -138,6 +148,12 @@ private RowGroupReader[] getRowGroupReaders() { } } + @NotNull + @Override + public List getSortedColumns() { + return sortingColumns; + } + @NotNull @Override protected ParquetColumnLocation makeColumnLocation(@NotNull final String columnName) { @@ -148,9 +164,9 @@ protected ParquetColumnLocation makeColumnLocation(@NotNull final String final ColumnChunkReader[] columnChunkReaders = Arrays.stream(getRowGroupReaders()) .map(rgr -> rgr.getColumnChunk(nameList)).toArray(ColumnChunkReader[]::new); final boolean exists = Arrays.stream(columnChunkReaders).anyMatch(ccr -> ccr != null && ccr.numRows() > 0); + // TODO NATE NOCOMMIT? return new ParquetColumnLocation<>(this, columnName, parquetColumnName, - exists ? columnChunkReaders : null, - exists && groupingColumns.containsKey(parquetColumnName)); + exists ? columnChunkReaders : null, false); } private RowSet computeIndex() { @@ -164,4 +180,15 @@ private RowSet computeIndex() { } return sequentialBuilder.build(); } + + @Override + public boolean hasDataIndexFor(@NotNull String... columns) { + return false; + } + + @Override + protected @Nullable Table getDataIndexImpl(@NotNull final String... columns) { + // TODO NATE NOCOMMIT + return null; // ParquetTools.readDataIndexTable(getParquetFile(), tableInfo, columns); + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java index 1925250eb8e..e2794a54117 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java @@ -125,7 +125,7 @@ public synchronized ParquetMetadata getMetadata() { return metadata; } try { - return metadata = new ParquetMetadataConverter().fromParquetMetadata(getFileReader().fileMetaData); + return metadata = new ParquetMetadataConverter().fromParquetMetadata(getFileReader().getFileMetaData()); } catch (IOException e) { throw new TableDataException("Failed to convert Parquet file metadata: " + getFile(), e); } @@ -151,7 +151,7 @@ public synchronized int[] getRowGroupIndices() { if (rowGroupIndices != null) { return rowGroupIndices; } - final List rowGroups = getFileReader().fileMetaData.getRow_groups(); + final List rowGroups = getFileReader().getFileMetaData().getRow_groups(); return rowGroupIndices = IntStream.range(0, rowGroups.size()).filter(rgi -> { // 1. We can safely assume there's always at least one column. Our tools will refuse to write a // column-less table, and other readers we've tested fail catastrophically. diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/metadata/DataIndexInfo.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/metadata/DataIndexInfo.java new file mode 100644 index 00000000000..4c5533c100c --- /dev/null +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/metadata/DataIndexInfo.java @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.parquet.table.metadata; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import io.deephaven.annotations.SimpleStyle; +import io.deephaven.engine.util.string.StringUtils; +import org.immutables.value.Value; +import org.jetbrains.annotations.NotNull; + +import java.util.Arrays; +import java.util.List; + +/** + * Representation class for grouping column information stored in key-value metadata for Deephaven-written Parquet + * files. + */ +@Value.Immutable +@SimpleStyle +@JsonSerialize(as = ImmutableDataIndexInfo.class) +@JsonDeserialize(as = ImmutableDataIndexInfo.class) +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public abstract class DataIndexInfo { + + /** + * @return The column name + */ + @Value.Parameter + public abstract List columns(); + + /** + * @return The relative path name for the column's grouping sidecar table + */ + @Value.Parameter + public abstract String indexTablePath(); + + @Value.Check + final void checkColumns() { + if (columns().isEmpty()) { + throw new IllegalArgumentException("No columns provided"); + } else if (columns().stream().anyMatch(StringUtils::isNullOrEmpty)) { + throw new IllegalArgumentException("Empty column name"); + } + } + + @Value.Check + final void checkIndexTablePath() { + if (indexTablePath().isEmpty()) { + throw new IllegalArgumentException("Empty grouping table path"); + } + } + + public boolean matchesColumns(final String... columnsToMatch) { + Arrays.sort(columnsToMatch); + return Arrays.asList(columnsToMatch).equals(columns()); + } + + public static DataIndexInfo of(@NotNull final String indexTablePath, final String... columnNames) { + Arrays.sort(columnNames); + return ImmutableDataIndexInfo.of(Arrays.asList(columnNames), indexTablePath); + } +} diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/metadata/SortColumnInfo.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/metadata/SortColumnInfo.java new file mode 100644 index 00000000000..a0b01bcbdd3 --- /dev/null +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/metadata/SortColumnInfo.java @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.parquet.table.metadata; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import io.deephaven.annotations.BuildableStyle; +import io.deephaven.api.ColumnName; +import io.deephaven.api.SortColumn; +import org.immutables.value.Value; +import org.jetbrains.annotations.NotNull; + +@Value.Immutable +@BuildableStyle +@JsonSerialize(as = ImmutableSortColumnInfo.class) +@JsonDeserialize(as = ImmutableSortColumnInfo.class) +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public abstract class SortColumnInfo { + public enum SortDirection { + Ascending, Descending + } + + public abstract String columnName(); + + public abstract SortDirection sortDirection(); + + // @Value.Check + final void checkColumnName() { + if (columnName().isEmpty()) { + throw new IllegalArgumentException("Empty column name"); + } + } + + public final SortColumn toSortColumn() { + ColumnName cname = ColumnName.of(columnName()); + return sortDirection() == SortDirection.Ascending ? SortColumn.asc(cname) : SortColumn.desc(cname); + } + + public static SortColumnInfo of(@NotNull final SortColumn sortingColumn) { + return builder() + .columnName(sortingColumn.column().name()) + .sortDirection(sortingColumn.order() == SortColumn.Order.ASCENDING + ? SortColumnInfo.SortDirection.Ascending + : SortColumnInfo.SortDirection.Descending) + .build(); + } + + public static Builder builder() { + return ImmutableSortColumnInfo.builder(); + } + + public interface Builder { + + Builder columnName(String columnName); + + Builder sortDirection(SortDirection sortDirection); + + SortColumnInfo build(); + } +} diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/metadata/TableInfo.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/metadata/TableInfo.java index fb6b8da2002..8e2b7f5d8a6 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/metadata/TableInfo.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/metadata/TableInfo.java @@ -84,6 +84,10 @@ public String version() { */ public abstract List columnTypes(); + public abstract List sortingColumns(); + + public abstract List dataIndexes(); + @Value.Check final void checkVersion() { if (version().isEmpty()) { @@ -111,6 +115,18 @@ public interface Builder { Builder addAllColumnTypes(Iterable columnTypes); + Builder addSortingColumns(SortColumnInfo sortPair); + + Builder addSortingColumns(SortColumnInfo... sortPairs); + + Builder addAllSortingColumns(Iterable sortPairs); + + Builder addDataIndexes(DataIndexInfo info); + + Builder addDataIndexes(DataIndexInfo... infos); + + Builder addAllDataIndexes(Iterable infos); + TableInfo build(); } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java index dcd7677ccf5..910b8135cae 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/ColumnChunkPageStore.java @@ -18,6 +18,7 @@ import io.deephaven.parquet.base.ColumnPageReader; import io.deephaven.util.SafeCloseable; import io.deephaven.vector.Vector; +import org.apache.parquet.column.statistics.Statistics; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.VisibleForTesting; @@ -154,8 +155,16 @@ public ChunkType getChunkType() { return toPage.getChunkType(); } + public boolean hasStatistics() { + return columnChunkReader.hasStatistics(); + } + + public , S extends Statistics> S getStatistics() { + return columnChunkReader.getStatistics(); + } + /** - * These implementations don't use the FillContext parameter, so we're create a helper method to ignore it. + * These implementations don't use the FillContext parameter, so we create a helper method to ignore it. */ @NotNull public ChunkPage getPageContaining(final long row) { @@ -171,4 +180,8 @@ public boolean usesDictionaryOnEveryPage() { @Override public void close() {} + + public T convertSingleValue(@NotNull final Object toConvert) { + return toPage.convertSingleResult(toConvert); + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToArrayPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToArrayPage.java index 667c119e1f1..ee7cde9e8b4 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToArrayPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToArrayPage.java @@ -54,10 +54,10 @@ public final ChunkType getChunkType() { @NotNull @Override - public final ARRAY_TYPE[] convertResult(Object object) { + public final ARRAY_TYPE[] convertResultArray(@NotNull final Object object) { final DataWithOffsets dataWithOffsets = (DataWithOffsets) object; - final Vector dataWrapper = toPage.makeVector(toPage.convertResult(dataWithOffsets.materializeResult)); + final Vector dataWrapper = toPage.makeVector(toPage.convertResultArray(dataWithOffsets.materializeResult)); final IntBuffer offsets = dataWithOffsets.offsets; // noinspection unchecked diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBigDecimalFromIntPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBigDecimalFromIntPage.java index c50a5187a05..36903956e2a 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBigDecimalFromIntPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBigDecimalFromIntPage.java @@ -25,7 +25,7 @@ protected ToBigDecimalFromIntPage(@NotNull final Class nativeType, final int } @Override - public BigDecimal[] convertResult(@NotNull final Object result) { + public BigDecimal[] convertResultArray(@NotNull final Object result) { final int[] in = (int[]) result; final int resultLength = in.length; final BigDecimal[] out = new BigDecimal[resultLength]; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBigDecimalFromLongPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBigDecimalFromLongPage.java index 392dc956e25..4d358924b86 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBigDecimalFromLongPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBigDecimalFromLongPage.java @@ -25,7 +25,7 @@ protected ToBigDecimalFromLongPage(@NotNull final Class nativeType, final int } @Override - public BigDecimal[] convertResult(@NotNull final Object result) { + public BigDecimal[] convertResultArray(@NotNull final Object result) { final long[] in = (long[]) result; final int resultLength = in.length; final BigDecimal[] out = new BigDecimal[resultLength]; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBooleanAsBytePage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBooleanAsBytePage.java index aea41095f12..936cf4ffba6 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBooleanAsBytePage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBooleanAsBytePage.java @@ -53,7 +53,7 @@ public final Object nullValue() { @Override @NotNull - public ObjectVector makeVector(byte[] result) { + public ObjectVector makeVector(@NotNull byte[] result) { Boolean[] to = new Boolean[result.length]; for (int i = 0; i < result.length; ++i) { diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBytePageFromInt.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBytePageFromInt.java index 5cdb88ca1c0..4d37e31ea78 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBytePageFromInt.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToBytePageFromInt.java @@ -51,7 +51,7 @@ public final Object nullValue() { @Override @NotNull - public final byte[] convertResult(Object result) { + public final byte[] convertResultArray(@NotNull final Object result) { int [] from = (int []) result; byte [] to = new byte [from.length]; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToCharPageFromInt.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToCharPageFromInt.java index 3a6326c9e1f..fb165902197 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToCharPageFromInt.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToCharPageFromInt.java @@ -46,7 +46,7 @@ public final Object nullValue() { @Override @NotNull - public final char[] convertResult(Object result) { + public final char[] convertResultArray(@NotNull final Object result) { int [] from = (int []) result; char [] to = new char [from.length]; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToDatePageFromInt.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToDatePageFromInt.java index eec5e11c596..be0acb3c28d 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToDatePageFromInt.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToDatePageFromInt.java @@ -45,15 +45,15 @@ public final Object nullValue() { return NULL_INT_BOXED; } - @Override - @NotNull - public final LocalDate[] convertResult(final Object result) { - final int[] from = (int[]) result; - final LocalDate[] to = new LocalDate[from.length]; - - for (int i = 0; i < from.length; ++i) { - to[i] = DateTimeUtils.epochDaysAsIntToLocalDate(from[i]); - } - return to; - } +// @Override +// @NotNull +// public final LocalDate[] convertResult(final Object result) { +// final int[] from = (int[]) result; +// final LocalDate[] to = new LocalDate[from.length]; +// +// for (int i = 0; i < from.length; ++i) { +// to[i] = DateTimeUtils.epochDaysAsIntToLocalDate(from[i]); +// } +// return to; +// } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToInstantPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToInstantPage.java index 97522a6c84a..41e156968e7 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToInstantPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToInstantPage.java @@ -73,7 +73,7 @@ public final Class getNativeComponentType() { private static final class ToInstantPageFromNanos extends ToInstantPage { @Override @NotNull - public ObjectVector makeVector(long[] result) { + public ObjectVector makeVector(@NotNull long[] result) { return makeVectorHelper(result, DateTimeUtils::epochNanosToInstant); } } @@ -81,12 +81,12 @@ public ObjectVector makeVector(long[] result) { private static final class ToInstantPageFromMicros extends ToInstantPage { @Override @NotNull - public ObjectVector makeVector(long[] result) { + public ObjectVector makeVector(@NotNull long[] result) { return makeVectorHelper(result, DateTimeUtils::epochMicrosToInstant); } @Override - public long[] convertResult(@NotNull final Object result) { + public long[] convertResultArray(@NotNull final Object result) { return convertResultHelper(result, DateTimeUtils::microsToNanos); } } @@ -94,12 +94,12 @@ public long[] convertResult(@NotNull final Object result) { private static final class ToInstantPageFromMillis extends ToInstantPage { @Override @NotNull - public ObjectVector makeVector(long[] result) { + public ObjectVector makeVector(@NotNull long[] result) { return makeVectorHelper(result, DateTimeUtils::epochMillisToInstant); } @Override - public long[] convertResult(@NotNull final Object result) { + public long[] convertResultArray(@NotNull final Object result) { return convertResultHelper(result, DateTimeUtils::millisToNanos); } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToInstantPageFromInt96.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToInstantPageFromInt96.java index 16d2ca9e397..49d7d68f292 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToInstantPageFromInt96.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToInstantPageFromInt96.java @@ -86,7 +86,7 @@ public final Class getNativeComponentType() { } @Override - public final long[] convertResult(@NotNull final Object result) { + public final long[] convertResultArray(@NotNull final Object result) { // result is delivered as an array of Binary[12] final Binary[] results = (Binary[])result; final int resultLength = results.length; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToLocalDateTimePage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToLocalDateTimePage.java index 356d547c74e..e3cdd70ca12 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToLocalDateTimePage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToLocalDateTimePage.java @@ -77,24 +77,24 @@ private static LocalDateTime[] convertResultHelper(@NotNull final Object result, } private static final class ToLocalDateTimePageFromMillis extends ToLocalDateTimePage { - @Override - public LocalDateTime[] convertResult(@NotNull final Object result) { - return convertResultHelper(result, TransferUtils::epochMillisToLocalDateTimeUTC); - } +// @Override +// public LocalDateTime[] convertResult(@NotNull final Object result) { +// return convertResultHelper(result, TransferUtils::epochMillisToLocalDateTimeUTC); +// } } private static final class ToLocalDateTimePageFromMicros extends ToLocalDateTimePage { - @Override - public LocalDateTime[] convertResult(@NotNull final Object result) { - return convertResultHelper(result, TransferUtils::epochMicrosToLocalDateTimeUTC); - } +// @Override +// public LocalDateTime[] convertResult(@NotNull final Object result) { +// return convertResultHelper(result, TransferUtils::epochMicrosToLocalDateTimeUTC); +// } } private static final class ToLocalDateTimePageFromNanos extends ToLocalDateTimePage { - @Override - public LocalDateTime[] convertResult(@NotNull final Object result) { - return convertResultHelper(result, TransferUtils::epochNanosToLocalDateTimeUTC); - } +// @Override +// public LocalDateTime[] convertResult(@NotNull final Object result) { +// return convertResultHelper(result, TransferUtils::epochNanosToLocalDateTimeUTC); +// } } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToObjectPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToObjectPage.java index 5b7dd26c1e9..4b804f4b14c 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToObjectPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToObjectPage.java @@ -9,6 +9,7 @@ import org.apache.parquet.column.Dictionary; import org.apache.parquet.io.api.Binary; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.lang.reflect.Array; import java.util.function.Supplier; @@ -58,10 +59,20 @@ public final ChunkType getChunkType() { @Override @NotNull - public final T[] convertResult(Object result) { + public final T[] convertResultArray(@NotNull final Object result) { return convertResult(nativeType, codec, result); } + @Override + public T convertSingleResult(@Nullable final Object result) { + if (result == null) { + return null; + } else { + final byte[] resultBytes = ((Binary) result).getBytes(); + return codec.decode(resultBytes, 0,resultBytes.length); + } + } + private static T2[] convertResult(final Class nativeType, final ObjectCodec codec, final Object result) { Binary[] from = (Binary[]) result; //noinspection unchecked diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToPage.java index a395c1a09da..8c02508eea4 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToPage.java @@ -15,6 +15,7 @@ import io.deephaven.util.annotations.FinalDefault; import io.deephaven.parquet.base.ColumnPageReader; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.io.IOException; @@ -53,22 +54,30 @@ default Object nullValue() { /** * @return Gets the result from the columnPageReader. */ - default Object getResult(ColumnPageReader columnPageReader) throws IOException { + default Object getResult(@NotNull final ColumnPageReader columnPageReader) throws IOException { return columnPageReader.materialize(nullValue()); } /** * @return Produce the array of values from the result */ - default RESULT convertResult(Object result) { + default RESULT convertResultArray(@NotNull Object result) { // noinspection unchecked return (RESULT) result; } + /** + * @return Produce the value from the result + */ + default R convertSingleResult(@Nullable final Object result) { + // noinspection unchecked + return (R) result; + } + /** * @return the method to create a Vector from RESULT. */ - default Vector makeVector(RESULT result) { + default Vector makeVector(@NotNull RESULT result) { return VectorFactory.forElementType(getNativeType()).vectorWrap(result); } @@ -78,10 +87,10 @@ default Vector makeVector(RESULT result) { */ @NotNull @FinalDefault - default ChunkPage toPage(long offset, ColumnPageReader columnPageReader, long mask) + default ChunkPage toPage(final long offset, @NotNull final ColumnPageReader columnPageReader, final long mask) throws IOException { return ChunkPageFactory.forChunkType(getChunkType()) - .pageWrap(offset, convertResult(getResult(columnPageReader)), mask); + .pageWrap(offset, convertResultArray(getResult(columnPageReader)), mask); } /** @@ -124,12 +133,12 @@ public Object nullValue() { @NotNull @Override - public Object getResult(ColumnPageReader columnPageReader) throws IOException { + public Object getResult(@NotNull final ColumnPageReader columnPageReader) throws IOException { return toPage.getResult(columnPageReader); } @Override - public abstract OUTER_RESULT convertResult(Object object); + public abstract OUTER_RESULT convertResultArray(@NotNull final Object object); @Override diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToPageWithDictionary.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToPageWithDictionary.java index 35258d16cd7..89e9c4e7aa5 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToPageWithDictionary.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToPageWithDictionary.java @@ -12,6 +12,7 @@ import io.deephaven.parquet.base.ColumnPageReader; import io.deephaven.parquet.base.DataWithOffsets; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.lang.reflect.Array; @@ -65,7 +66,7 @@ public final Object getResult(@NotNull final ColumnPageReader columnPageReader) @Override @NotNull - public final DATA_TYPE[] convertResult(@NotNull final Object result) { + public final DATA_TYPE[] convertResultArray(@NotNull final Object result) { if (!(result instanceof int[])) { return convertResultFallbackFun.apply(result); } @@ -81,6 +82,12 @@ public final DATA_TYPE[] convertResult(@NotNull final Object result) { return to; } + @SuppressWarnings("unchecked") + @Override + public Integer convertSingleResult(@Nullable final Object result) { + return result == null ? Integer.valueOf(NULL_INT) : (Integer) chunkDictionary.get((int) result); + } + @Override @NotNull public final ObjectChunk getDictionaryChunk() { @@ -121,7 +128,7 @@ public Object getResult(@NotNull final ColumnPageReader columnPageReader) } @Override - public long[] convertResult(@NotNull final Object result) { + public long[] convertResultArray(@NotNull final Object result) { final int[] from = (int[]) result; final long[] to = new long[from.length]; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToShortPageFromInt.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToShortPageFromInt.java index 29c70cbdecf..10a7585a5af 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToShortPageFromInt.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToShortPageFromInt.java @@ -51,7 +51,7 @@ public final Object nullValue() { @Override @NotNull - public final short[] convertResult(Object result) { + public final short[] convertResultArray(@NotNull final Object result) { int [] from = (int []) result; short [] to = new short [from.length]; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToStringPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToStringPage.java index b9a43add297..ae42304693e 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToStringPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToStringPage.java @@ -8,6 +8,7 @@ import org.apache.parquet.column.Dictionary; import org.apache.parquet.io.api.Binary; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.util.function.Supplier; @@ -26,7 +27,7 @@ public static ToPage create( new ChunkDictionary<>( (dictionary, key) -> dictionary.decodeToBinary(key).toStringUsingUTF8(), dictionarySupplier), - INSTANCE::convertResult); + INSTANCE::convertResultArray); } throw new IllegalArgumentException( @@ -49,7 +50,7 @@ public final ChunkType getChunkType() { @Override @NotNull - public final String[] convertResult(final Object result) { + public final String[] convertResultArray(@NotNull final Object result) { final Binary[] from = (Binary[]) result; final String[] to = new String[from.length]; for (int ri = 0; ri < to.length; ++ri) { @@ -59,4 +60,10 @@ public final String[] convertResult(final Object result) { } return to; } + + @SuppressWarnings("unchecked") + @Override + public String convertSingleResult(@Nullable final Object result) { + return result == null ? null : ((Binary) result).toStringUsingUTF8(); + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToStringSetPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToStringSetPage.java index 66166a6f9da..b1edc1405ea 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToStringSetPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToStringSetPage.java @@ -12,6 +12,7 @@ import io.deephaven.parquet.base.ColumnPageReader; import io.deephaven.parquet.base.DataWithOffsets; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.nio.IntBuffer; @@ -56,9 +57,9 @@ public final ChunkType getChunkType() { @Override @NotNull - public final StringSet[] convertResult(Object result) { + public final StringSet[] convertResultArray(@NotNull final Object result) { DataWithOffsets dataWithOffsets = (DataWithOffsets) result; - String[] from = (String[]) toPage.convertResult(dataWithOffsets.materializeResult); + String[] from = (String[]) toPage.convertResultArray(dataWithOffsets.materializeResult); IntBuffer offsets = dataWithOffsets.offsets; StringSet[] to = new StringSet[offsets.remaining()]; @@ -77,6 +78,11 @@ public final StringSet[] convertResult(Object result) { return to; } + @Override + public R convertSingleResult(@Nullable Object result) { + throw new UnsupportedOperationException("Not implemented"); + } + private static final class WithShortDictionary extends ToPage.Wrap { @@ -86,25 +92,25 @@ private static final class WithShortDictionary @Override @NotNull - public final Class getNativeType() { + public Class getNativeType() { return StringSet.class; } @Override @NotNull - public final ChunkType getChunkType() { + public ChunkType getChunkType() { return ChunkType.Object; } @Override @NotNull - public final Object getResult(ColumnPageReader columnPageReader) throws IOException { + public Object getResult(@NotNull final ColumnPageReader columnPageReader) throws IOException { return toPage.getDictionaryKeysToPage().getResult(columnPageReader); } @Override @NotNull - public final StringSet[] convertResult(Object result) { + public StringSet[] convertResultArray(@NotNull final Object result) { DataWithOffsets dataWithOffsets = (DataWithOffsets) result; int[] from = (int[]) dataWithOffsets.materializeResult; IntBuffer offsets = dataWithOffsets.offsets; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToTimePage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToTimePage.java index fd64896b4c8..0b3be45013e 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToTimePage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToTimePage.java @@ -64,16 +64,16 @@ public Object nullValue() { return QueryConstants.NULL_INT_BOXED; } - @Override - public LocalTime[] convertResult(@NotNull final Object result) { - final int[] from = (int[]) result; - final LocalTime[] to = new LocalTime[from.length]; - - for (int i = 0; i < from.length; ++i) { - to[i] = DateTimeUtils.millisOfDayToLocalTime(from[i]); - } - return to; - } +// @Override +// public LocalTime[] convertResult(@NotNull final Object result) { +// final int[] from = (int[]) result; +// final LocalTime[] to = new LocalTime[from.length]; +// +// for (int i = 0; i < from.length; ++i) { +// to[i] = DateTimeUtils.millisOfDayToLocalTime(from[i]); +// } +// return to; +// } } private static class ToTimePageFromLong extends ToTimePage { @@ -96,16 +96,16 @@ static LocalTime[] convertResultHelper(@NotNull final Object result, } private static final class ToTimePageFromMicros extends ToTimePageFromLong { - @Override - public LocalTime[] convertResult(@NotNull final Object result) { - return convertResultHelper(result, DateTimeUtils::microsOfDayToLocalTime); - } +// @Override +// public LocalTime[] convertResult(@NotNull final Object result) { +// return convertResultHelper(result, DateTimeUtils::microsOfDayToLocalTime); +// } } private static final class ToTimePageFromNanos extends ToTimePageFromLong { - @Override - public LocalTime[] convertResult(@NotNull final Object result) { - return convertResultHelper(result, DateTimeUtils::nanosOfDayToLocalTime); - } +// @Override +// public LocalTime[] convertResult(@NotNull final Object result) { +// return convertResultHelper(result, DateTimeUtils::nanosOfDayToLocalTime); +// } } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToVectorPage.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToVectorPage.java index e66763ce87e..3c8322a2826 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToVectorPage.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ToVectorPage.java @@ -54,12 +54,12 @@ public final ChunkType getChunkType() { @NotNull @Override - public final VECTOR_TYPE[] convertResult(final Object object) { + public final VECTOR_TYPE[] convertResultArray(@NotNull final Object object) { final DataWithOffsets dataWithOffsets = (DataWithOffsets) object; // noinspection unchecked final VECTOR_TYPE dataWrapper = - (VECTOR_TYPE) toPage.makeVector(toPage.convertResult(dataWithOffsets.materializeResult)); + (VECTOR_TYPE) toPage.makeVector(toPage.convertResultArray(dataWithOffsets.materializeResult)); final IntBuffer offsets = dataWithOffsets.offsets; // noinspection unchecked diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionBase.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionBase.java index 98377e09627..81c45e5f20b 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionBase.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionBase.java @@ -4,9 +4,15 @@ package io.deephaven.parquet.table.region; import io.deephaven.base.verify.Require; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; +import io.deephaven.engine.table.impl.chunkfilter.ChunkMatchFilterFactory; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.sources.regioned.GenericColumnRegionBase; +import io.deephaven.parquet.table.location.ParquetColumnLocation; import io.deephaven.parquet.table.pagestore.ColumnChunkPageStore; -import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.Chunk; import io.deephaven.engine.table.SharedContext; import io.deephaven.chunk.WritableChunk; @@ -16,14 +22,19 @@ import javax.annotation.OverridingMethodsMustInvokeSuper; -public abstract class ParquetColumnRegionBase +public abstract class ParquetColumnRegionBase extends GenericColumnRegionBase implements ParquetColumnRegion { + final ParquetColumnLocation location; final ColumnChunkPageStore columnChunkPageStore; - ParquetColumnRegionBase(final long pageMask, @NotNull final ColumnChunkPageStore columnChunkPageStore) { + ParquetColumnRegionBase( + final long pageMask, + @NotNull final ColumnChunkPageStore columnChunkPageStore, + @NotNull final ParquetColumnLocation location) { super(pageMask); + this.location = location; this.columnChunkPageStore = Require.neqNull(columnChunkPageStore, "columnChunkPageStore"); // We are making the following assumptions, so these basic functions are inlined rather than virtual calls. @@ -86,4 +97,28 @@ public final GetContext makeGetContext(final int chunkCapacity, final SharedCont throwIfInvalidated(); return columnChunkPageStore.makeGetContext(chunkCapacity, sharedContext); } + + @Override + public ColumnLocation getLocation() { + return location; + } + + @Override + public final boolean supportsMatching() { + return true; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // TODO: push down to fill in native type + try (final RowSet rows = rowSequence.asRowSet()) { + return ChunkFilter.applyChunkFilter(rows, this, false, + ChunkMatchFilterFactory.getChunkFilter(null, caseInsensitive, invertMatch, sortedKeys)); + } + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionByte.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionByte.java index a734279cec7..95b1ff2a3e5 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionByte.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionByte.java @@ -13,22 +13,37 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.RowSequenceFactory; +import io.deephaven.api.SortColumn; +import io.deephaven.chunk.WritableByteChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.table.impl.sort.timsort.ByteTimsortKernel; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionByte; +import io.deephaven.engine.table.impl.sources.regioned.RegionedPageStore; +import io.deephaven.engine.table.impl.sources.regioned.kernel.ByteRegionBinarySearchKernel; +import io.deephaven.parquet.table.location.ParquetColumnLocation; import io.deephaven.parquet.table.pagestore.ColumnChunkPageStore; -import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.ChunkPage; +import io.deephaven.util.QueryConstants; +import io.deephaven.util.type.ArrayTypeUtils; +import org.apache.parquet.column.statistics.IntStatistics; import org.jetbrains.annotations.NotNull; /** * {@link ColumnRegionByte} implementation for regions that support fetching primitive bytes from * {@link ColumnChunkPageStore column chunk page stores}. */ -public final class ParquetColumnRegionByte extends ParquetColumnRegionBase +public final class ParquetColumnRegionByte extends ParquetColumnRegionBase implements ColumnRegionByte, ParquetColumnRegion { - public ParquetColumnRegionByte(@NotNull final ColumnChunkPageStore columnChunkPageStore) { - super(columnChunkPageStore.mask(), columnChunkPageStore); + public ParquetColumnRegionByte( + @NotNull final ColumnChunkPageStore columnChunkPageStore, + @NotNull final ParquetColumnLocation location) { + super(columnChunkPageStore.mask(), columnChunkPageStore, location); } // region getBytes @@ -55,4 +70,105 @@ public byte getByte(final long rowKey) { throw new TableDataException("Error retrieving byte at row key " + rowKey + " from a parquet table", e); } } + + // TODO NATE NOCOMMIT: these are still useful aren't they? + public RowSet binSearchMatch( + long firstRowKey, + final long lastRowKey, + @NotNull final SortColumn sortColumn, + @NotNull final Object[] keys) { + return ByteRegionBinarySearchKernel.binarySearchMatch(this, firstRowKey, lastRowKey, sortColumn, keys); + } + + public boolean mightContain(@NotNull final Object[] keys) { + // We don't have statistics, so we need to assume there is something in the range that fits. + if (!columnChunkPageStore.hasStatistics()) { + return true; + } + + final IntStatistics stats = columnChunkPageStore.getStatistics(); + // TODO: make sure this actually does what you think it does. + if (!stats.hasNonNullValue()) { + // Statistics are incomplete, we have to assume the region might have the data. + return true; + } + + final byte[] typed = ArrayTypeUtils.getUnboxedByteArray(keys); + try (final ByteTimsortKernel.ByteSortKernelContext sortContext = + ByteTimsortKernel.createContext(typed.length)) { + sortContext.sort(WritableByteChunk.writableChunkWrap(typed)); + } + + int firstNonNullValue = 0; + if (typed[firstNonNullValue] == QueryConstants.NULL_BYTE) { + // If there were nulls in the region, we can just say 'yes' + if (stats.getNumNulls() > 0) { + return true; + } + + // If there were not, then find the last value to find that is not null + while (firstNonNullValue < typed.length && typed[firstNonNullValue] == QueryConstants.NULL_BYTE) { + firstNonNullValue++; + } + + // Everything was null, we are done. + if (firstNonNullValue >= typed.length) { + return false; + } + } + + // Look through the keys and find anything that fits in the range. + for (int ii = firstNonNullValue; ii < typed.length; ii++) { + if (typed[ii] >= stats.getMin() && typed[ii] <= stats.getMax()) { + return true; + } + } + + // Nothing matches, we can skip this region. + return false; + } + + public static final class StaticPageStore + extends RegionedPageStore.Static> + implements ColumnRegionByte { + + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionByte[] regions, + @NotNull final ParquetColumnLocation location) { + super(parameters, regions, location); + } + + @Override + public byte getByte(final long elementIndex) { + return lookupRegion(elementIndex).getByte(elementIndex); + } + + @Override + public byte getByte(@NotNull final FillContext context, final long elementIndex) { + return lookupRegion(elementIndex).getByte(context, elementIndex); + } + + // region StaticRegion.getBytes + @Override + public byte[] getBytes(long firstElementIndex, @NotNull byte[] destination, int destinationOffset, int length) { + return lookupRegion(firstElementIndex).getBytes(firstElementIndex, destination, destinationOffset, length); + } + // endregion StaticRegion.getBytes + + @Override + public void invalidate() { + + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionChar.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionChar.java index 93a08222476..04aeae880fd 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionChar.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionChar.java @@ -3,22 +3,37 @@ */ package io.deephaven.parquet.table.region; +import io.deephaven.api.SortColumn; +import io.deephaven.chunk.WritableCharChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.table.impl.sort.timsort.CharTimsortKernel; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionChar; +import io.deephaven.engine.table.impl.sources.regioned.RegionedPageStore; +import io.deephaven.engine.table.impl.sources.regioned.kernel.CharRegionBinarySearchKernel; +import io.deephaven.parquet.table.location.ParquetColumnLocation; import io.deephaven.parquet.table.pagestore.ColumnChunkPageStore; -import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.ChunkPage; +import io.deephaven.util.QueryConstants; +import io.deephaven.util.type.ArrayTypeUtils; +import org.apache.parquet.column.statistics.IntStatistics; import org.jetbrains.annotations.NotNull; /** * {@link ColumnRegionChar} implementation for regions that support fetching primitive chars from * {@link ColumnChunkPageStore column chunk page stores}. */ -public final class ParquetColumnRegionChar extends ParquetColumnRegionBase +public final class ParquetColumnRegionChar extends ParquetColumnRegionBase implements ColumnRegionChar, ParquetColumnRegion { - public ParquetColumnRegionChar(@NotNull final ColumnChunkPageStore columnChunkPageStore) { - super(columnChunkPageStore.mask(), columnChunkPageStore); + public ParquetColumnRegionChar( + @NotNull final ColumnChunkPageStore columnChunkPageStore, + @NotNull final ParquetColumnLocation location) { + super(columnChunkPageStore.mask(), columnChunkPageStore, location); } // region getBytes @@ -33,4 +48,101 @@ public char getChar(final long rowKey) { throw new TableDataException("Error retrieving char at row key " + rowKey + " from a parquet table", e); } } + + // TODO NATE NOCOMMIT: these are still useful aren't they? + public RowSet binSearchMatch( + long firstRowKey, + final long lastRowKey, + @NotNull final SortColumn sortColumn, + @NotNull final Object[] keys) { + return CharRegionBinarySearchKernel.binarySearchMatch(this, firstRowKey, lastRowKey, sortColumn, keys); + } + + public boolean mightContain(@NotNull final Object[] keys) { + // We don't have statistics, so we need to assume there is something in the range that fits. + if (!columnChunkPageStore.hasStatistics()) { + return true; + } + + final IntStatistics stats = columnChunkPageStore.getStatistics(); + // TODO: make sure this actually does what you think it does. + if (!stats.hasNonNullValue()) { + // Statistics are incomplete, we have to assume the region might have the data. + return true; + } + + final char[] typed = ArrayTypeUtils.getUnboxedCharArray(keys); + try (final CharTimsortKernel.CharSortKernelContext sortContext = + CharTimsortKernel.createContext(typed.length)) { + sortContext.sort(WritableCharChunk.writableChunkWrap(typed)); + } + + int firstNonNullValue = 0; + if (typed[firstNonNullValue] == QueryConstants.NULL_CHAR) { + // If there were nulls in the region, we can just say 'yes' + if (stats.getNumNulls() > 0) { + return true; + } + + // If there were not, then find the last value to find that is not null + while (firstNonNullValue < typed.length && typed[firstNonNullValue] == QueryConstants.NULL_CHAR) { + firstNonNullValue++; + } + + // Everything was null, we are done. + if (firstNonNullValue >= typed.length) { + return false; + } + } + + // Look through the keys and find anything that fits in the range. + for (int ii = firstNonNullValue; ii < typed.length; ii++) { + if (typed[ii] >= stats.getMin() && typed[ii] <= stats.getMax()) { + return true; + } + } + + // Nothing matches, we can skip this region. + return false; + } + + public static final class StaticPageStore + extends RegionedPageStore.Static> + implements ColumnRegionChar { + + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionChar[] regions, + @NotNull final ParquetColumnLocation location) { + super(parameters, regions, location); + } + + @Override + public char getChar(final long elementIndex) { + return lookupRegion(elementIndex).getChar(elementIndex); + } + + @Override + public char getChar(@NotNull final FillContext context, final long elementIndex) { + return lookupRegion(elementIndex).getChar(context, elementIndex); + } + + // region StaticRegion.getBytes + // endregion StaticRegion.getBytes + + @Override + public void invalidate() { + + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionDouble.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionDouble.java index 7198855c27e..84d88a1f299 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionDouble.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionDouble.java @@ -8,22 +8,37 @@ */ package io.deephaven.parquet.table.region; +import io.deephaven.api.SortColumn; +import io.deephaven.chunk.WritableDoubleChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.table.impl.sort.timsort.DoubleTimsortKernel; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionDouble; +import io.deephaven.engine.table.impl.sources.regioned.RegionedPageStore; +import io.deephaven.engine.table.impl.sources.regioned.kernel.DoubleRegionBinarySearchKernel; +import io.deephaven.parquet.table.location.ParquetColumnLocation; import io.deephaven.parquet.table.pagestore.ColumnChunkPageStore; -import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.ChunkPage; +import io.deephaven.util.QueryConstants; +import io.deephaven.util.type.ArrayTypeUtils; +import org.apache.parquet.column.statistics.DoubleStatistics; import org.jetbrains.annotations.NotNull; /** * {@link ColumnRegionDouble} implementation for regions that support fetching primitive doubles from * {@link ColumnChunkPageStore column chunk page stores}. */ -public final class ParquetColumnRegionDouble extends ParquetColumnRegionBase +public final class ParquetColumnRegionDouble extends ParquetColumnRegionBase implements ColumnRegionDouble, ParquetColumnRegion { - public ParquetColumnRegionDouble(@NotNull final ColumnChunkPageStore columnChunkPageStore) { - super(columnChunkPageStore.mask(), columnChunkPageStore); + public ParquetColumnRegionDouble( + @NotNull final ColumnChunkPageStore columnChunkPageStore, + @NotNull final ParquetColumnLocation location) { + super(columnChunkPageStore.mask(), columnChunkPageStore, location); } // region getBytes @@ -38,4 +53,101 @@ public double getDouble(final long rowKey) { throw new TableDataException("Error retrieving double at row key " + rowKey + " from a parquet table", e); } } + + // TODO NATE NOCOMMIT: these are still useful aren't they? + public RowSet binSearchMatch( + long firstRowKey, + final long lastRowKey, + @NotNull final SortColumn sortColumn, + @NotNull final Object[] keys) { + return DoubleRegionBinarySearchKernel.binarySearchMatch(this, firstRowKey, lastRowKey, sortColumn, keys); + } + + public boolean mightContain(@NotNull final Object[] keys) { + // We don't have statistics, so we need to assume there is something in the range that fits. + if (!columnChunkPageStore.hasStatistics()) { + return true; + } + + final DoubleStatistics stats = columnChunkPageStore.getStatistics(); + // TODO: make sure this actually does what you think it does. + if (!stats.hasNonNullValue()) { + // Statistics are incomplete, we have to assume the region might have the data. + return true; + } + + final double[] typed = ArrayTypeUtils.getUnboxedDoubleArray(keys); + try (final DoubleTimsortKernel.DoubleSortKernelContext sortContext = + DoubleTimsortKernel.createContext(typed.length)) { + sortContext.sort(WritableDoubleChunk.writableChunkWrap(typed)); + } + + int firstNonNullValue = 0; + if (typed[firstNonNullValue] == QueryConstants.NULL_DOUBLE) { + // If there were nulls in the region, we can just say 'yes' + if (stats.getNumNulls() > 0) { + return true; + } + + // If there were not, then find the last value to find that is not null + while (firstNonNullValue < typed.length && typed[firstNonNullValue] == QueryConstants.NULL_DOUBLE) { + firstNonNullValue++; + } + + // Everything was null, we are done. + if (firstNonNullValue >= typed.length) { + return false; + } + } + + // Look through the keys and find anything that fits in the range. + for (int ii = firstNonNullValue; ii < typed.length; ii++) { + if (typed[ii] >= stats.getMin() && typed[ii] <= stats.getMax()) { + return true; + } + } + + // Nothing matches, we can skip this region. + return false; + } + + public static final class StaticPageStore + extends RegionedPageStore.Static> + implements ColumnRegionDouble { + + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionDouble[] regions, + @NotNull final ParquetColumnLocation location) { + super(parameters, regions, location); + } + + @Override + public double getDouble(final long elementIndex) { + return lookupRegion(elementIndex).getDouble(elementIndex); + } + + @Override + public double getDouble(@NotNull final FillContext context, final long elementIndex) { + return lookupRegion(elementIndex).getDouble(context, elementIndex); + } + + // region StaticRegion.getBytes + // endregion StaticRegion.getBytes + + @Override + public void invalidate() { + + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionFloat.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionFloat.java index b01e8eab937..8977f33ccd6 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionFloat.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionFloat.java @@ -8,22 +8,37 @@ */ package io.deephaven.parquet.table.region; +import io.deephaven.api.SortColumn; +import io.deephaven.chunk.WritableFloatChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.table.impl.sort.timsort.FloatTimsortKernel; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionFloat; +import io.deephaven.engine.table.impl.sources.regioned.RegionedPageStore; +import io.deephaven.engine.table.impl.sources.regioned.kernel.FloatRegionBinarySearchKernel; +import io.deephaven.parquet.table.location.ParquetColumnLocation; import io.deephaven.parquet.table.pagestore.ColumnChunkPageStore; -import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.ChunkPage; +import io.deephaven.util.QueryConstants; +import io.deephaven.util.type.ArrayTypeUtils; +import org.apache.parquet.column.statistics.FloatStatistics; import org.jetbrains.annotations.NotNull; /** * {@link ColumnRegionFloat} implementation for regions that support fetching primitive floats from * {@link ColumnChunkPageStore column chunk page stores}. */ -public final class ParquetColumnRegionFloat extends ParquetColumnRegionBase +public final class ParquetColumnRegionFloat extends ParquetColumnRegionBase implements ColumnRegionFloat, ParquetColumnRegion { - public ParquetColumnRegionFloat(@NotNull final ColumnChunkPageStore columnChunkPageStore) { - super(columnChunkPageStore.mask(), columnChunkPageStore); + public ParquetColumnRegionFloat( + @NotNull final ColumnChunkPageStore columnChunkPageStore, + @NotNull final ParquetColumnLocation location) { + super(columnChunkPageStore.mask(), columnChunkPageStore, location); } // region getBytes @@ -38,4 +53,101 @@ public float getFloat(final long rowKey) { throw new TableDataException("Error retrieving float at row key " + rowKey + " from a parquet table", e); } } + + // TODO NATE NOCOMMIT: these are still useful aren't they? + public RowSet binSearchMatch( + long firstRowKey, + final long lastRowKey, + @NotNull final SortColumn sortColumn, + @NotNull final Object[] keys) { + return FloatRegionBinarySearchKernel.binarySearchMatch(this, firstRowKey, lastRowKey, sortColumn, keys); + } + + public boolean mightContain(@NotNull final Object[] keys) { + // We don't have statistics, so we need to assume there is something in the range that fits. + if (!columnChunkPageStore.hasStatistics()) { + return true; + } + + final FloatStatistics stats = columnChunkPageStore.getStatistics(); + // TODO: make sure this actually does what you think it does. + if (!stats.hasNonNullValue()) { + // Statistics are incomplete, we have to assume the region might have the data. + return true; + } + + final float[] typed = ArrayTypeUtils.getUnboxedFloatArray(keys); + try (final FloatTimsortKernel.FloatSortKernelContext sortContext = + FloatTimsortKernel.createContext(typed.length)) { + sortContext.sort(WritableFloatChunk.writableChunkWrap(typed)); + } + + int firstNonNullValue = 0; + if (typed[firstNonNullValue] == QueryConstants.NULL_FLOAT) { + // If there were nulls in the region, we can just say 'yes' + if (stats.getNumNulls() > 0) { + return true; + } + + // If there were not, then find the last value to find that is not null + while (firstNonNullValue < typed.length && typed[firstNonNullValue] == QueryConstants.NULL_FLOAT) { + firstNonNullValue++; + } + + // Everything was null, we are done. + if (firstNonNullValue >= typed.length) { + return false; + } + } + + // Look through the keys and find anything that fits in the range. + for (int ii = firstNonNullValue; ii < typed.length; ii++) { + if (typed[ii] >= stats.getMin() && typed[ii] <= stats.getMax()) { + return true; + } + } + + // Nothing matches, we can skip this region. + return false; + } + + public static final class StaticPageStore + extends RegionedPageStore.Static> + implements ColumnRegionFloat { + + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionFloat[] regions, + @NotNull final ParquetColumnLocation location) { + super(parameters, regions, location); + } + + @Override + public float getFloat(final long elementIndex) { + return lookupRegion(elementIndex).getFloat(elementIndex); + } + + @Override + public float getFloat(@NotNull final FillContext context, final long elementIndex) { + return lookupRegion(elementIndex).getFloat(context, elementIndex); + } + + // region StaticRegion.getBytes + // endregion StaticRegion.getBytes + + @Override + public void invalidate() { + + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionInt.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionInt.java index fbc6d5ab223..c019c35ceed 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionInt.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionInt.java @@ -8,22 +8,37 @@ */ package io.deephaven.parquet.table.region; +import io.deephaven.api.SortColumn; +import io.deephaven.chunk.WritableIntChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.table.impl.sort.timsort.IntTimsortKernel; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionInt; +import io.deephaven.engine.table.impl.sources.regioned.RegionedPageStore; +import io.deephaven.engine.table.impl.sources.regioned.kernel.IntRegionBinarySearchKernel; +import io.deephaven.parquet.table.location.ParquetColumnLocation; import io.deephaven.parquet.table.pagestore.ColumnChunkPageStore; -import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.ChunkPage; +import io.deephaven.util.QueryConstants; +import io.deephaven.util.type.ArrayTypeUtils; +import org.apache.parquet.column.statistics.IntStatistics; import org.jetbrains.annotations.NotNull; /** * {@link ColumnRegionInt} implementation for regions that support fetching primitive ints from * {@link ColumnChunkPageStore column chunk page stores}. */ -public final class ParquetColumnRegionInt extends ParquetColumnRegionBase +public final class ParquetColumnRegionInt extends ParquetColumnRegionBase implements ColumnRegionInt, ParquetColumnRegion { - public ParquetColumnRegionInt(@NotNull final ColumnChunkPageStore columnChunkPageStore) { - super(columnChunkPageStore.mask(), columnChunkPageStore); + public ParquetColumnRegionInt( + @NotNull final ColumnChunkPageStore columnChunkPageStore, + @NotNull final ParquetColumnLocation location) { + super(columnChunkPageStore.mask(), columnChunkPageStore, location); } // region getBytes @@ -38,4 +53,101 @@ public int getInt(final long rowKey) { throw new TableDataException("Error retrieving int at row key " + rowKey + " from a parquet table", e); } } + + // TODO NATE NOCOMMIT: these are still useful aren't they? + public RowSet binSearchMatch( + long firstRowKey, + final long lastRowKey, + @NotNull final SortColumn sortColumn, + @NotNull final Object[] keys) { + return IntRegionBinarySearchKernel.binarySearchMatch(this, firstRowKey, lastRowKey, sortColumn, keys); + } + + public boolean mightContain(@NotNull final Object[] keys) { + // We don't have statistics, so we need to assume there is something in the range that fits. + if (!columnChunkPageStore.hasStatistics()) { + return true; + } + + final IntStatistics stats = columnChunkPageStore.getStatistics(); + // TODO: make sure this actually does what you think it does. + if (!stats.hasNonNullValue()) { + // Statistics are incomplete, we have to assume the region might have the data. + return true; + } + + final int[] typed = ArrayTypeUtils.getUnboxedIntArray(keys); + try (final IntTimsortKernel.IntSortKernelContext sortContext = + IntTimsortKernel.createContext(typed.length)) { + sortContext.sort(WritableIntChunk.writableChunkWrap(typed)); + } + + int firstNonNullValue = 0; + if (typed[firstNonNullValue] == QueryConstants.NULL_INT) { + // If there were nulls in the region, we can just say 'yes' + if (stats.getNumNulls() > 0) { + return true; + } + + // If there were not, then find the last value to find that is not null + while (firstNonNullValue < typed.length && typed[firstNonNullValue] == QueryConstants.NULL_INT) { + firstNonNullValue++; + } + + // Everything was null, we are done. + if (firstNonNullValue >= typed.length) { + return false; + } + } + + // Look through the keys and find anything that fits in the range. + for (int ii = firstNonNullValue; ii < typed.length; ii++) { + if (typed[ii] >= stats.getMin() && typed[ii] <= stats.getMax()) { + return true; + } + } + + // Nothing matches, we can skip this region. + return false; + } + + public static final class StaticPageStore + extends RegionedPageStore.Static> + implements ColumnRegionInt { + + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionInt[] regions, + @NotNull final ParquetColumnLocation location) { + super(parameters, regions, location); + } + + @Override + public int getInt(final long elementIndex) { + return lookupRegion(elementIndex).getInt(elementIndex); + } + + @Override + public int getInt(@NotNull final FillContext context, final long elementIndex) { + return lookupRegion(elementIndex).getInt(context, elementIndex); + } + + // region StaticRegion.getBytes + // endregion StaticRegion.getBytes + + @Override + public void invalidate() { + + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionLong.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionLong.java index 0104dbd4d50..1bd6e80da90 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionLong.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionLong.java @@ -8,22 +8,37 @@ */ package io.deephaven.parquet.table.region; +import io.deephaven.api.SortColumn; +import io.deephaven.chunk.WritableLongChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.table.impl.sort.timsort.LongTimsortKernel; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionLong; +import io.deephaven.engine.table.impl.sources.regioned.RegionedPageStore; +import io.deephaven.engine.table.impl.sources.regioned.kernel.LongRegionBinarySearchKernel; +import io.deephaven.parquet.table.location.ParquetColumnLocation; import io.deephaven.parquet.table.pagestore.ColumnChunkPageStore; -import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.ChunkPage; +import io.deephaven.util.QueryConstants; +import io.deephaven.util.type.ArrayTypeUtils; +import org.apache.parquet.column.statistics.LongStatistics; import org.jetbrains.annotations.NotNull; /** * {@link ColumnRegionLong} implementation for regions that support fetching primitive longs from * {@link ColumnChunkPageStore column chunk page stores}. */ -public final class ParquetColumnRegionLong extends ParquetColumnRegionBase +public final class ParquetColumnRegionLong extends ParquetColumnRegionBase implements ColumnRegionLong, ParquetColumnRegion { - public ParquetColumnRegionLong(@NotNull final ColumnChunkPageStore columnChunkPageStore) { - super(columnChunkPageStore.mask(), columnChunkPageStore); + public ParquetColumnRegionLong( + @NotNull final ColumnChunkPageStore columnChunkPageStore, + @NotNull final ParquetColumnLocation location) { + super(columnChunkPageStore.mask(), columnChunkPageStore, location); } // region getBytes @@ -38,4 +53,101 @@ public long getLong(final long rowKey) { throw new TableDataException("Error retrieving long at row key " + rowKey + " from a parquet table", e); } } + + // TODO NATE NOCOMMIT: these are still useful aren't they? + public RowSet binSearchMatch( + long firstRowKey, + final long lastRowKey, + @NotNull final SortColumn sortColumn, + @NotNull final Object[] keys) { + return LongRegionBinarySearchKernel.binarySearchMatch(this, firstRowKey, lastRowKey, sortColumn, keys); + } + + public boolean mightContain(@NotNull final Object[] keys) { + // We don't have statistics, so we need to assume there is something in the range that fits. + if (!columnChunkPageStore.hasStatistics()) { + return true; + } + + final LongStatistics stats = columnChunkPageStore.getStatistics(); + // TODO: make sure this actually does what you think it does. + if (!stats.hasNonNullValue()) { + // Statistics are incomplete, we have to assume the region might have the data. + return true; + } + + final long[] typed = ArrayTypeUtils.getUnboxedLongArray(keys); + try (final LongTimsortKernel.LongSortKernelContext sortContext = + LongTimsortKernel.createContext(typed.length)) { + sortContext.sort(WritableLongChunk.writableChunkWrap(typed)); + } + + int firstNonNullValue = 0; + if (typed[firstNonNullValue] == QueryConstants.NULL_LONG) { + // If there were nulls in the region, we can just say 'yes' + if (stats.getNumNulls() > 0) { + return true; + } + + // If there were not, then find the last value to find that is not null + while (firstNonNullValue < typed.length && typed[firstNonNullValue] == QueryConstants.NULL_LONG) { + firstNonNullValue++; + } + + // Everything was null, we are done. + if (firstNonNullValue >= typed.length) { + return false; + } + } + + // Look through the keys and find anything that fits in the range. + for (int ii = firstNonNullValue; ii < typed.length; ii++) { + if (typed[ii] >= stats.getMin() && typed[ii] <= stats.getMax()) { + return true; + } + } + + // Nothing matches, we can skip this region. + return false; + } + + public static final class StaticPageStore + extends RegionedPageStore.Static> + implements ColumnRegionLong { + + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionLong[] regions, + @NotNull final ParquetColumnLocation location) { + super(parameters, regions, location); + } + + @Override + public long getLong(final long elementIndex) { + return lookupRegion(elementIndex).getLong(elementIndex); + } + + @Override + public long getLong(@NotNull final FillContext context, final long elementIndex) { + return lookupRegion(elementIndex).getLong(context, elementIndex); + } + + // region StaticRegion.getBytes + // endregion StaticRegion.getBytes + + @Override + public void invalidate() { + + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionObject.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionObject.java index 8bad3352397..ce98e292367 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionObject.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionObject.java @@ -3,26 +3,40 @@ */ package io.deephaven.parquet.table.region; +import io.deephaven.api.SortColumn; +import io.deephaven.chunk.WritableObjectChunk; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.configuration.Configuration; +import io.deephaven.engine.page.ChunkPage; +import io.deephaven.engine.page.Page; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.chunkattributes.DictionaryKeys; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.table.impl.sort.timsort.ObjectTimsortKernel; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionLong; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionObject; import io.deephaven.engine.table.impl.sources.regioned.RegionVisitResult; +import io.deephaven.engine.table.impl.sources.regioned.kernel.ObjectRegionBinarySearchKernel; +import io.deephaven.parquet.table.location.ParquetColumnLocation; import io.deephaven.parquet.table.pagestore.ColumnChunkPageStore; -import io.deephaven.chunk.attributes.Any; -import io.deephaven.engine.table.impl.chunkattributes.DictionaryKeys; -import io.deephaven.engine.page.ChunkPage; -import io.deephaven.engine.page.Page; -import io.deephaven.engine.rowset.RowSet; +import io.deephaven.util.compare.ObjectComparisons; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.parquet.column.statistics.BinaryStatistics; import org.jetbrains.annotations.NotNull; +import java.util.Arrays; import java.util.function.Supplier; /** * {@link ColumnRegionObject} implementation for regions that support fetching objects from * {@link ColumnChunkPageStore column chunk page stores}. */ -public final class ParquetColumnRegionObject extends ParquetColumnRegionBase +public final class ParquetColumnRegionObject extends ParquetColumnRegionBase implements ColumnRegionObject, ParquetColumnRegion, Page { + private static final double STANDARD_SORT_FRACTION_THRESHOLD = Configuration.getInstance() + .getDoubleForClassWithDefault(ParquetColumnRegionObject.class, "sortFractionThreshold", 0.5d); private volatile Supplier> dictionaryKeysRegionSupplier; private volatile Supplier> dictionaryValuesRegionSupplier; @@ -30,10 +44,12 @@ public final class ParquetColumnRegionObject extend private ColumnRegionLong dictionaryKeysRegion; private ColumnRegionObject dictionaryValuesRegion; - public ParquetColumnRegionObject(@NotNull final ColumnChunkPageStore columnChunkPageStore, - @NotNull final Supplier> dictionaryKeysRegionSupplier, - @NotNull final Supplier> dictionaryValuesRegionSupplier) { - super(columnChunkPageStore.mask(), columnChunkPageStore); + public ParquetColumnRegionObject( + @NotNull final ColumnChunkPageStore columnChunkPageStore, + @NotNull final Supplier> dictionaryKeysRegionSupplier, + @NotNull final Supplier> dictionaryValuesRegionSupplier, + @NotNull final ParquetColumnLocation location) { + super(columnChunkPageStore.mask(), columnChunkPageStore, location); this.dictionaryKeysRegionSupplier = dictionaryKeysRegionSupplier; this.dictionaryValuesRegionSupplier = dictionaryValuesRegionSupplier; } @@ -80,4 +96,89 @@ public ColumnRegionObject getDictionaryValuesRegion() { } return dictionaryValuesRegion; } + + // TODO NATE NOCOMMIT: These are still helpful aren't they? + public RowSet binSearchMatch( + final long firstRowKey, + final long lastRowKey, + @NotNull final SortColumn sortColumn, + @NotNull final Object[] keys) { + return ObjectRegionBinarySearchKernel.binarySearchMatch(this, firstRowKey, lastRowKey, sortColumn, keys); + } + + public boolean mightContain(@NotNull final Object[] keys) { + // We don't have statistics, so we need to assume there is something in the range that fits. + if (!columnChunkPageStore.hasStatistics()) { + return true; + } + + final BinaryStatistics stats = columnChunkPageStore.getStatistics(); + if (!stats.hasNonNullValue()) { + // We don't have min/max data recorded. We have to assume it might contain the data + return true; + } + + final Object[] sortedKeys = Arrays.copyOf(keys, keys.length); + try (final ObjectTimsortKernel.ObjectSortKernelContext kernel = + ObjectTimsortKernel.createContext(sortedKeys.length)) { + kernel.sort(WritableObjectChunk.writableChunkWrap(sortedKeys)); + } + + int firstNonNullValue = 0; + if(sortedKeys[firstNonNullValue] == null) { + // If there were nulls in the region, we can just say 'yes' + if(stats.getNumNulls() > 0) { + return true; + } + + // If there were not, then find the last value to find that is not null + while(firstNonNullValue < sortedKeys.length && sortedKeys[firstNonNullValue] == null) { + firstNonNullValue++; + } + + // Everything was null, we are done. + if(firstNonNullValue >= sortedKeys.length) { + return false; + } + } + + final DATA_TYPE minVal = columnChunkPageStore.convertSingleValue(stats.genericGetMin()); + final DATA_TYPE maxVal = columnChunkPageStore.convertSingleValue(stats.genericGetMax()); + // Look through the keys and find anything that fits in the range. + for (int ii = 0; ii <= firstNonNullValue; ii++) { + if (ObjectComparisons.geq(sortedKeys[ii], minVal) && ObjectComparisons.leq(sortedKeys[ii], maxVal)) { + return true; + } + } + + // Nothing matches, we can skip this region. + return false; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + if (!columnChunkPageStore.usesDictionaryOnEveryPage()) { + return match(invertMatch, usePrev, caseInsensitive, rowSequence, sortedKeys); + } + + final ColumnRegionObject valuesRegion = getDictionaryValuesRegion(); + // TODO NATE NOCOMMIT: this isn't a helpful test + final long maxRegionSize = valuesRegion.mask(); + if ((maxRegionSize / (double)rowSequence.size()) > STANDARD_SORT_FRACTION_THRESHOLD) { + return super.match(invertMatch, usePrev, caseInsensitive, rowSequence, sortedKeys); + } + + final RowSet matchedKeys = getDictionaryValuesRegion().match( + invertMatch, usePrev, caseInsensitive, rowSequence, sortedKeys); + final Object[] convertedKeys = new Object[sortedKeys.length]; + MutableInt pos = new MutableInt(0); + matchedKeys.forAllRowKeys(k -> convertedKeys[pos.getAndIncrement()] = k); + + return getDictionaryKeysRegion().match(invertMatch, usePrev, caseInsensitive, rowSequence, convertedKeys); + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionShort.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionShort.java index 9686cb1390f..4a70f944c8d 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionShort.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/region/ParquetColumnRegionShort.java @@ -8,22 +8,37 @@ */ package io.deephaven.parquet.table.region; +import io.deephaven.api.SortColumn; +import io.deephaven.chunk.WritableShortChunk; +import io.deephaven.chunk.attributes.Any; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.impl.locations.TableDataException; +import io.deephaven.engine.table.impl.sort.timsort.ShortTimsortKernel; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionShort; +import io.deephaven.engine.table.impl.sources.regioned.RegionedPageStore; +import io.deephaven.engine.table.impl.sources.regioned.kernel.ShortRegionBinarySearchKernel; +import io.deephaven.parquet.table.location.ParquetColumnLocation; import io.deephaven.parquet.table.pagestore.ColumnChunkPageStore; -import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.ChunkPage; +import io.deephaven.util.QueryConstants; +import io.deephaven.util.type.ArrayTypeUtils; +import org.apache.parquet.column.statistics.IntStatistics; import org.jetbrains.annotations.NotNull; /** * {@link ColumnRegionShort} implementation for regions that support fetching primitive shorts from * {@link ColumnChunkPageStore column chunk page stores}. */ -public final class ParquetColumnRegionShort extends ParquetColumnRegionBase +public final class ParquetColumnRegionShort extends ParquetColumnRegionBase implements ColumnRegionShort, ParquetColumnRegion { - public ParquetColumnRegionShort(@NotNull final ColumnChunkPageStore columnChunkPageStore) { - super(columnChunkPageStore.mask(), columnChunkPageStore); + public ParquetColumnRegionShort( + @NotNull final ColumnChunkPageStore columnChunkPageStore, + @NotNull final ParquetColumnLocation location) { + super(columnChunkPageStore.mask(), columnChunkPageStore, location); } // region getBytes @@ -38,4 +53,101 @@ public short getShort(final long rowKey) { throw new TableDataException("Error retrieving short at row key " + rowKey + " from a parquet table", e); } } + + // TODO NATE NOCOMMIT: these are still useful aren't they? + public RowSet binSearchMatch( + long firstRowKey, + final long lastRowKey, + @NotNull final SortColumn sortColumn, + @NotNull final Object[] keys) { + return ShortRegionBinarySearchKernel.binarySearchMatch(this, firstRowKey, lastRowKey, sortColumn, keys); + } + + public boolean mightContain(@NotNull final Object[] keys) { + // We don't have statistics, so we need to assume there is something in the range that fits. + if (!columnChunkPageStore.hasStatistics()) { + return true; + } + + final IntStatistics stats = columnChunkPageStore.getStatistics(); + // TODO: make sure this actually does what you think it does. + if (!stats.hasNonNullValue()) { + // Statistics are incomplete, we have to assume the region might have the data. + return true; + } + + final short[] typed = ArrayTypeUtils.getUnboxedShortArray(keys); + try (final ShortTimsortKernel.ShortSortKernelContext sortContext = + ShortTimsortKernel.createContext(typed.length)) { + sortContext.sort(WritableShortChunk.writableChunkWrap(typed)); + } + + int firstNonNullValue = 0; + if (typed[firstNonNullValue] == QueryConstants.NULL_SHORT) { + // If there were nulls in the region, we can just say 'yes' + if (stats.getNumNulls() > 0) { + return true; + } + + // If there were not, then find the last value to find that is not null + while (firstNonNullValue < typed.length && typed[firstNonNullValue] == QueryConstants.NULL_SHORT) { + firstNonNullValue++; + } + + // Everything was null, we are done. + if (firstNonNullValue >= typed.length) { + return false; + } + } + + // Look through the keys and find anything that fits in the range. + for (int ii = firstNonNullValue; ii < typed.length; ii++) { + if (typed[ii] >= stats.getMin() && typed[ii] <= stats.getMax()) { + return true; + } + } + + // Nothing matches, we can skip this region. + return false; + } + + public static final class StaticPageStore + extends RegionedPageStore.Static> + implements ColumnRegionShort { + + public StaticPageStore( + @NotNull final Parameters parameters, + @NotNull final ColumnRegionShort[] regions, + @NotNull final ParquetColumnLocation location) { + super(parameters, regions, location); + } + + @Override + public short getShort(final long elementIndex) { + return lookupRegion(elementIndex).getShort(elementIndex); + } + + @Override + public short getShort(@NotNull final FillContext context, final long elementIndex) { + return lookupRegion(elementIndex).getShort(context, elementIndex); + } + + // region StaticRegion.getBytes + // endregion StaticRegion.getBytes + + @Override + public void invalidate() { + + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // where is the data stored here? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/transfer/LocalDateTimeArrayTransfer.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/transfer/LocalDateTimeArrayTransfer.java index 4b1e8a74da4..f625be53321 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/transfer/LocalDateTimeArrayTransfer.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/transfer/LocalDateTimeArrayTransfer.java @@ -16,8 +16,7 @@ import java.nio.LongBuffer; import java.time.LocalDateTime; -final class LocalDateTimeArrayTransfer - extends PrimitiveArrayAndVectorTransfer { +final class LocalDateTimeArrayTransfer extends PrimitiveArrayAndVectorTransfer { // We encode LocalDateTime as primitive longs LocalDateTimeArrayTransfer(@NotNull final ColumnSource columnSource, @NotNull final RowSequence tableRowSet, final int targetPageSizeInBytes) { diff --git a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionByte.java b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionByte.java index c1759a33384..be6c8a02220 100644 --- a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionByte.java +++ b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionByte.java @@ -11,15 +11,18 @@ import io.deephaven.engine.rowset.RowSequenceFactory; import io.deephaven.base.MathUtil; -import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableByteChunk; import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.PageStore; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionByte; import io.deephaven.engine.table.impl.sources.regioned.GenericColumnRegionBase; import io.deephaven.generic.page.ChunkHolderPageByte; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.lang.ref.SoftReference; import java.util.Arrays; @@ -35,6 +38,7 @@ public class AppendOnlyFixedSizePageRegionByte private final int pageSize; private final AppendOnlyRegionAccessor accessor; + private final ColumnLocation location; @SuppressWarnings("unchecked") private volatile SoftReference>[] pageHolderRefs = new SoftReference[0]; @@ -42,10 +46,12 @@ public class AppendOnlyFixedSizePageRegionByte public AppendOnlyFixedSizePageRegionByte( final long pageMask, final int pageSize, - @NotNull final AppendOnlyRegionAccessor accessor) { + @NotNull final AppendOnlyRegionAccessor accessor, + @NotNull final ColumnLocation location) { super(pageMask); this.pageSize = pageSize; this.accessor = accessor; + this.location = location; } @Override @@ -156,4 +162,20 @@ private void ensureFilled( pageHolder.acceptAppend(destination, currentSize); } } + + @Override + public @Nullable ColumnLocation getLocation() { + return location; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // what should we do on an append only region? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } diff --git a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionChar.java b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionChar.java index 2da72d2f94a..d5d8667dfa0 100644 --- a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionChar.java +++ b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionChar.java @@ -1,15 +1,18 @@ package io.deephaven.generic.region; import io.deephaven.base.MathUtil; -import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableCharChunk; import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.PageStore; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionChar; import io.deephaven.engine.table.impl.sources.regioned.GenericColumnRegionBase; import io.deephaven.generic.page.ChunkHolderPageChar; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.lang.ref.SoftReference; import java.util.Arrays; @@ -25,6 +28,7 @@ public class AppendOnlyFixedSizePageRegionChar private final int pageSize; private final AppendOnlyRegionAccessor accessor; + private final ColumnLocation location; @SuppressWarnings("unchecked") private volatile SoftReference>[] pageHolderRefs = new SoftReference[0]; @@ -32,10 +36,12 @@ public class AppendOnlyFixedSizePageRegionChar public AppendOnlyFixedSizePageRegionChar( final long pageMask, final int pageSize, - @NotNull final AppendOnlyRegionAccessor accessor) { + @NotNull final AppendOnlyRegionAccessor accessor, + @NotNull final ColumnLocation location) { super(pageMask); this.pageSize = pageSize; this.accessor = accessor; + this.location = location; } @Override @@ -134,4 +140,20 @@ private void ensureFilled( pageHolder.acceptAppend(destination, currentSize); } } + + @Override + public @Nullable ColumnLocation getLocation() { + return location; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // what should we do on an append only region? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } diff --git a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionDouble.java b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionDouble.java index a7b3df45eee..9ccbe7b6b07 100644 --- a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionDouble.java +++ b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionDouble.java @@ -6,15 +6,18 @@ package io.deephaven.generic.region; import io.deephaven.base.MathUtil; -import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableDoubleChunk; import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.PageStore; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionDouble; import io.deephaven.engine.table.impl.sources.regioned.GenericColumnRegionBase; import io.deephaven.generic.page.ChunkHolderPageDouble; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.lang.ref.SoftReference; import java.util.Arrays; @@ -30,6 +33,7 @@ public class AppendOnlyFixedSizePageRegionDouble private final int pageSize; private final AppendOnlyRegionAccessor accessor; + private final ColumnLocation location; @SuppressWarnings("unchecked") private volatile SoftReference>[] pageHolderRefs = new SoftReference[0]; @@ -37,10 +41,12 @@ public class AppendOnlyFixedSizePageRegionDouble public AppendOnlyFixedSizePageRegionDouble( final long pageMask, final int pageSize, - @NotNull final AppendOnlyRegionAccessor accessor) { + @NotNull final AppendOnlyRegionAccessor accessor, + @NotNull final ColumnLocation location) { super(pageMask); this.pageSize = pageSize; this.accessor = accessor; + this.location = location; } @Override @@ -139,4 +145,20 @@ private void ensureFilled( pageHolder.acceptAppend(destination, currentSize); } } + + @Override + public @Nullable ColumnLocation getLocation() { + return location; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // what should we do on an append only region? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } diff --git a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionFloat.java b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionFloat.java index b97b9bfc4f5..2170bcaff14 100644 --- a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionFloat.java +++ b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionFloat.java @@ -6,15 +6,18 @@ package io.deephaven.generic.region; import io.deephaven.base.MathUtil; -import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableFloatChunk; import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.PageStore; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionFloat; import io.deephaven.engine.table.impl.sources.regioned.GenericColumnRegionBase; import io.deephaven.generic.page.ChunkHolderPageFloat; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.lang.ref.SoftReference; import java.util.Arrays; @@ -30,6 +33,7 @@ public class AppendOnlyFixedSizePageRegionFloat private final int pageSize; private final AppendOnlyRegionAccessor accessor; + private final ColumnLocation location; @SuppressWarnings("unchecked") private volatile SoftReference>[] pageHolderRefs = new SoftReference[0]; @@ -37,10 +41,12 @@ public class AppendOnlyFixedSizePageRegionFloat public AppendOnlyFixedSizePageRegionFloat( final long pageMask, final int pageSize, - @NotNull final AppendOnlyRegionAccessor accessor) { + @NotNull final AppendOnlyRegionAccessor accessor, + @NotNull final ColumnLocation location) { super(pageMask); this.pageSize = pageSize; this.accessor = accessor; + this.location = location; } @Override @@ -139,4 +145,20 @@ private void ensureFilled( pageHolder.acceptAppend(destination, currentSize); } } + + @Override + public @Nullable ColumnLocation getLocation() { + return location; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // what should we do on an append only region? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } diff --git a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionInt.java b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionInt.java index 72c581a0846..16ffd1967c6 100644 --- a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionInt.java +++ b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionInt.java @@ -6,15 +6,18 @@ package io.deephaven.generic.region; import io.deephaven.base.MathUtil; -import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableIntChunk; import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.PageStore; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionInt; import io.deephaven.engine.table.impl.sources.regioned.GenericColumnRegionBase; import io.deephaven.generic.page.ChunkHolderPageInt; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.lang.ref.SoftReference; import java.util.Arrays; @@ -30,6 +33,7 @@ public class AppendOnlyFixedSizePageRegionInt private final int pageSize; private final AppendOnlyRegionAccessor accessor; + private final ColumnLocation location; @SuppressWarnings("unchecked") private volatile SoftReference>[] pageHolderRefs = new SoftReference[0]; @@ -37,10 +41,12 @@ public class AppendOnlyFixedSizePageRegionInt public AppendOnlyFixedSizePageRegionInt( final long pageMask, final int pageSize, - @NotNull final AppendOnlyRegionAccessor accessor) { + @NotNull final AppendOnlyRegionAccessor accessor, + @NotNull final ColumnLocation location) { super(pageMask); this.pageSize = pageSize; this.accessor = accessor; + this.location = location; } @Override @@ -139,4 +145,20 @@ private void ensureFilled( pageHolder.acceptAppend(destination, currentSize); } } + + @Override + public @Nullable ColumnLocation getLocation() { + return location; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // what should we do on an append only region? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } diff --git a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionLong.java b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionLong.java index 9b9bbc4f294..b636f35208c 100644 --- a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionLong.java +++ b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionLong.java @@ -6,15 +6,18 @@ package io.deephaven.generic.region; import io.deephaven.base.MathUtil; -import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableLongChunk; import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.PageStore; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionLong; import io.deephaven.engine.table.impl.sources.regioned.GenericColumnRegionBase; import io.deephaven.generic.page.ChunkHolderPageLong; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.lang.ref.SoftReference; import java.util.Arrays; @@ -30,6 +33,7 @@ public class AppendOnlyFixedSizePageRegionLong private final int pageSize; private final AppendOnlyRegionAccessor accessor; + private final ColumnLocation location; @SuppressWarnings("unchecked") private volatile SoftReference>[] pageHolderRefs = new SoftReference[0]; @@ -37,10 +41,12 @@ public class AppendOnlyFixedSizePageRegionLong public AppendOnlyFixedSizePageRegionLong( final long pageMask, final int pageSize, - @NotNull final AppendOnlyRegionAccessor accessor) { + @NotNull final AppendOnlyRegionAccessor accessor, + @NotNull final ColumnLocation location) { super(pageMask); this.pageSize = pageSize; this.accessor = accessor; + this.location = location; } @Override @@ -139,4 +145,20 @@ private void ensureFilled( pageHolder.acceptAppend(destination, currentSize); } } + + @Override + public @Nullable ColumnLocation getLocation() { + return location; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // what should we do on an append only region? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } diff --git a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionObject.java b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionObject.java index 6e57df6c67c..fde05eee5cf 100644 --- a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionObject.java +++ b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionObject.java @@ -6,15 +6,18 @@ package io.deephaven.generic.region; import io.deephaven.base.MathUtil; -import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableObjectChunk; import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.PageStore; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionObject; import io.deephaven.engine.table.impl.sources.regioned.GenericColumnRegionBase; import io.deephaven.generic.page.ChunkHolderPageObject; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.lang.ref.SoftReference; import java.util.Arrays; @@ -30,6 +33,7 @@ public class AppendOnlyFixedSizePageRegionObject private final int pageSize; private final AppendOnlyRegionAccessor accessor; + private final ColumnLocation location; @SuppressWarnings("unchecked") private volatile SoftReference>[] pageHolderRefs = new SoftReference[0]; @@ -37,10 +41,12 @@ public class AppendOnlyFixedSizePageRegionObject public AppendOnlyFixedSizePageRegionObject( final long pageMask, final int pageSize, - @NotNull final AppendOnlyRegionAccessor accessor) { + @NotNull final AppendOnlyRegionAccessor accessor, + @NotNull final ColumnLocation location) { super(pageMask); this.pageSize = pageSize; this.accessor = accessor; + this.location = location; } @Override @@ -140,4 +146,20 @@ private void ensureFilled( pageHolder.acceptAppend(destination, currentSize); } } + + @Override + public @Nullable ColumnLocation getLocation() { + return location; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // what should we do on an append only region? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } diff --git a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionShort.java b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionShort.java index d6081a5b9c1..d1200e59cdf 100644 --- a/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionShort.java +++ b/extensions/source-support/src/main/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionShort.java @@ -6,15 +6,18 @@ package io.deephaven.generic.region; import io.deephaven.base.MathUtil; -import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableShortChunk; import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.page.PageStore; +import io.deephaven.engine.rowset.RowSequence; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.table.impl.locations.ColumnLocation; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.sources.regioned.ColumnRegionShort; import io.deephaven.engine.table.impl.sources.regioned.GenericColumnRegionBase; import io.deephaven.generic.page.ChunkHolderPageShort; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.lang.ref.SoftReference; import java.util.Arrays; @@ -30,6 +33,7 @@ public class AppendOnlyFixedSizePageRegionShort private final int pageSize; private final AppendOnlyRegionAccessor accessor; + private final ColumnLocation location; @SuppressWarnings("unchecked") private volatile SoftReference>[] pageHolderRefs = new SoftReference[0]; @@ -37,10 +41,12 @@ public class AppendOnlyFixedSizePageRegionShort public AppendOnlyFixedSizePageRegionShort( final long pageMask, final int pageSize, - @NotNull final AppendOnlyRegionAccessor accessor) { + @NotNull final AppendOnlyRegionAccessor accessor, + @NotNull final ColumnLocation location) { super(pageMask); this.pageSize = pageSize; this.accessor = accessor; + this.location = location; } @Override @@ -139,4 +145,20 @@ private void ensureFilled( pageHolder.acceptAppend(destination, currentSize); } } + + @Override + public @Nullable ColumnLocation getLocation() { + return location; + } + + @Override + public WritableRowSet match( + final boolean invertMatch, + final boolean usePrev, + final boolean caseInsensitive, + @NotNull final RowSequence rowSequence, + final Object... sortedKeys) { + // what should we do on an append only region? + throw new UnsupportedOperationException("TODO NATE NOCOMMIT"); + } } diff --git a/extensions/source-support/src/test/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionTest.java b/extensions/source-support/src/test/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionTest.java index f438df72d3c..1acec299228 100644 --- a/extensions/source-support/src/test/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionTest.java +++ b/extensions/source-support/src/test/java/io/deephaven/generic/region/AppendOnlyFixedSizePageRegionTest.java @@ -93,7 +93,7 @@ private static Table makeRegioned( return new SimpleSourceTable( constituents[0].getDefinition(), "Test SimpleSourceTable", - RegionedTableComponentFactoryImpl.INSTANCE, + RegionedTableComponentFactoryImpl.make(), new TableBackedTableLocationProvider(registrar, false, constituents), registrar).coalesce(); } diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateRegionAndRegionedSourceTests.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateRegionAndRegionedSourceTests.java index c8404aa967a..483f076db04 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateRegionAndRegionedSourceTests.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateRegionAndRegionedSourceTests.java @@ -9,8 +9,8 @@ import static io.deephaven.replication.ReplicatePrimitiveCode.charToAllButBooleanAndByte; /** - * Code generation for tests of {@link RegionedColumnSource} implementations as well as well as the primary region - * interfaces for some primitive types. + * Code generation for tests of {@code RegionedColumnSource} implementations as well as the primary region interfaces + * for some primitive types. */ public class ReplicateRegionAndRegionedSourceTests { diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateRegionsAndRegionedSources.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateRegionsAndRegionedSources.java index a4895c9d434..7d9e7d6bcd9 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateRegionsAndRegionedSources.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateRegionsAndRegionedSources.java @@ -4,6 +4,7 @@ package io.deephaven.replicators; import org.apache.commons.io.FileUtils; +import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; @@ -80,6 +81,11 @@ private static void fixupChunkColumnRegionByte(final String bytePath) throws IOE " }", " return destination;", " }")); + lines = replaceRegion(lines, "StaticRegion.getBytes", Arrays.asList( + " @Override", + " public byte[] getBytes(long firstElementIndex, @NotNull byte[] destination, int destinationOffset, int length) {", + " return lookupRegion(firstElementIndex).getBytes(firstElementIndex, destination, destinationOffset, length);", + " }")); FileUtils.writeLines(byteFile, lines); } @@ -98,6 +104,10 @@ private static void fixupChunkColumnRegionObject(final String objectPath) throws .collect(Collectors.toList()); lines = lines.stream().map(x -> x.replaceAll("ChunkHolderPageObject<([^,>]+)>", "ChunkHolderPageObject")) .collect(Collectors.toList()); + lines = lines.stream().map(x -> x.replaceAll("T\\[] keys", "Object[] keys")) + .collect(Collectors.toList()); + lines = lines.stream().map(x -> x.replaceAll("T\\[] sortedKeys", "Object[] sortedKeys")) + .collect(Collectors.toList()); lines = replaceRegion(lines, "allocatePage", Arrays.asList( " // noinspection unchecked", " pageHolder = new ChunkHolderPageObject(mask(), pageFirstRowInclusive, (T[]) new Object[pageSize]);")); diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateTransfer.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateTransfer.java new file mode 100644 index 00000000000..9308aadd9c7 --- /dev/null +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateTransfer.java @@ -0,0 +1,29 @@ +/** + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.replicators; + +import java.io.IOException; + +import static io.deephaven.replication.ReplicatePrimitiveCode.*; + +public class ReplicateTransfer { + private static final String PARQUET_INT_TRANSFER_PATH = + "extensions/parquet/table/src/main/java/io/deephaven/parquet/table/transfer/IntTransfer.java"; + + public static void main(String[] args) throws IOException { + + intToChar(PARQUET_INT_TRANSFER_PATH, null, + "IntBuffer", "IntStatistics", "int rowCount", "int targetSize", "int ii"); + intToByte(PARQUET_INT_TRANSFER_PATH, null, + "IntBuffer", "IntStatistics", "int rowCount", "int targetSize", "int ii"); + intToShort(PARQUET_INT_TRANSFER_PATH, null, + "IntBuffer", "IntStatistics", "int rowCount", "int targetSize", "int ii"); + intToLong(PARQUET_INT_TRANSFER_PATH, null, + "int rowCount", "int targetSize", "int ii"); + intToDouble(PARQUET_INT_TRANSFER_PATH, null, + "int rowCount", "int targetSize", "int ii"); + intToFloat(PARQUET_INT_TRANSFER_PATH, null, + "int rowCount", "int targetSize", "int ii"); + } +} diff --git a/server/src/main/java/io/deephaven/server/table/inputtables/InputTableServiceGrpcImpl.java b/server/src/main/java/io/deephaven/server/table/inputtables/InputTableServiceGrpcImpl.java index 673e39e35b1..00b7087f3eb 100644 --- a/server/src/main/java/io/deephaven/server/table/inputtables/InputTableServiceGrpcImpl.java +++ b/server/src/main/java/io/deephaven/server/table/inputtables/InputTableServiceGrpcImpl.java @@ -6,8 +6,8 @@ import com.google.rpc.Code; import io.deephaven.auth.codegen.impl.InputTableServiceContextualAuthWiring; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.exceptions.IncompatibleTableDefinitionException; import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.perf.QueryPerformanceNugget; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.util.config.MutableInputTable; @@ -83,6 +83,14 @@ public void addTableToInputTable( MutableInputTable mutableInputTable = (MutableInputTable) inputTable; Table tableToAdd = tableToAddExport.get(); + // validate that the columns are compatible + try { + mutableInputTable.validateAddOrModify(tableToAdd); + } catch (IncompatibleTableDefinitionException exception) { + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, + "Provided tables's columns are not compatible: " + exception.getMessage()); + } + authWiring.checkPermissionAddTableToInputTable( ExecutionContext.getContext().getAuthContext(), request, List.of(targetTable.get(), tableToAdd)); @@ -90,7 +98,7 @@ public void addTableToInputTable( // validate that the columns are compatible try { mutableInputTable.validateAddOrModify(tableToAdd); - } catch (TableDefinition.IncompatibleTableDefinitionException exception) { + } catch (IncompatibleTableDefinitionException exception) { throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Provided tables's columns are not compatible: " + exception.getMessage()); } @@ -140,6 +148,16 @@ public void deleteTableFromInputTable( MutableInputTable mutableInputTable = (MutableInputTable) inputTable; Table tableToRemove = tableToRemoveExport.get(); + // validate that the columns are compatible + try { + mutableInputTable.validateDelete(tableToRemove); + } catch (IncompatibleTableDefinitionException exception) { + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, + "Provided tables's columns are not compatible: " + exception.getMessage()); + } catch (UnsupportedOperationException exception) { + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, + "Provided input table does not support delete."); + } authWiring.checkPermissionDeleteTableFromInputTable( ExecutionContext.getContext().getAuthContext(), request, @@ -148,7 +166,7 @@ public void deleteTableFromInputTable( // validate that the columns are compatible try { mutableInputTable.validateDelete(tableToRemove); - } catch (TableDefinition.IncompatibleTableDefinitionException exception) { + } catch (IncompatibleTableDefinitionException exception) { throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Provided tables's columns are not compatible: " + exception.getMessage()); } catch (UnsupportedOperationException exception) {