ExecutionContext: record state of globals before deferring work (#2539)

The QueryScope, QueryLibrary, CompilerTools were merged into a new package `io.deephaven.engine.context` and are bundled as a distinct artifact `engine-context`. By default the ThreadLocal is "poisoned" to prevent accidental use of this state in unexpected contexts. The session's execution context will be installed from all entry points to the gRPC API as well as scripts run in a deephaven script session. To use elsewhere it is encouraged to capture the least state needed and pass the new ExecutionContext to use on another thread or during a deferred task. 

WhereFilters and SelectColumns will be initialized immediately and copies will retain details instead of needing to reevaluate in deferred contexts.

This commit introduces `TableOperationsDefaults` which lifts many default `TableOperations` implementations from `TableDefaults`. This helps reinforce consistency in other implementations of TableOperations.

The QueryLanguageParser can now deduce the return type of a generic method if it is fixed due to inheritance rules (both via interfaces and parent classes).

Base/src/main/java/io/deephaven/base/StringUtils.java

@@ -177,10 +177,4 @@ public static String pad(String s, int length, char padChar) {
 
         return buf.toString();
     }
-
-    public static Collection<String> splitToCollection(String string) {
-        return string.trim().isEmpty() ? Collections.emptyList()
-                : Arrays.stream(string.split(",")).map(String::trim).filter(s -> !s.isEmpty())
-                        .collect(Collectors.toList());
-    }
 }

BenchmarkSupport/src/main/java/io/deephaven/benchmarking/BenchmarkTools.java

@@ -5,10 +5,10 @@
 
 import io.deephaven.base.StringUtils;
 import io.deephaven.configuration.Configuration;
+import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.engine.table.ColumnDefinition;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.TableDefinition;
-import io.deephaven.engine.table.lang.QueryScope;
 import io.deephaven.time.DateTime;
 import io.deephaven.util.annotations.ScriptApi;
 import io.deephaven.benchmarking.generator.*;
@@ -300,7 +300,7 @@ public static Table applySparsity(Table table, int size, int sparsity, long seed
             throw new IllegalStateException("Sparsity must be in the range of 1 through 100");
         }
         Random random = new Random(seed);
-        QueryScope.getScope().putParam("__random__", random);
+        ExecutionContext.getContext().getQueryScope().putParam("__random__", random);
         return table.where("__random__.nextInt(100) < " + sparsity).head(size);
     }
 

BenchmarkSupport/src/main/java/io/deephaven/benchmarking/generator/ColumnGenerator.java

@@ -6,7 +6,7 @@
 import io.deephaven.engine.table.ColumnDefinition;
 import io.deephaven.benchmarking.generator.random.ExtendedRandom;
 import io.deephaven.engine.table.Table;
-import io.deephaven.engine.table.lang.QueryScope;
+import io.deephaven.engine.context.QueryScope;
 
 /**
  * An interface that defines a class which will create Columns for a {@link io.deephaven.benchmarking.BenchmarkTable}
@@ -22,7 +22,7 @@ public interface ColumnGenerator<T> {
 
     /**
      * Initialize any internal state with the specified RNG
-     * 
+     *
      * @param random the RNG to use.
      */
     void init(ExtendedRandom random);

BenchmarkSupport/src/main/java/io/deephaven/benchmarking/impl/AbstractBenchmarkTable.java

@@ -4,7 +4,7 @@
 package io.deephaven.benchmarking.impl;
 
 import io.deephaven.engine.table.Table;
-import io.deephaven.engine.table.lang.QueryScope;
+import io.deephaven.engine.context.QueryScope;
 import io.deephaven.benchmarking.BenchmarkTable;
 import io.deephaven.benchmarking.generator.ColumnGenerator;
 import io.deephaven.benchmarking.generator.random.ExtendedRandom;

ClientSupport/src/main/java/io/deephaven/treetable/TreeSnapshotQuery.java

@@ -6,7 +6,7 @@
 import io.deephaven.base.formatters.FormatBitSet;
 import io.deephaven.base.verify.Assert;
 import io.deephaven.engine.table.Table;
-import io.deephaven.engine.updategraph.ConcurrentMethod;
+import io.deephaven.api.util.ConcurrentMethod;
 import io.deephaven.engine.table.impl.HierarchicalTable;
 import io.deephaven.engine.table.impl.HierarchicalTableInfo;
 import io.deephaven.engine.table.impl.RollupInfo;

ClientSupport/src/test/java/io/deephaven/treetable/SnapshotStateTest.java

@@ -3,20 +3,11 @@
  */
 package io.deephaven.treetable;
 
-import io.deephaven.csv.CsvTools;
-import io.deephaven.csv.util.CsvReaderException;
-import io.deephaven.engine.table.Table;
-import io.deephaven.engine.table.lang.QueryLibrary;
-import io.deephaven.engine.util.TableTools;
-import io.deephaven.engine.table.impl.BaseTable;
-import io.deephaven.engine.table.impl.HierarchicalTable;
 import io.deephaven.engine.table.impl.QueryTableTestBase;
-import org.junit.Test;
 
 import java.util.*;
 
 import static io.deephaven.engine.util.TableTools.emptyTable;
-import static io.deephaven.treetable.TreeTableConstants.ROOT_TABLE_KEY;
 
 public class SnapshotStateTest extends QueryTableTestBase {
 

ClientSupport/src/test/java/io/deephaven/treetable/TreeSnapshotQueryTest.java

@@ -3,42 +3,12 @@
  */
 package io.deephaven.treetable;
 
-import io.deephaven.base.Pair;
-import io.deephaven.csv.CsvTools;
-import io.deephaven.csv.util.CsvReaderException;
-import io.deephaven.datastructures.util.SmartKey;
-import io.deephaven.engine.table.ColumnDefinition;
-import io.deephaven.engine.table.Table;
-import io.deephaven.engine.table.TableMap;
-import io.deephaven.engine.table.impl.select.WhereFilterFactory;
-import io.deephaven.engine.table.impl.sources.InMemoryColumnSource;
-import io.deephaven.engine.table.lang.QueryLibrary;
-import io.deephaven.engine.updategraph.UpdateGraphProcessor;
-import io.deephaven.engine.util.TableTools;
 import io.deephaven.engine.table.impl.*;
-import io.deephaven.engine.table.impl.select.WhereFilter;
-import io.deephaven.engine.table.ColumnSource;
-import io.deephaven.engine.updategraph.LogicalClock;
-import io.deephaven.table.sort.SortDirective;
-import io.deephaven.util.annotations.ReflexiveUse;
-import gnu.trove.map.TIntObjectMap;
-import gnu.trove.map.hash.TIntObjectHashMap;
-import gnu.trove.map.hash.TObjectIntHashMap;
 
 import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Function;
-import java.util.stream.Collectors;
 
-import static io.deephaven.engine.table.impl.TstUtils.*;
 import static io.deephaven.engine.util.TableTools.emptyTable;
-import static io.deephaven.treetable.TreeTableConstants.*;
 import static org.junit.Assert.assertArrayEquals;
-import static io.deephaven.api.agg.Aggregation.AggLast;
-import static io.deephaven.api.agg.Aggregation.AggSum;
 
 public class TreeSnapshotQueryTest extends QueryTableTestBase {
 

Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java

@@ -7,8 +7,8 @@
 import io.deephaven.base.verify.Assert;
 import io.deephaven.configuration.Configuration;
 import io.deephaven.engine.exceptions.CancellationException;
-import io.deephaven.engine.table.lang.QueryLibrary;
-import io.deephaven.engine.table.lang.QueryScope;
+import io.deephaven.engine.context.QueryLibrary;
+import io.deephaven.engine.context.QueryScope;
 import io.deephaven.engine.updategraph.UpdateGraphProcessor;
 import io.deephaven.engine.util.AbstractScriptSession;
 import io.deephaven.engine.util.PythonEvaluator;
@@ -75,20 +75,22 @@ public class PythonDeephavenSession extends AbstractScriptSession<PythonSnapshot
      * @param objectTypeLookup the object type lookup
      * @param listener an optional listener that will be notified whenever the query scope changes
      * @param runInitScripts if init scripts should be executed
-     * @param isDefaultScriptSession true if this is in the default context of a worker jvm
      * @param pythonEvaluator
      * @throws IOException if an IO error occurs running initialization scripts
      */
     public PythonDeephavenSession(
             ObjectTypeLookup objectTypeLookup, @Nullable final Listener listener, boolean runInitScripts,
-            boolean isDefaultScriptSession, PythonEvaluatorJpy pythonEvaluator)
+            PythonEvaluatorJpy pythonEvaluator)
             throws IOException {
-        super(objectTypeLookup, listener, isDefaultScriptSession);
+        super(objectTypeLookup, listener);
 
         evaluator = pythonEvaluator;
         scope = pythonEvaluator.getScope();
-        this.module = (PythonScriptSessionModule) PyModule.importModule("deephaven.server.script_session")
-                .createProxy(CallableKind.FUNCTION, PythonScriptSessionModule.class);
+        executionContext.getQueryLibrary().importClass(org.jpy.PyObject.class);
+        try (final SafeCloseable ignored = executionContext.open()) {
+            this.module = (PythonScriptSessionModule) PyModule.importModule("deephaven.server.script_session")
+                    .createProxy(CallableKind.FUNCTION, PythonScriptSessionModule.class);
+        }
         this.scriptFinder = new ScriptFinder(DEFAULT_SCRIPT_PATH);
 
         /*
@@ -108,22 +110,14 @@ public PythonDeephavenSession(
                 runScript(script);
             }
         }
-
-        final QueryLibrary currLibrary = QueryLibrary.getLibrary();
-        try {
-            QueryLibrary.setLibrary(queryLibrary);
-            QueryLibrary.importClass(org.jpy.PyObject.class);
-        } finally {
-            QueryLibrary.setLibrary(currLibrary);
-        }
     }
 
     /**
      * Creates a Python "{@link ScriptSession}", for use where we should only be reading from the scope, such as an
      * IPython kernel session.
      */
     public PythonDeephavenSession(PythonScope<?> scope) {
-        super(NoOp.INSTANCE, null, false);
+        super(NoOp.INSTANCE, null);
         this.scope = (PythonScope<PyObject>) scope;
         this.module = null;
         this.evaluator = null;

ModelFarm/src/test/java/io/deephaven/modelfarm/TestModelFarm.java

@@ -5,12 +5,16 @@
 
 import io.deephaven.base.verify.Require;
 import io.deephaven.configuration.Configuration;
+import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.util.TableTools;
 import io.deephaven.engine.table.ColumnSource;
+import io.deephaven.util.SafeCloseable;
 import junit.framework.TestCase;
 import org.apache.commons.lang3.mutable.MutableLong;
 import org.jetbrains.annotations.NotNull;
+import org.junit.After;
+import org.junit.Before;
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -24,6 +28,22 @@ public class TestModelFarm extends TestCase {
             Configuration.getInstance().getIntegerWithDefault("TestModelFarm.testShutdownTimeoutSecs", 1);
     private final int nModelFarmThreadsDefault = 8;
 
+    private SafeCloseable executionContext;
+
+    @Before
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        executionContext = ExecutionContext.createForUnitTests().open();
+    }
+
+    @After
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        executionContext.close();
+    }
+
     /**
      * Ensure that the ModelFarm terminates immediately if it is shut down while not busy with an empty queue.
      */

ModelFarm/src/test/java/io/deephaven/modelfarm/util/TestModelFarmUtils.java

@@ -5,14 +5,34 @@
 
 import io.deephaven.base.testing.BaseArrayTestCase;
 import io.deephaven.base.verify.RequirementFailure;
+import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.engine.table.Table;
 import io.deephaven.time.DateTimeUtils;
+import io.deephaven.util.SafeCloseable;
 import io.deephaven.vector.*;
 import io.deephaven.time.DateTime;
 import io.deephaven.engine.util.TableTools;
+import org.junit.After;
+import org.junit.Before;
 
 public class TestModelFarmUtils extends BaseArrayTestCase {
 
+    private SafeCloseable executionContext;
+
+    @Before
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        executionContext = ExecutionContext.createForUnitTests().open();
+    }
+
+    @After
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        executionContext.close();
+    }
+
     public void testRequireTable() {
         final Table t = TableTools.emptyTable(5).updateView("A=(int)i", "B=(long)i", "C=(double)i");
 

Plot/src/main/java/io/deephaven/plot/datasets/category/AbstractSwappableTableBasedCategoryDataSeries.java

@@ -3,15 +3,15 @@
  */
 package io.deephaven.plot.datasets.category;
 
+import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.plot.AxesImpl;
 import io.deephaven.plot.datasets.ColumnNameConstants;
 import io.deephaven.plot.datasets.data.AssociativeDataSwappableTable;
 import io.deephaven.plot.util.ArgumentValidations;
 import io.deephaven.plot.util.functions.FigureImplFunction;
 import io.deephaven.plot.util.tables.SwappableTable;
 import io.deephaven.engine.table.Table;
-import io.deephaven.engine.table.lang.QueryLibrary;
-import io.deephaven.engine.table.lang.QueryScope;
+import io.deephaven.engine.context.QueryScope;
 import io.deephaven.gui.color.Paint;
 import io.deephaven.gui.shape.NamedShape;
 import io.deephaven.gui.shape.Shape;
@@ -146,7 +146,7 @@ protected <S, T> Function<Table, Table> constructPartitionedTableFromFunction(fi
         final String queryFunction = columnName + "Function";
         return t -> {
             QueryScope.addParam(queryFunction, function);
-            QueryLibrary.importClass(resultClass);
+            ExecutionContext.getContext().getQueryLibrary().importClass(resultClass);
             return t.update(columnName + " = (" + resultClass.getSimpleName() + ") " + queryFunction + ".apply("
                     + onColumn + ")");
         };

Plot/src/main/java/io/deephaven/plot/datasets/category/AbstractTableBasedCategoryDataSeries.java

@@ -3,13 +3,13 @@
  */
 package io.deephaven.plot.datasets.category;
 
+import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.plot.AxesImpl;
 import io.deephaven.plot.datasets.ColumnNameConstants;
 import io.deephaven.plot.util.ArgumentValidations;
 import io.deephaven.plot.util.functions.FigureImplFunction;
 import io.deephaven.engine.table.Table;
-import io.deephaven.engine.table.lang.QueryLibrary;
-import io.deephaven.engine.table.lang.QueryScope;
+import io.deephaven.engine.context.QueryScope;
 import io.deephaven.gui.color.Paint;
 
 import java.util.function.Function;
@@ -94,7 +94,7 @@ protected <S, T> Table constructTableFromFunction(final Table t, final Function<
         ArgumentValidations.assertNotNull(function, "function", getPlotInfo());
         final String queryFunction = columnName + "Function";
         QueryScope.addParam(queryFunction, function);
-        QueryLibrary.importClass(resultClass);
+        ExecutionContext.getContext().getQueryLibrary().importClass(resultClass);
         return t.update(
                 columnName + " = (" + resultClass.getSimpleName() + ") " + queryFunction + ".apply(" + onColumn + ")");
     }

Plot/src/main/java/io/deephaven/plot/datasets/multiseries/AbstractPartitionedTableHandleMultiSeries.java

@@ -3,12 +3,12 @@
  */
 package io.deephaven.plot.datasets.multiseries;
 
+import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.plot.AxesImpl;
 import io.deephaven.plot.datasets.DataSeriesInternal;
 import io.deephaven.plot.util.ArgumentValidations;
 import io.deephaven.plot.util.tables.TableBackedPartitionedTableHandle;
-import io.deephaven.engine.table.lang.QueryLibrary;
-import io.deephaven.engine.table.lang.QueryScope;
+import io.deephaven.engine.context.QueryScope;
 import io.deephaven.engine.table.PartitionedTable;
 
 import java.util.Arrays;
@@ -90,7 +90,7 @@ public TableBackedPartitionedTableHandle getPartitionedTableHandle() {
     public void applyTransform(final String columnName, final String update, final Class[] classesToImport,
             final Map<String, Object> params, boolean columnTypesPreserved) {
         ArgumentValidations.assertNull(partitionedTable, "partitionedTable must be null", getPlotInfo());
-        Arrays.stream(classesToImport).forEach(QueryLibrary::importClass);
+        Arrays.stream(classesToImport).forEach(aClass -> ExecutionContext.getContext().getQueryLibrary().importClass(aClass));
         params.forEach(QueryScope::addParam);
         partitionedTableHandle.addColumn(columnName);
 

Plot/src/main/java/io/deephaven/plot/datasets/multiseries/AbstractSwappableMultiSeries.java

@@ -3,14 +3,14 @@
  */
 package io.deephaven.plot.datasets.multiseries;
 
+import io.deephaven.engine.context.ExecutionContext;
 import io.deephaven.plot.AxesImpl;
 import io.deephaven.plot.datasets.DataSeriesInternal;
 import io.deephaven.plot.util.ArgumentValidations;
 import io.deephaven.plot.util.tables.SwappableTable;
 import io.deephaven.plot.util.tables.TableHandle;
 import io.deephaven.engine.table.Table;
-import io.deephaven.engine.table.lang.QueryLibrary;
-import io.deephaven.engine.table.lang.QueryScope;
+import io.deephaven.engine.context.QueryScope;
 import io.deephaven.engine.table.PartitionedTable;
 
 import java.util.*;
@@ -121,7 +121,7 @@ public void applyTransform(final String columnName, final String update, final C
         ArgumentValidations.assertNull(partitionedTable, "partitionedTable must be null", getPlotInfo());
         swappableTable.addColumn(columnName);
         final Function<Table, Table> tableTransform = t -> {
-            Arrays.stream(classesToImport).forEach(QueryLibrary::importClass);
+            Arrays.stream(classesToImport).forEach(aClass -> ExecutionContext.getContext().getQueryLibrary().importClass(aClass));
             params.forEach(QueryScope::addParam);
             return t.update(update);
         };