From 3f2d095e1cf5d1546b77ea354e3b443f62f784cc Mon Sep 17 00:00:00 2001 From: Nate Bauernfeind Date: Tue, 23 Apr 2024 15:16:53 -0600 Subject: [PATCH] QueryCompiler Batch Formula Compilation (#5070) This PR makes significant changes to the `QueryCompiler` API and behavior. Highlights: - introduce `QueryCompiler` as an interface and move implementation to engine-table - introduce `QueryCompilerRequest`, an builder-style immutable to capture single compilation arguments - introduce `QueryCompilerRequestProcessor` interface that has an Immediate and Batch based implementations - introduce `CompletionStageFuture` as a friendly interface for a `CompleteableFuture` that does not expose completion methods to users - added `SelectColumn#initDef` and `WhereFilter#init` that defer compilation to the provided `QueryCompilerRequestProcessor` - all call sites that initialized `SelectColumn`s or `WhereFilter`s that might initialize more than one now use a batch processor - a batch of compilation requests share a single `JavaFileManager` (reducing open file handles and compilation latency) - the `QueryCompiler` caches at most one `JavaFileManager` to decrease latency of future compilation requests - if there are enough compilation units, the `QueryCompiler` will compile in parallel using the `OperationInitializer` - Generated Formulas no longer include the destination column name in the source body; enabling more compiled-formula sharing Bug Fixes: - `WhereFilter` and `SelectColumn` formulas now share `FormulaAnalyzer`s logic to ensure variable resolution is well-defined and ordered by: - renamed columns, columns, renamed column array access, column array access, and last query scope variables - `PartionAwareSourceTable`s `selectDistinct` implementation now behaves like `QueryTable`s implementation w.r.t. result columns as input to latter formulae - When a formula generates an error it displays the original expression rather than the `QueryLanguageParser` transformed result --- .../src/main/java/io/deephaven/base/Lazy.java | 22 - .../io/deephaven/plot/colors/ColorMaps.java | 6 +- .../util/functions/ClosureBiFunction.java | 2 +- .../ClosureDoubleBinaryOperator.java | 2 +- .../functions/ClosureDoubleUnaryOperator.java | 2 +- .../plot/util/functions/ClosureFunction.java | 4 +- .../plot/util/functions/HasClosure.java | 35 + .../util/functions/SerializableClosure.java | 96 -- .../plot/util/functions/TestHasClosure.java | 37 + .../functions/TestSerializableClosure.java | 55 - .../deephaven/util/CompletionStageFuture.java | 211 ++++ .../util/CompletionStageFutureImpl.java | 346 ++++++ .../util/datastructures/CachingSupplier.java | 52 + ...Function.java => SoftCachingFunction.java} | 6 +- ...Supplier.java => SoftCachingSupplier.java} | 4 +- engine/context/build.gradle | 1 + .../engine/context/PoisonedQueryCompiler.java | 22 +- .../engine/context/QueryCompiler.java | 858 +------------ .../engine/context/QueryCompilerRequest.java | 72 ++ .../util/SynchronizedJavaFileManager.java | 134 ++ .../engine/context/TestQueryCompiler.java | 95 +- .../engine/context/QueryCompilerImpl.java | 1084 +++++++++++++++++ .../engine/table/impl/DeferredViewTable.java | 14 +- .../table/impl/PartitionAwareSourceTable.java | 41 +- .../impl/QueryCompilerRequestProcessor.java | 178 +++ .../engine/table/impl/QueryTable.java | 39 +- .../engine/table/impl/RedefinableTable.java | 6 +- .../table/impl/WouldMatchOperation.java | 5 +- .../table/impl/by/AggregationProcessor.java | 19 +- .../table/impl/by/FormulaChunkedOperator.java | 12 +- .../impl/by/typed/TypedHasherFactory.java | 12 +- .../BaseNodeOperationsRecorder.java | 19 +- .../impl/hierarchical/RollupTableImpl.java | 6 +- .../TreeNodeOperationsRecorder.java | 11 +- .../impl/hierarchical/TreeTableFilter.java | 5 +- .../impl/hierarchical/TreeTableImpl.java | 7 +- .../table/impl/lang/QueryLanguageParser.java | 91 +- .../BaseTableTransformationColumn.java | 5 + .../impl/partitioned/LongConstantColumn.java | 6 + .../partitioned/PartitionedTableImpl.java | 9 +- .../PartitionedTableProxyImpl.java | 4 +- .../impl/select/AbstractConditionFilter.java | 131 +- .../impl/select/AbstractFormulaColumn.java | 70 +- .../impl/select/AbstractRangeFilter.java | 6 +- .../select/BaseIncrementalReleaseFilter.java | 4 +- .../table/impl/select/ByteRangeFilter.java | 2 +- .../table/impl/select/CharRangeFilter.java | 2 +- .../impl/select/ComparableRangeFilter.java | 2 +- .../table/impl/select/ComposedFilter.java | 14 +- .../table/impl/select/ConditionFilter.java | 116 +- .../table/impl/select/DhFormulaColumn.java | 169 +-- .../table/impl/select/DoubleRangeFilter.java | 2 +- .../impl/select/DownsampledWhereFilter.java | 2 +- .../table/impl/select/DynamicWhereFilter.java | 2 +- .../table/impl/select/FloatRangeFilter.java | 2 +- .../table/impl/select/FunctionalColumn.java | 7 +- .../impl/select/FunctionalColumnLong.java | 7 +- .../table/impl/select/InstantRangeFilter.java | 2 +- .../table/impl/select/IntRangeFilter.java | 2 +- .../table/impl/select/LongRangeFilter.java | 2 +- .../engine/table/impl/select/MatchFilter.java | 20 +- .../select/MultiSourceFunctionalColumn.java | 7 +- .../table/impl/select/NullSelectColumn.java | 7 +- .../impl/select/QueryScopeParamTypeUtil.java | 4 +- .../impl/select/RangeConditionFilter.java | 16 +- .../impl/select/ReinterpretedColumn.java | 14 +- .../impl/select/RollingReleaseFilter.java | 2 +- .../table/impl/select/SelectColumn.java | 34 +- .../table/impl/select/ShortRangeFilter.java | 2 +- .../SingleSidedComparableRangeFilter.java | 2 +- .../table/impl/select/SourceColumn.java | 13 +- .../table/impl/select/SwitchColumn.java | 17 +- .../table/impl/select/TimeSeriesFilter.java | 2 +- .../engine/table/impl/select/WhereFilter.java | 18 +- .../impl/select/WhereFilterInvertedImpl.java | 12 +- .../impl/select/WhereFilterPatternImpl.java | 2 +- .../table/impl/select/WhereNoneFilter.java | 2 +- .../impl/select/analyzers/BaseLayer.java | 10 - .../select/analyzers/DependencyLayerBase.java | 8 - .../select/analyzers/RedirectionLayer.java | 5 - .../analyzers/SelectAndViewAnalyzer.java | 100 +- .../select/analyzers/StaticFlattenLayer.java | 5 - .../impl/select/codegen/FormulaAnalyzer.java | 132 +- .../select/codegen/JavaKernelBuilder.java | 76 +- .../impl/select/formula/FormulaFactory.java | 1 + .../select/python/FormulaColumnPython.java | 15 +- .../updateby/UpdateByOperatorFactory.java | 31 +- .../BaseRollingFormulaOperator.java | 6 +- .../BooleanRollingFormulaOperator.java | 6 +- .../ByteRollingFormulaOperator.java | 6 +- .../CharRollingFormulaOperator.java | 6 +- .../DoubleRollingFormulaOperator.java | 6 +- .../FloatRollingFormulaOperator.java | 6 +- .../IntRollingFormulaOperator.java | 6 +- .../LongRollingFormulaOperator.java | 6 +- .../ObjectRollingFormulaOperator.java | 6 +- .../ShortRollingFormulaOperator.java | 6 +- .../engine/util/AbstractScriptSession.java | 35 +- .../engine/util/DynamicCompileUtils.java | 33 +- .../engine/util/GroovyDeephavenSession.java | 155 ++- .../engine/table/impl/FuzzerTest.java | 2 +- .../impl/QueryTableWhereParallelTest.java | 2 +- .../impl/lang/TestQueryLanguageParser.java | 7 +- .../impl/select/FormulaKernelSample.java | 2 +- .../table/impl/select/FormulaSample.java | 7 +- .../impl/TestEventDrivenUpdateGraph.java | 3 +- .../util/TestCompileSimpleFunction.java | 2 +- .../scripts/TestGroovyDeephavenSession.java | 5 +- .../engine/context/TestExecutionContext.java | 2 +- .../testcase/RefreshingTableTestCase.java | 6 +- .../time/TimeLiteralReplacedExpression.java | 2 - .../parquet/base/ColumnChunkReaderImpl.java | 4 +- .../parquet/table/TableWriteBenchmark.java | 4 +- .../pagestore/topage/ChunkDictionary.java | 6 +- .../impl/util/PerformanceQueriesGeneral.java | 6 +- open-api/lang-parser/lang-parser.gradle | 1 + .../deephaven/lang/parse/ParsedDocument.java | 6 +- .../lang/completion/CompletionLookups.java | 10 +- .../replicators/ReplicateCachingSupplier.java | 8 +- .../groovy/GroovyConsoleSessionModule.java | 2 +- .../server/appmode/ApplicationTest.java | 2 +- 121 files changed, 3359 insertions(+), 1749 deletions(-) delete mode 100644 Base/src/main/java/io/deephaven/base/Lazy.java create mode 100644 Plot/src/main/java/io/deephaven/plot/util/functions/HasClosure.java delete mode 100644 Plot/src/main/java/io/deephaven/plot/util/functions/SerializableClosure.java create mode 100644 Plot/src/test/java/io/deephaven/plot/util/functions/TestHasClosure.java delete mode 100644 Plot/src/test/java/io/deephaven/plot/util/functions/TestSerializableClosure.java create mode 100644 Util/src/main/java/io/deephaven/util/CompletionStageFuture.java create mode 100644 Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java create mode 100644 Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java rename Util/src/main/java/io/deephaven/util/datastructures/{LazyCachingFunction.java => SoftCachingFunction.java} (89%) rename Util/src/main/java/io/deephaven/util/datastructures/{LazyCachingSupplier.java => SoftCachingSupplier.java} (91%) create mode 100644 engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java create mode 100644 engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java diff --git a/Base/src/main/java/io/deephaven/base/Lazy.java b/Base/src/main/java/io/deephaven/base/Lazy.java deleted file mode 100644 index 47744e787af..00000000000 --- a/Base/src/main/java/io/deephaven/base/Lazy.java +++ /dev/null @@ -1,22 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.base; - -import java.util.function.Supplier; - -public class Lazy implements Supplier { - private Supplier supplier; - - public Lazy(Supplier x) { - supplier = () -> { - T val = x.get(); - supplier = () -> val; - return val; - }; - } - - public synchronized T get() { - return supplier.get(); - } -}; diff --git a/Plot/src/main/java/io/deephaven/plot/colors/ColorMaps.java b/Plot/src/main/java/io/deephaven/plot/colors/ColorMaps.java index 27c7885a37a..92ae63bde93 100644 --- a/Plot/src/main/java/io/deephaven/plot/colors/ColorMaps.java +++ b/Plot/src/main/java/io/deephaven/plot/colors/ColorMaps.java @@ -6,7 +6,7 @@ import io.deephaven.base.verify.Require; import io.deephaven.gui.color.Color; import io.deephaven.gui.color.Paint; -import io.deephaven.plot.util.functions.SerializableClosure; +import io.deephaven.plot.util.functions.HasClosure; import io.deephaven.plot.util.Range; import groovy.lang.Closure; @@ -294,13 +294,13 @@ public static Function closureMap(final Map for (final Map.Entry, COLOR> e : map.entrySet()) { final Closure closure = e.getKey(); final COLOR color = e.getValue(); - final SerializableClosure serializableClosure = new SerializableClosure<>(closure); + final HasClosure hasClosure = new HasClosure<>(closure); final SerializablePredicate predicate = new SerializablePredicate() { private static final long serialVersionUID = 613420989214281949L; @Override public boolean test(Double aDouble) { - return serializableClosure.getClosure().call(aDouble); + return hasClosure.getClosure().call(aDouble); } }; diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureBiFunction.java b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureBiFunction.java index 15b61bc4c64..1a264d12097 100644 --- a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureBiFunction.java +++ b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureBiFunction.java @@ -8,7 +8,7 @@ /** * Wraps a {@link SerializableBiFunction} with the API of a function.
*/ -public class ClosureBiFunction extends SerializableClosure implements SerializableBiFunction { +public class ClosureBiFunction extends HasClosure implements SerializableBiFunction { private static final long serialVersionUID = 697974379939190730L; /** diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleBinaryOperator.java b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleBinaryOperator.java index 81fe43ddbf7..f0dd895ffc3 100644 --- a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleBinaryOperator.java +++ b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleBinaryOperator.java @@ -10,7 +10,7 @@ /** * A serializable closure which maps pair of doubles to doubles. */ -public class ClosureDoubleBinaryOperator extends SerializableClosure +public class ClosureDoubleBinaryOperator extends HasClosure implements DoubleBinaryOperator { private static final long serialVersionUID = -6533578879266557626L; diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleUnaryOperator.java b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleUnaryOperator.java index 211818dde1a..c8cb4d90919 100644 --- a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleUnaryOperator.java +++ b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleUnaryOperator.java @@ -10,7 +10,7 @@ /** * A serializable closure which maps doubles to doubles. */ -public class ClosureDoubleUnaryOperator extends SerializableClosure +public class ClosureDoubleUnaryOperator extends HasClosure implements DoubleUnaryOperator { private static final long serialVersionUID = -4092987117189101803L; diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureFunction.java b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureFunction.java index 8994468a8e9..fcc397d3f89 100644 --- a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureFunction.java +++ b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureFunction.java @@ -6,9 +6,9 @@ import groovy.lang.Closure; /** - * Wraps a {@link SerializableClosure} with the API of a function. + * Wraps a {@link HasClosure} with the API of a function. */ -public class ClosureFunction extends SerializableClosure implements SerializableFunction { +public class ClosureFunction extends HasClosure implements SerializableFunction { private static final long serialVersionUID = 3693316124178311688L; diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/HasClosure.java b/Plot/src/main/java/io/deephaven/plot/util/functions/HasClosure.java new file mode 100644 index 00000000000..2bc28132f75 --- /dev/null +++ b/Plot/src/main/java/io/deephaven/plot/util/functions/HasClosure.java @@ -0,0 +1,35 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.plot.util.functions; + +import io.deephaven.base.verify.Require; +import groovy.lang.Closure; + +/** + * A serializable closure. + */ +public class HasClosure { + + private final Closure closure; + + /** + * Creates a SerializableClosure instance with the {@code closure}. + * + * @param closure closure + */ + public HasClosure(final Closure closure) { + Require.neqNull(closure, "closure"); + this.closure = closure.dehydrate(); + this.closure.setResolveStrategy(Closure.DELEGATE_ONLY); + } + + /** + * Gets this SerializableClosure's closure. + * + * @return this SerializableClosure's closure + */ + public Closure getClosure() { + return closure; + } +} diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/SerializableClosure.java b/Plot/src/main/java/io/deephaven/plot/util/functions/SerializableClosure.java deleted file mode 100644 index 79117fba62a..00000000000 --- a/Plot/src/main/java/io/deephaven/plot/util/functions/SerializableClosure.java +++ /dev/null @@ -1,96 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.plot.util.functions; - -import io.deephaven.base.classloaders.MapBackedClassLoader; -import io.deephaven.base.verify.Require; -import io.deephaven.engine.util.GroovyDeephavenSession; -import groovy.lang.Closure; - -import java.io.*; - -/** - * A serializable closure. - */ -public class SerializableClosure implements Serializable { - - private static final long serialVersionUID = -1438975554979636320L; - private Closure closure; - - /** - * Creates a SerializableClosure instance with the {@code closure}. - * - * @param closure closure - */ - public SerializableClosure(final Closure closure) { - Require.neqNull(closure, "closure"); - this.closure = closure.dehydrate(); - this.closure.setResolveStrategy(Closure.DELEGATE_ONLY); - } - - /** - * Gets this SerializableClosure's closure. - * - * @return this SerializableClosure's closure - */ - public Closure getClosure() { - return closure; - } - - private void writeObject(final ObjectOutputStream oos) throws IOException { - final Class c = closure.getClass(); - final String closureClassName = c.getName(); - - final byte[] closureCode = GroovyDeephavenSession.getDynamicClass(closureClassName); - - final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - final ObjectOutputStream objectStream = new ObjectOutputStream(byteStream); - objectStream.writeObject(closure); - final byte[] closureData = byteStream.toByteArray(); - - oos.writeObject(closureClassName); - oos.writeObject(closureCode); - oos.writeObject(closureData); - } - - @SuppressWarnings("unchecked") - private void readObject(final ObjectInputStream ois) throws IOException, ClassNotFoundException { - final String closureClassName = (String) ois.readObject(); - final byte[] closureCode = (byte[]) ois.readObject(); - final byte[] closureData = (byte[]) ois.readObject(); - - final MapBackedClassLoader cl = new MapBackedClassLoader(); - cl.addClassData(closureClassName, closureCode); - final ObjectInputStream objectStream = - new CustomClassLoaderObjectInputStream<>(new ByteArrayInputStream(closureData), cl); - closure = (Closure) objectStream.readObject(); - } - - private static class CustomClassLoaderObjectInputStream extends ObjectInputStream { - - private final CLT classLoader; - - public CustomClassLoaderObjectInputStream(InputStream inputStream, CLT classLoader) throws IOException { - super(inputStream); - this.classLoader = classLoader; - } - - @Override - protected Class resolveClass(ObjectStreamClass desc) throws ClassNotFoundException, IOException { - if (classLoader != null) { - try { - return Class.forName(desc.getName(), false, classLoader); - } catch (ClassNotFoundException cnfe) { - /* - * The default implementation in ObjectInputStream handles primitive types with a map from name to - * class. Rather than duplicate the functionality, we are delegating to the super method for all - * failures that may be of this kind, as well as any case where the passed in ClassLoader fails to - * find the class. - */ - } - } - return super.resolveClass(desc); - } - } -} diff --git a/Plot/src/test/java/io/deephaven/plot/util/functions/TestHasClosure.java b/Plot/src/test/java/io/deephaven/plot/util/functions/TestHasClosure.java new file mode 100644 index 00000000000..0ecfd2413cd --- /dev/null +++ b/Plot/src/test/java/io/deephaven/plot/util/functions/TestHasClosure.java @@ -0,0 +1,37 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.plot.util.functions; + +import io.deephaven.base.testing.BaseArrayTestCase; +import groovy.lang.Closure; + +public class TestHasClosure extends BaseArrayTestCase { + + private final String value = "S"; + + private final Closure closure = new Closure(null) { + @Override + public String call() { + return value; + } + + @Override + public String call(Object... args) { + return value; + } + + @Override + public String call(Object arguments) { + return value; + } + }; + + public void testSerializableClosure() { + HasClosure hasClosure = new ClosureFunction<>(closure); + + assertEquals(value, hasClosure.getClosure().call()); + assertEquals(value, hasClosure.getClosure().call("T")); + assertEquals(value, hasClosure.getClosure().call("A", "B")); + } +} diff --git a/Plot/src/test/java/io/deephaven/plot/util/functions/TestSerializableClosure.java b/Plot/src/test/java/io/deephaven/plot/util/functions/TestSerializableClosure.java deleted file mode 100644 index 07de3ddc68a..00000000000 --- a/Plot/src/test/java/io/deephaven/plot/util/functions/TestSerializableClosure.java +++ /dev/null @@ -1,55 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.plot.util.functions; - -import io.deephaven.base.testing.BaseArrayTestCase; -import groovy.lang.Closure; - -public class TestSerializableClosure extends BaseArrayTestCase { - - private final String value = "S"; - - private final Closure closure = new Closure(null) { - @Override - public String call() { - return value; - } - - @Override - public String call(Object... args) { - return value; - } - - @Override - public String call(Object arguments) { - return value; - } - }; - - public void testSerializableClosure() { - SerializableClosure serializableClosure = new ClosureFunction(closure); - - assertEquals(value, serializableClosure.getClosure().call()); - assertEquals(value, serializableClosure.getClosure().call("T")); - assertEquals(value, serializableClosure.getClosure().call("A", "B")); - - // testing serialization is too hard - /* - * Object copy = null; try { ByteArrayOutputStream bos = new ByteArrayOutputStream(); ObjectOutputStream oos = - * new ObjectOutputStream(bos); oos.writeObject(serializableClosure); oos.flush(); byte[] obis = - * bos.toByteArray(); oos.close(); bos.close(); - * - * ByteArrayInputStream bis = new ByteArrayInputStream(obis); ObjectInputStream ois = new - * ObjectInputStream(bis); copy = ois.readObject(); ois.close(); bis.close(); } catch (IOException | - * ClassNotFoundException e) { e.printStackTrace(); } - * - * serializableClosure = (SerializableClosure) copy; if(serializableClosure == null) { - * TestCase.fail("Null return from serialization"); } - * - * assertEquals(value, serializableClosure.getClosure().call()); - */ - } - - -} diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java new file mode 100644 index 00000000000..ed24c99ba4d --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java @@ -0,0 +1,211 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.util; + +import org.jetbrains.annotations.NotNull; + +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * A safe version of CompletableFuture that does not expose the completion API. + * + * @param The result type returned by this future's {@link #get()} + */ +public interface CompletionStageFuture extends Future, CompletionStage { + + /** + * Create a new incomplete future. + * + * @param The result type returned by this future's {@link #get()} + * @return a resolver for the future + */ + static Resolver make() { + return new CompletionStageFutureImpl().new ResolverImpl(); + } + + /** + * Returns a new CompletionStageFuture that is already completed with the given value. + * + * @param value the value + * @param the type of the value + * @return the completed CompletionStageFuture + * @see java.util.concurrent.CompletableFuture#completedFuture(Object) + */ + static CompletionStageFuture completedFuture(U value) { + final CompletionStageFutureImpl.Resolver resolver = CompletionStageFuture.make(); + resolver.complete(value); + return resolver.getFuture(); + } + + /** + * Returns a new CompletionStageFuture that is already completed exceptionally with the given exception. + * + * @param ex the exception + * @param the type of the value + * @return the exceptionally completed CompletionStageFuture + * @see java.util.concurrent.CompletableFuture#failedFuture(Throwable) + */ + static CompletionStageFuture failedFuture(Throwable ex) { + final CompletionStageFutureImpl.Resolver resolver = CompletionStageFuture.make(); + resolver.completeExceptionally(ex); + return resolver.getFuture(); + } + + interface Resolver { + + /** + * If not already completed, sets the value returned by {@link #get()} and related methods to the given value. + * + * @param value the result value + * @return {@code true} if this invocation caused this CompletionStageFuture to transition to a completed state, + * else {@code false} + * @see java.util.concurrent.CompletableFuture#complete(Object) + */ + boolean complete(T value); + + /** + * If not already completed, causes invocations of {@link #get()} and related methods to throw the given + * exception wrapped in an {@link ExecutionException}. + * + * @param ex the exception + * @return {@code true} if this invocation caused this CompletionStageFuture to transition to a completed state, + * else {@code false} + * @see java.util.concurrent.CompletableFuture#completeExceptionally(Throwable) + */ + boolean completeExceptionally(@NotNull Throwable ex); + + /** + * @return the underlying future to provide to the recipient; implementations must ensure that this method + * always returns an identical result for a given Resolver instance + */ + CompletionStageFuture getFuture(); + } + + @Override + CompletionStageFuture thenApply(Function fn); + + @Override + CompletionStageFuture thenApplyAsync(Function fn); + + @Override + CompletionStageFuture thenApplyAsync(Function fn, Executor executor); + + @Override + CompletionStageFuture thenAccept(Consumer action); + + @Override + CompletionStageFuture thenAcceptAsync(Consumer action); + + @Override + CompletionStageFuture thenAcceptAsync(Consumer action, Executor executor); + + @Override + CompletionStageFuture thenRun(Runnable action); + + @Override + CompletionStageFuture thenRunAsync(Runnable action); + + @Override + CompletionStageFuture thenRunAsync(Runnable action, Executor executor); + + @Override + CompletionStageFuture thenCombine( + CompletionStage other, BiFunction fn); + + @Override + CompletionStageFuture thenCombineAsync( + CompletionStage other, BiFunction fn); + + @Override + CompletionStageFuture thenCombineAsync( + CompletionStage other, BiFunction fn, Executor executor); + + @Override + CompletionStageFuture thenAcceptBoth( + CompletionStage other, BiConsumer action); + + @Override + CompletionStageFuture thenAcceptBothAsync( + CompletionStage other, BiConsumer action); + + @Override + CompletionStageFuture thenAcceptBothAsync( + CompletionStage other, BiConsumer action, Executor executor); + + @Override + CompletionStageFuture runAfterBoth(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterBothAsync(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterBothAsync(CompletionStage other, Runnable action, Executor executor); + + @Override + CompletionStageFuture applyToEither(CompletionStage other, Function fn); + + @Override + CompletionStageFuture applyToEitherAsync(CompletionStage other, Function fn); + + @Override + CompletionStageFuture applyToEitherAsync( + CompletionStage other, Function fn, Executor executor); + + @Override + CompletionStageFuture acceptEither(CompletionStage other, Consumer action); + + @Override + CompletionStageFuture acceptEitherAsync(CompletionStage other, Consumer action); + + @Override + CompletionStageFuture acceptEitherAsync( + CompletionStage other, Consumer action, Executor executor); + + @Override + CompletionStageFuture runAfterEither(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterEitherAsync(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterEitherAsync(CompletionStage other, Runnable action, Executor executor); + + @Override + CompletionStageFuture thenCompose(Function> fn); + + @Override + CompletionStageFuture thenComposeAsync(Function> fn); + + @Override + CompletionStageFuture thenComposeAsync( + Function> fn, Executor executor); + + @Override + CompletionStageFuture handle(BiFunction fn); + + @Override + CompletionStageFuture handleAsync(BiFunction fn); + + @Override + CompletionStageFuture handleAsync(BiFunction fn, Executor executor); + + @Override + CompletionStageFuture whenComplete(BiConsumer action); + + @Override + CompletionStageFuture whenCompleteAsync(BiConsumer action); + + @Override + CompletionStageFuture whenCompleteAsync(BiConsumer action, Executor executor); + + @Override + CompletionStageFuture exceptionally(Function fn); +} diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java new file mode 100644 index 00000000000..df86fb942b0 --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java @@ -0,0 +1,346 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.util; + +import org.jetbrains.annotations.NotNull; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.function.*; + +/** + * A {@link CompletableFuture} that prevents users from completing the future by hiding behind + * {@link CompletionStageFuture}. + * + * @param The result type returned by this future's {@code join} + */ +@SuppressWarnings("unchecked") +public class CompletionStageFutureImpl extends CompletableFuture implements CompletionStageFuture { + + /** + * A resolver for this future implementation. + */ + class ResolverImpl implements CompletionStageFuture.Resolver { + public boolean complete(final T value) { + return safelyComplete(value); + } + + public boolean completeExceptionally(@NotNull final Throwable ex) { + return safelyCompleteExceptionally(ex); + } + + public CompletionStageFuture getFuture() { + return CompletionStageFutureImpl.this; + } + } + + private boolean safelyComplete(final T value) { + return super.complete(value); + } + + private boolean safelyCompleteExceptionally(@NotNull final Throwable ex) { + return super.completeExceptionally(ex); + } + + @Override + public CompletableFuture newIncompleteFuture() { + return new CompletionStageFutureImpl<>(); + } + + /////////////////////////// + // CompletableFuture API // + /////////////////////////// + + @Override + public boolean complete(final T value) { + throw erroneousCompletionException(); + } + + @Override + public boolean completeExceptionally(@NotNull final Throwable ex) { + throw erroneousCompletionException(); + } + + @Override + public void obtrudeValue(final T value) { + throw erroneousCompletionException(); + } + + @Override + public void obtrudeException(@NotNull final Throwable ex) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeAsync( + @NotNull final Supplier supplier, + @NotNull final Executor executor) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeAsync(@NotNull final Supplier supplier) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeOnTimeout(final T value, long timeout, TimeUnit unit) { + throw erroneousCompletionException(); + } + + private static UnsupportedOperationException erroneousCompletionException() { + return new UnsupportedOperationException("Users should not complete futures."); + } + + ///////////////////////// + // CompletionStage API // + ///////////////////////// + + @Override + public CompletionStageFutureImpl thenApply(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.thenApply(fn); + } + + @Override + public CompletionStageFutureImpl thenApplyAsync(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.thenApplyAsync(fn); + } + + @Override + public CompletionStageFutureImpl thenApplyAsync( + @NotNull final Function fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenApplyAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl thenAccept(@NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.thenAccept(action); + } + + @Override + public CompletionStageFutureImpl thenAcceptAsync(@NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.thenAcceptAsync(action); + } + + @Override + public CompletionStageFutureImpl thenAcceptAsync( + @NotNull final Consumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenAcceptAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl thenRun(@NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.thenRun(action); + } + + @Override + public CompletionStageFutureImpl thenRunAsync(@NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.thenRunAsync(action); + } + + @Override + public CompletionStageFutureImpl thenRunAsync( + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenRunAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl thenCombine( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.thenCombine(other, fn); + } + + @Override + public CompletionStageFutureImpl thenCombineAsync( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.thenCombineAsync(other, fn); + } + + @Override + public CompletionStageFutureImpl thenCombineAsync( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenCombineAsync(other, fn, executor); + } + + + @Override + public CompletionStageFutureImpl thenAcceptBoth( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.thenAcceptBoth(other, action); + } + + + @Override + public CompletionStageFutureImpl thenAcceptBothAsync( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.thenAcceptBothAsync(other, action); + } + + @Override + public CompletionStageFutureImpl thenAcceptBothAsync( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenAcceptBothAsync(other, action, executor); + } + + + @Override + public CompletionStageFutureImpl runAfterBoth( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterBoth(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterBothAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterBothAsync(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterBothAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.runAfterBothAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl applyToEither( + @NotNull final CompletionStage other, + @NotNull final Function fn) { + return (CompletionStageFutureImpl) super.applyToEither(other, fn); + } + + @Override + public CompletionStageFutureImpl applyToEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Function fn) { + return (CompletionStageFutureImpl) super.applyToEitherAsync(other, fn); + } + + @Override + public CompletionStageFutureImpl applyToEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Function fn, Executor executor) { + return (CompletionStageFutureImpl) super.applyToEitherAsync(other, fn, executor); + } + + @Override + public CompletionStageFutureImpl acceptEither( + @NotNull final CompletionStage other, + @NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.acceptEither(other, action); + } + + @Override + public CompletionStageFutureImpl acceptEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.acceptEitherAsync(other, action); + } + + @Override + public CompletionStageFutureImpl acceptEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Consumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.acceptEitherAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl runAfterEither( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterEither(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterEitherAsync(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.runAfterEitherAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl thenCompose( + @NotNull final Function> fn) { + return (CompletionStageFutureImpl) super.thenCompose(fn); + } + + @Override + public CompletionStageFutureImpl thenComposeAsync( + @NotNull final Function> fn) { + return (CompletionStageFutureImpl) super.thenComposeAsync(fn); + } + + @Override + public CompletionStageFutureImpl thenComposeAsync( + @NotNull final Function> fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenComposeAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl handle(@NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.handle(fn); + } + + @Override + public CompletionStageFutureImpl handleAsync( + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.handleAsync(fn); + } + + @Override + public CompletionStageFutureImpl handleAsync( + @NotNull final BiFunction fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.handleAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl whenComplete(@NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.whenComplete(action); + } + + @Override + public CompletionStageFutureImpl whenCompleteAsync( + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.whenCompleteAsync(action); + } + + @Override + public CompletionStageFutureImpl whenCompleteAsync( + @NotNull final BiConsumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.whenCompleteAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl exceptionally(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.exceptionally(fn); + } +} diff --git a/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java new file mode 100644 index 00000000000..0a0880f7896 --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java @@ -0,0 +1,52 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.util.datastructures; + +import org.jetbrains.annotations.NotNull; + +import java.util.function.Supplier; + +/** + * {@link Supplier} wrapper that caches the result. + * + * @param the type of results supplied by this supplier + */ +public final class CachingSupplier implements Supplier { + + private final Supplier internalSupplier; + + private volatile boolean hasCachedResult; + private OUTPUT_TYPE cachedResult; + private RuntimeException errorResult; + + /** + * Construct a {@link Supplier} wrapper. + * + * @param internalSupplier The {@link Supplier} to wrap. + */ + public CachingSupplier(@NotNull final Supplier internalSupplier) { + this.internalSupplier = internalSupplier; + } + + @Override + public OUTPUT_TYPE get() { + if (!hasCachedResult) { + synchronized (this) { + if (!hasCachedResult) { + try { + cachedResult = internalSupplier.get(); + } catch (RuntimeException err) { + errorResult = err; + } + hasCachedResult = true; + } + } + } + + if (errorResult != null) { + throw errorResult; + } + return cachedResult; + } +} diff --git a/Util/src/main/java/io/deephaven/util/datastructures/LazyCachingFunction.java b/Util/src/main/java/io/deephaven/util/datastructures/SoftCachingFunction.java similarity index 89% rename from Util/src/main/java/io/deephaven/util/datastructures/LazyCachingFunction.java rename to Util/src/main/java/io/deephaven/util/datastructures/SoftCachingFunction.java index 05b654fc1bc..e3521335875 100644 --- a/Util/src/main/java/io/deephaven/util/datastructures/LazyCachingFunction.java +++ b/Util/src/main/java/io/deephaven/util/datastructures/SoftCachingFunction.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit LazyCachingSupplier and run "./gradlew replicateCachingSupplier" to regenerate +// ****** Edit SoftCachingSupplier and run "./gradlew replicateCachingSupplier" to regenerate // // @formatter:off package io.deephaven.util.datastructures; @@ -18,7 +18,7 @@ * * @param the type of results supplied by this function */ -public final class LazyCachingFunction implements Function { +public final class SoftCachingFunction implements Function { private final Function internalFunction; @@ -29,7 +29,7 @@ public final class LazyCachingFunction implements Funct * * @param internalFunction The {@link Function} to wrap. Must be safely repeatable and must not return {@code null}. */ - public LazyCachingFunction(@NotNull final Function internalFunction) { + public SoftCachingFunction(@NotNull final Function internalFunction) { this.internalFunction = internalFunction; } diff --git a/Util/src/main/java/io/deephaven/util/datastructures/LazyCachingSupplier.java b/Util/src/main/java/io/deephaven/util/datastructures/SoftCachingSupplier.java similarity index 91% rename from Util/src/main/java/io/deephaven/util/datastructures/LazyCachingSupplier.java rename to Util/src/main/java/io/deephaven/util/datastructures/SoftCachingSupplier.java index 478d33c0901..788a61c954e 100644 --- a/Util/src/main/java/io/deephaven/util/datastructures/LazyCachingSupplier.java +++ b/Util/src/main/java/io/deephaven/util/datastructures/SoftCachingSupplier.java @@ -14,7 +14,7 @@ * * @param the type of results supplied by this supplier */ -public final class LazyCachingSupplier implements Supplier { +public final class SoftCachingSupplier implements Supplier { private final Supplier internalSupplier; @@ -25,7 +25,7 @@ public final class LazyCachingSupplier implements Supplier internalSupplier) { + public SoftCachingSupplier(@NotNull final Supplier internalSupplier) { this.internalSupplier = internalSupplier; } diff --git a/engine/context/build.gradle b/engine/context/build.gradle index 0f683dcb5b9..da73cfac799 100644 --- a/engine/context/build.gradle +++ b/engine/context/build.gradle @@ -24,6 +24,7 @@ dependencies { implementation 'com.github.f4b6a3:uuid-creator:5.2.0' Classpaths.inheritCommonsText(project, 'implementation') + Classpaths.inheritImmutables(project) testImplementation TestTools.projectDependency(project, 'Base') testImplementation project(':engine-test-utils') diff --git a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java index d6909402ced..72a78c7173a 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java @@ -3,14 +3,11 @@ // package io.deephaven.engine.context; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.ExecutionContextRegistrationException; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import java.io.File; -import java.util.Map; - -public class PoisonedQueryCompiler extends QueryCompiler { +public class PoisonedQueryCompiler implements QueryCompiler { public static final PoisonedQueryCompiler INSTANCE = new PoisonedQueryCompiler(); @@ -21,18 +18,9 @@ private T fail() { } @Override - public File getFakeClassDestination() { - return fail(); - } - - @Override - public void setParentClassLoader(ClassLoader parentClassLoader) { + public void compile( + @NotNull final QueryCompilerRequest[] requests, + @NotNull final CompletionStageFuture.Resolver>[] resolvers) { fail(); } - - @Override - public Class compile(@NotNull String className, @NotNull String classBody, @NotNull String packageNameRoot, - @Nullable StringBuilder codeLog, @NotNull Map> parameterClasses) { - return fail(); - } } diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 8f67f1ab2c2..c198d28ebf7 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -4,851 +4,57 @@ package io.deephaven.engine.context; import io.deephaven.UncheckedDeephavenException; -import io.deephaven.base.FileUtils; -import io.deephaven.base.Pair; -import io.deephaven.configuration.Configuration; -import io.deephaven.configuration.DataDir; -import io.deephaven.datastructures.util.CollectionUtil; -import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.logger.Logger; -import io.deephaven.util.ByteUtils; -import org.apache.commons.text.StringEscapeUtils; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import javax.tools.*; -import java.io.*; -import java.lang.reflect.Field; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.*; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import java.util.jar.Attributes; -import java.util.jar.Manifest; -import java.util.stream.Collectors; -import java.util.stream.Stream; -public class QueryCompiler { - private static final Logger log = LoggerFactory.getLogger(QueryCompiler.class); - /** - * We pick a number just shy of 65536, leaving a little elbow room for good luck. - */ - private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; - - private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); - private static final int MAX_CLASS_COLLISIONS = 128; - - private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; - - private static final String CODEGEN_TIMEOUT_PROP = "QueryCompiler.codegen.timeoutMs"; - private static final long CODEGEN_TIMEOUT_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); // 10 seconds - private static final String CODEGEN_LOOP_DELAY_PROP = "QueryCompiler.codegen.retry.delay"; - private static final long CODEGEN_LOOP_DELAY_MS_DEFAULT = 100; - private static final long codegenTimeoutMs = - Configuration.getInstance().getLongWithDefault(CODEGEN_TIMEOUT_PROP, CODEGEN_TIMEOUT_MS_DEFAULT); - private static final long codegenLoopDelayMs = - Configuration.getInstance().getLongWithDefault(CODEGEN_LOOP_DELAY_PROP, CODEGEN_LOOP_DELAY_MS_DEFAULT); - - private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); - - public static final String FORMULA_PREFIX = "io.deephaven.temp"; - public static final String DYNAMIC_GROOVY_CLASS_PREFIX = "io.deephaven.dynamic"; - - public static QueryCompiler create(File cacheDirectory, ClassLoader classLoader) { - return new QueryCompiler(cacheDirectory, classLoader, true); - } - - static QueryCompiler createForUnitTests() { - final Path queryCompilerDir = DataDir.get() - .resolve("io.deephaven.engine.context.QueryCompiler.createForUnitTests"); - return new QueryCompiler(queryCompilerDir.toFile()); - } - - private final Map>> knownClasses = new HashMap<>(); - - private final String[] dynamicPatterns = new String[] {DYNAMIC_GROOVY_CLASS_PREFIX, FORMULA_PREFIX}; - - private final File classDestination; - private final boolean isCacheDirectory; - private final Set additionalClassLocations; - private volatile WritableURLClassLoader ucl; - - /** package-private constructor for {@link io.deephaven.engine.context.PoisonedQueryCompiler} */ - QueryCompiler() { - classDestination = null; - isCacheDirectory = false; - additionalClassLocations = null; - } - - private QueryCompiler(File classDestination) { - this(classDestination, null, false); - } - - private QueryCompiler( - final File classDestination, - final ClassLoader parentClassLoader, - final boolean isCacheDirectory) { - final ClassLoader parentClassLoaderToUse = parentClassLoader == null - ? QueryCompiler.class.getClassLoader() - : parentClassLoader; - this.classDestination = classDestination; - this.isCacheDirectory = isCacheDirectory; - ensureDirectories(this.classDestination, () -> "Failed to create missing class destination directory " + - classDestination.getAbsolutePath()); - additionalClassLocations = new LinkedHashSet<>(); - - URL[] urls = new URL[1]; - try { - urls[0] = (classDestination.toURI().toURL()); - } catch (MalformedURLException e) { - throw new UncheckedDeephavenException(e); - } - this.ucl = new WritableURLClassLoader(urls, parentClassLoaderToUse); - - if (isCacheDirectory) { - addClassSource(classDestination); - } - } - - /** - * Enables or disables compilation logging. - * - * @param logEnabled Whether or not logging should be enabled - * @return The value of {@code logEnabled} before calling this method. - */ - public static boolean setLogEnabled(boolean logEnabled) { - boolean original = QueryCompiler.logEnabled; - QueryCompiler.logEnabled = logEnabled; - return original; - } - - /* - * NB: This is (obviously) not thread safe if code tries to write the same className to the same - * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving - * synchronization as an external concern. - */ - public static void writeClass(final File destinationDirectory, final String className, final byte[] data) - throws IOException { - writeClass(destinationDirectory, className, data, null); - } - - /* - * NB: This is (obviously) not thread safe if code tries to write the same className to the same - * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving - * synchronization as an external concern. - */ - public static void writeClass(final File destinationDirectory, final String className, final byte[] data, - final String message) throws IOException { - final File destinationFile = new File(destinationDirectory, - className.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - - if (destinationFile.exists()) { - final byte[] existingBytes = Files.readAllBytes(destinationFile.toPath()); - if (Arrays.equals(existingBytes, data)) { - if (message == null) { - log.info().append("Ignoring pushed class ").append(className) - .append(" because it already exists in this context!").endl(); - } else { - log.info().append("Ignoring pushed class ").append(className).append(message) - .append(" because it already exists in this context!").endl(); - } - return; - } else { - if (message == null) { - log.info().append("Pushed class ").append(className) - .append(" already exists in this context, but has changed!").endl(); - } else { - log.info().append("Pushed class ").append(className).append(message) - .append(" already exists in this context, but has changed!").endl(); - } - if (!destinationFile.delete()) { - throw new IOException("Could not delete existing class file: " + destinationFile); - } - } - } - - final File parentDir = destinationFile.getParentFile(); - ensureDirectories(parentDir, - () -> "Unable to create missing destination directory " + parentDir.getAbsolutePath()); - if (!destinationFile.createNewFile()) { - throw new UncheckedDeephavenException( - "Unable to create destination file " + destinationFile.getAbsolutePath()); - } - final ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(data.length); - byteOutStream.write(data, 0, data.length); - final FileOutputStream fileOutStream = new FileOutputStream(destinationFile); - byteOutStream.writeTo(fileOutStream); - fileOutStream.close(); - } - - public File getFakeClassDestination() { - // Groovy classes need to be written out to a location where they can be found by the compiler - // (so that filters and formulae can use them). - // - // We don't want the regular runtime class loader to find them, because then they get "stuck" in there - // even if the class itself changes, and we can't forget it. So instead we use a single-use class loader - // for each formula, that will always read the class from disk. - return isCacheDirectory ? classDestination : null; - } - - public void setParentClassLoader(final ClassLoader parentClassLoader) { - ucl = new WritableURLClassLoader(ucl.getURLs(), parentClassLoader); - } - - public final Class compile(String className, String classBody, String packageNameRoot) { - return compile(className, classBody, packageNameRoot, null, Collections.emptyMap()); - } - - public final Class compile(String className, String classBody, String packageNameRoot, - Map> parameterClasses) { - return compile(className, classBody, packageNameRoot, null, parameterClasses); - } - - public final Class compile(String className, String classBody, String packageNameRoot, StringBuilder codeLog) { - return compile(className, classBody, packageNameRoot, codeLog, Collections.emptyMap()); - } +public interface QueryCompiler { /** - * Compile a class. + * Compile a class and wait for completion. * - * @param className Class name - * @param classBody Class body, before update with "$CLASS_NAME$" replacement and package name prefixing - * @param packageNameRoot Package name prefix - * @param codeLog Optional "log" for final class code - * @param parameterClasses Generic parameters, empty if none required - * @return The compiled class - */ - public Class compile(@NotNull final String className, - @NotNull final String classBody, - @NotNull final String packageNameRoot, - @Nullable final StringBuilder codeLog, - @NotNull final Map> parameterClasses) { - CompletableFuture> future; - final boolean alreadyExists; - - synchronized (this) { - future = knownClasses.get(classBody); - if (future != null) { - alreadyExists = true; - } else { - future = new CompletableFuture<>(); - knownClasses.put(classBody, future); - alreadyExists = false; - } - } - - // Someone else has already made the future. I'll just wait for the answer. - if (alreadyExists) { - try { - return future.get(); - } catch (InterruptedException | ExecutionException error) { - throw new UncheckedDeephavenException(error); - } - } - - // It's my job to fulfill the future. - try { - return compileHelper(className, classBody, packageNameRoot, codeLog, parameterClasses, future); - } catch (RuntimeException e) { - future.completeExceptionally(e); - throw e; - } - } - - private static void ensureDirectories(final File file, final Supplier runtimeErrMsg) { - // File.mkdirs() checks for existence on entry, in which case it returns false. - // It may also return false on a failure to create. - // Also note, two separate threads or JVMs may be running this code in parallel. It's possible that we could - // lose the race - // (and therefore mkdirs() would return false), but still get the directory we need (and therefore exists() - // would return true) - if (!file.mkdirs() && !file.isDirectory()) { - throw new UncheckedDeephavenException(runtimeErrMsg.get()); - } - } - - private ClassLoader getClassLoaderForFormula(final Map> parameterClasses) { - return new URLClassLoader(ucl.getURLs(), ucl) { - // Once we find a class that is missing, we should not attempt to load it again, - // otherwise we can end up with a StackOverflow Exception - final HashSet missingClasses = new HashSet<>(); - - @Override - protected Class findClass(String name) throws ClassNotFoundException { - // If we have a parameter that uses this class, return it - final Class paramClass = parameterClasses.get(name); - if (paramClass != null) { - return paramClass; - } - - // Unless we are looking for a formula or Groovy class, we should use the default behavior - if (!isFormulaClass(name)) { - return super.findClass(name); - } - - // if it is a groovy class, always try to use the instance in the shell - if (name.startsWith(DYNAMIC_GROOVY_CLASS_PREFIX)) { - try { - return ucl.getParent().loadClass(name); - } catch (final ClassNotFoundException ignored) { - // we'll try to load it otherwise - } - } - - // We've already not found this class, so we should not try to search again - if (missingClasses.contains(name)) { - return super.findClass(name); - } - - final byte[] bytes; - try { - bytes = loadClassData(name); - } catch (IOException ioe) { - missingClasses.add(name); - return super.loadClass(name); - } - return defineClass(name, bytes, 0, bytes.length); - } - - @SuppressWarnings("BooleanMethodIsAlwaysInverted") - private boolean isFormulaClass(String name) { - return Arrays.stream(dynamicPatterns).anyMatch(name::startsWith); - } - - @Override - public Class loadClass(String name) throws ClassNotFoundException { - if (!isFormulaClass(name)) { - return super.loadClass(name); - } - return findClass(name); - } - - private byte[] loadClassData(String name) throws IOException { - final File destFile = new File(classDestination, - name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - if (destFile.exists()) { - return Files.readAllBytes(destFile.toPath()); - } - - for (File location : additionalClassLocations) { - final File checkFile = new File(location, - name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - if (checkFile.exists()) { - return Files.readAllBytes(checkFile.toPath()); - } - } - - throw new FileNotFoundException(name); - } - }; - } - - private static class WritableURLClassLoader extends URLClassLoader { - private WritableURLClassLoader(URL[] urls, ClassLoader parent) { - super(urls, parent); - } - - @Override - protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { - Class clazz = findLoadedClass(name); - if (clazz != null) { - return clazz; - } - - try { - clazz = findClass(name); - } catch (ClassNotFoundException e) { - if (getParent() != null) { - clazz = getParent().loadClass(name); - } - } - - if (resolve) { - resolveClass(clazz); - } - return clazz; - } - - @Override - public synchronized void addURL(URL url) { - super.addURL(url); - } - } - - private void addClassSource(File classSourceDirectory) { - synchronized (additionalClassLocations) { - if (additionalClassLocations.contains(classSourceDirectory)) { - return; - } - additionalClassLocations.add(classSourceDirectory); - } - try { - ucl.addURL(classSourceDirectory.toURI().toURL()); - } catch (MalformedURLException e) { - throw new UncheckedDeephavenException(e); - } - } - - private File getClassDestination() { - return classDestination; - } - - private String getClassPath() { - StringBuilder sb = new StringBuilder(); - sb.append(classDestination.getAbsolutePath()); - synchronized (additionalClassLocations) { - for (File classLoc : additionalClassLocations) { - sb.append(File.pathSeparatorChar).append(classLoc.getAbsolutePath()); - } - } - return sb.toString(); - } - - private Class compileHelper(@NotNull final String className, - @NotNull final String classBody, - @NotNull final String packageNameRoot, - @Nullable final StringBuilder codeLog, - @NotNull final Map> parameterClasses, - @NotNull final CompletableFuture> resultFuture) { - final MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); - } - final String basicHashText = - ByteUtils.byteArrToHex(digest.digest(classBody.getBytes(StandardCharsets.UTF_8))); - - for (int pi = 0; pi < MAX_CLASS_COLLISIONS; ++pi) { - final String packageNameSuffix = "c_" + basicHashText - + (pi == 0 ? "" : ("p" + pi)) - + "v" + JAVA_CLASS_VERSION; - final String packageName = (packageNameRoot.isEmpty() - ? packageNameSuffix - : packageNameRoot + (packageNameRoot.endsWith(".") ? "" : ".") + packageNameSuffix); - final String fqClassName = packageName + "." + className; - - // Ask the classloader to load an existing class with this name. This might: - // 1. Fail to find a class (returning null) - // 2. Find a class whose body has the formula we are looking for - // 3. Find a class whose body has a different formula (hash collision) - Class result = tryLoadClassByFqName(fqClassName, parameterClasses); - if (result == null) { - // Couldn't find one, so try to create it. This might: - // A. succeed - // B. Lose a race to another process on the same file system which is compiling the identical formula - // C. Lose a race to another process on the same file system compiling a different formula that - // happens to have the same hash (same packageName). - // However, regardless of A-C, there will be *some* class being found (i.e. tryLoadClassByFqName won't - // return null). - maybeCreateClass(className, classBody, packageName, fqClassName); - - // We could be running on a screwy filesystem that is slow (e.g. NFS). - // If we wrote a file and can't load it ... then give the filesystem some time. - result = tryLoadClassByFqName(fqClassName, parameterClasses); - try { - final long deadline = System.currentTimeMillis() + codegenTimeoutMs - codegenLoopDelayMs; - while (result == null && System.currentTimeMillis() < deadline) { - Thread.sleep(codegenLoopDelayMs); - result = tryLoadClassByFqName(fqClassName, parameterClasses); - } - } catch (InterruptedException ignored) { - // we got interrupted, just quit looping and ignore it. - } - - if (result == null) { - throw new IllegalStateException("Should have been able to load *some* class here"); - } - } - - final String identifyingFieldValue = loadIdentifyingField(result); - - // If the class we found was indeed the class we were looking for, then complete the future and return it. - if (classBody.equals(identifyingFieldValue)) { - if (codeLog != null) { - // If the caller wants a textual copy of the code we either made, or just found in the cache. - codeLog.append(makeFinalCode(className, classBody, packageName)); - } - resultFuture.complete(result); - synchronized (this) { - // Note we are doing something kind of subtle here. We are removing an entry whose key was matched - // by value equality and replacing it with a value-equal but reference-different string that is a - // static member of the class we just loaded. This should be easier on the garbage collector because - // we are replacing a calculated value with a classloaded value and so in effect we are - // "canonicalizing" the string. This is important because these long strings stay in knownClasses - // forever. - knownClasses.remove(identifyingFieldValue); - knownClasses.put(identifyingFieldValue, resultFuture); - } - return result; - } - // Try the next hash name - } - throw new IllegalStateException("Found too many collisions for package name root " + packageNameRoot - + ", class name=" + className - + ", class body hash=" + basicHashText + " - contact Deephaven support!"); - } - - private Class tryLoadClassByFqName(String fqClassName, Map> parameterClasses) { - try { - return getClassLoaderForFormula(parameterClasses).loadClass(fqClassName); - } catch (ClassNotFoundException cnfe) { - return null; - } - } - - private static String loadIdentifyingField(Class c) { - try { - final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); - return (String) field.get(null); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new IllegalStateException("Malformed class in cache", e); - } - } - - private static String makeFinalCode(String className, String classBody, String packageName) { - final String joinedEscapedBody = createEscapedJoinedString(classBody); - classBody = classBody.replaceAll("\\$CLASSNAME\\$", className); - classBody = classBody.substring(0, classBody.lastIndexOf("}")); - classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; - return "package " + packageName + ";\n" + classBody; - } - - /** - * Transform a string into the corresponding Java source code that compiles into that string. This involves escaping - * special characters, surrounding it with quotes, and (if the string is larger than the max string length for Java - * literals), splitting it into substrings and constructing a call to String.join() that combines those substrings. + * @param request The compilation request */ - public static String createEscapedJoinedString(final String originalString) { - return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); - } - - public static String createEscapedJoinedString(final String originalString, int maxStringLength) { - final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); - - // Turn each split into a Java source string by escaping it and surrounding it with " - for (int ii = 0; ii < splits.length; ++ii) { - final String escaped = StringEscapeUtils.escapeJava(splits[ii]); - splits[ii] = "\"" + escaped + "\""; - - } - assert splits.length > 0; - if (splits.length == 1) { - return splits[0]; - } - final String formattedInnards = String.join(",\n", splits); - return "String.join(\"\", " + formattedInnards + ")"; - } - - private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { - final List splits = new ArrayList<>(); - // exclusive end position of the previous substring. - int previousEnd = 0; - // Number of bytes in the "modified UTF-8" representation of the substring we are currently scanning. - int currentByteCount = 0; - for (int ii = 0; ii < originalString.length(); ++ii) { - final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); - if (currentByteCount + bytesConsumed > maxBytes) { - // This character won't fit in this string, so we flush the buffer. - splits.add(originalString.substring(previousEnd, ii)); - previousEnd = ii; - currentByteCount = 0; - } - currentByteCount += bytesConsumed; - } - // At the end of the loop, either - // 1. there are one or more characters that still need to be added to splits - // 2. originalString was empty and so splits is empty and we need to add a single empty string to splits - splits.add(originalString.substring(previousEnd)); - return splits.toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY); - } - - private static int calcBytesConsumed(final char ch) { - if (ch == 0) { - return 2; - } - if (ch <= 0x7f) { - return 1; - } - if (ch <= 0x7ff) { - return 2; - } - return 3; - } - - private static class JavaSourceFromString extends SimpleJavaFileObject { - final String code; - - JavaSourceFromString(String name, String code) { - super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); - this.code = code; - } - - public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return code; - } - } - - private static class JavaSourceFromFile extends SimpleJavaFileObject { - private static final int JAVA_LENGTH = Kind.SOURCE.extension.length(); - final String code; - - private JavaSourceFromFile(File basePath, File file) { - super(URI.create("string:///" + createName(basePath, file).replace('.', '/') + Kind.SOURCE.extension), - Kind.SOURCE); - try { - this.code = FileUtils.readTextFile(file); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static String createName(File basePath, File file) { - final String base = basePath.getAbsolutePath(); - final String fileName = file.getAbsolutePath(); - if (!fileName.startsWith(base)) { - throw new IllegalArgumentException(file + " is not in " + basePath); - } - final String basename = fileName.substring(base.length()); - if (basename.endsWith(".java")) { - return basename.substring(0, basename.length() - JAVA_LENGTH); - } else { - return basename; - } - } - - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return code; - } - } - - private void maybeCreateClass(String className, String code, String packageName, String fqClassName) { - final String finalCode = makeFinalCode(className, code, packageName); - - if (logEnabled) { - log.info().append("Generating code ").append(finalCode).endl(); - } - - final File ctxClassDestination = getClassDestination(); - - final String[] splitPackageName = packageName.split("\\."); - if (splitPackageName.length == 0) { - throw new UncheckedDeephavenException(String.format( - "packageName %s expected to have at least one .", packageName)); - } - final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); - - // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package - // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. - // Also create a temp directory e.g. /tmp/workspace/cache/classes/temporaryCompilationDirectory12345 - // This temp directory will be where the compiler drops files into, e.g. - // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/Formula.class - // Foreshadowing: we will eventually atomically move cm12862183232603186v52_0 from the above to - // /tmp/workspace/cache/classes/io/deephaven/test - // Note: for this atomic move to work, this temp directory must be on the same file system as the destination - // directory. - final String rootPathAsString; - final String tempDirAsString; - try { - rootPathAsString = ctxClassDestination.getAbsolutePath(); - final Path rootPathWithPackage = Paths.get(rootPathAsString, truncatedSplitPackageName); - final File rpf = rootPathWithPackage.toFile(); - ensureDirectories(rpf, () -> "Couldn't create package directories: " + rootPathWithPackage); - final Path tempPath = - Files.createTempDirectory(Paths.get(rootPathAsString), "temporaryCompilationDirectory"); - tempDirAsString = tempPath.toFile().getAbsolutePath(); - } catch (IOException ioe) { - throw new UncheckedIOException(ioe); - } - - try { - maybeCreateClassHelper(fqClassName, finalCode, splitPackageName, rootPathAsString, tempDirAsString); - } finally { - try { - FileUtils.deleteRecursively(new File(tempDirAsString)); - } catch (Exception e) { - // ignore errors here - } - } - } - - private void maybeCreateClassHelper(String fqClassName, String finalCode, String[] splitPackageName, - String rootPathAsString, String tempDirAsString) { - final StringWriter compilerOutput = new StringWriter(); - - final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); - } - - final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); - final List compilerOptions = Arrays.asList("-d", tempDirAsString, "-cp", classPathAsString); - - final JavaFileManager fileManager = compiler.getStandardFileManager(null, null, null); - - boolean result = false; - boolean exceptionThrown = false; + @FinalDefault + default Class compile(@NotNull final QueryCompilerRequest request) { + final CompletionStageFuture.Resolver> resolver = CompletionStageFuture.make(); + compile(request, resolver); try { - result = compiler.getTask(compilerOutput, - fileManager, - null, - compilerOptions, - null, - Collections.singletonList(new JavaSourceFromString(fqClassName, finalCode))) - .call(); - } catch (final Throwable err) { - exceptionThrown = true; - throw err; - } finally { - try { - fileManager.close(); - } catch (final IOException ioe) { - if (!exceptionThrown) { - // noinspection ThrowFromFinallyBlock - throw new UncheckedIOException("Could not close JavaFileManager", ioe); - } - } - } - if (!result) { - throw new UncheckedDeephavenException("Error compiling class " + fqClassName + ":\n" + compilerOutput); - } - // The above has compiled into e.g. - // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various - // class files} - // We want to atomically move it to e.g. - // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} - Path srcDir = Paths.get(tempDirAsString, splitPackageName); - Path destDir = Paths.get(rootPathAsString, splitPackageName); - try { - Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException ioe) { - // The move might have failed for a variety of bad reasons. However, if the reason was because - // we lost the race to some other process, that's a harmless/desirable outcome, and we can ignore - // it. - if (!Files.exists(destDir)) { - throw new UncheckedIOException("Move failed for some reason other than destination already existing", - ioe); + return resolver.getFuture().get(); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; } + throw new UncheckedDeephavenException("Error while compiling class", cause); + } catch (InterruptedException e) { + throw new UncheckedDeephavenException("Interrupted while compiling class", e); } } /** - * Try to compile the set of files, returning a pair of success and compiler output. + * Compile a class. * - * @param basePath the base path for the java classes - * @param javaFiles the java source files - * @return a Pair of success, and the compiler output + * @param request The compilation request + * @param resolver The resolver to use for delivering compilation results */ - private Pair tryCompile(File basePath, Collection javaFiles) throws IOException { - final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); - } - - final File outputDirectory = Files.createTempDirectory("temporaryCompilationDirectory").toFile(); - - try { - final StringWriter compilerOutput = new StringWriter(); - final String javaClasspath = getJavaClassPath(); - - final Collection javaFileObjects = javaFiles.stream() - .map(f -> new JavaSourceFromFile(basePath, f)).collect(Collectors.toList()); - - final boolean result = compiler.getTask(compilerOutput, null, null, - Arrays.asList("-d", outputDirectory.getAbsolutePath(), "-cp", - getClassPath() + File.pathSeparator + javaClasspath), - null, javaFileObjects).call(); - - return new Pair<>(result, compilerOutput.toString()); - } finally { - FileUtils.deleteRecursively(outputDirectory); - } + @FinalDefault + default void compile( + @NotNull final QueryCompilerRequest request, + @NotNull final CompletionStageFuture.Resolver> resolver) { + // noinspection unchecked + compile(new QueryCompilerRequest[] {request}, new CompletionStageFuture.Resolver[] {resolver}); } /** - * Retrieve the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables. + * Compiles all requests. * - * @return + * @param requests The compilation requests; these must be independent of each other + * @param resolvers The resolvers to use for delivering compilation results */ - private static String getJavaClassPath() { - String javaClasspath; - { - final StringBuilder javaClasspathBuilder = new StringBuilder(System.getProperty("java.class.path")); - - final String teamCityWorkDir = System.getProperty("teamcity.build.workingDir"); - if (teamCityWorkDir != null) { - // We are running in TeamCity, get the classpath differently - final File[] classDirs = new File(teamCityWorkDir + "/_out_/classes").listFiles(); - - for (File f : classDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); - } - final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); - - for (File f : testDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); - } - - final File[] jars = FileUtils.findAllFiles(new File(teamCityWorkDir + "/lib")); - for (File f : jars) { - if (f.getName().endsWith(".jar")) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); - } - } - } - javaClasspath = javaClasspathBuilder.toString(); - } - - // IntelliJ will bundle a very large class path into an empty jar with a Manifest that will define the full - // class path - // Look for this being used during compile time, so the full class path can be sent into the compile call - final String intellijClassPathJarRegex = ".*classpath[0-9]*\\.jar.*"; - if (javaClasspath.matches(intellijClassPathJarRegex)) { - try { - final Enumeration resources = - QueryCompiler.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); - final Attributes.Name createdByAttribute = new Attributes.Name("Created-By"); - final Attributes.Name classPathAttribute = new Attributes.Name("Class-Path"); - while (resources.hasMoreElements()) { - // Check all manifests -- looking for the Intellij created one - final Manifest manifest = new Manifest(resources.nextElement().openStream()); - final Attributes attributes = manifest.getMainAttributes(); - final Object createdBy = attributes.get(createdByAttribute); - if ("IntelliJ IDEA".equals(createdBy)) { - final String extendedClassPath = (String) attributes.get(classPathAttribute); - if (extendedClassPath != null) { - // Parses the files in the manifest description an changes their format to drop the "file:/" - // and - // use the default path separator - final String filePaths = Stream.of(extendedClassPath.split("file:/")) - .map(String::trim) - .filter(fileName -> fileName.length() > 0) - .collect(Collectors.joining(File.pathSeparator)); - - // Remove the classpath jar in question, and expand it with the files from the manifest - javaClasspath = Stream.of(javaClasspath.split(File.pathSeparator)) - .map(cp -> cp.matches(intellijClassPathJarRegex) ? filePaths : cp) - .collect(Collectors.joining(File.pathSeparator)); - } - } - } - } catch (IOException e) { - throw new UncheckedIOException("Error extract manifest file from " + javaClasspath + ".\n", e); - } - } - return javaClasspath; - } + void compile( + @NotNull QueryCompilerRequest[] requests, + @NotNull CompletionStageFuture.Resolver>[] resolvers); } diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java new file mode 100644 index 00000000000..6f7608fa5ef --- /dev/null +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java @@ -0,0 +1,72 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.context; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Immutable; + +import java.util.Map; +import java.util.Optional; + +/** + * A request to compile a java class. + */ +@Immutable +@BuildableStyle +public abstract class QueryCompilerRequest { + public static Builder builder() { + return ImmutableQueryCompilerRequest.builder(); + } + + /** + * @return the description to add to the query performance recorder nugget for this request + */ + public abstract String description(); + + /** + * @return the class name to use for the generated class + */ + public abstract String className(); + + /** + * @return the class body, before update with "$CLASS_NAME$" replacement and package name prefixing + */ + public abstract String classBody(); + + /** + * @return the package name prefix + */ + public abstract String packageNameRoot(); + + /** Optional "log" for final class code. */ + public abstract Optional codeLog(); + + /** + * @return the generic parameters, empty if none required + */ + public abstract Map> parameterClasses(); + + String getPackageName(final String packageNameSuffix) { + final String root = packageNameRoot(); + return root.isEmpty() + ? packageNameSuffix + : root + (root.endsWith(".") ? "" : ".") + packageNameSuffix; + } + + public interface Builder { + Builder description(String description); + + Builder className(String className); + + Builder classBody(String classBody); + + Builder packageNameRoot(String packageNameRoot); + + Builder codeLog(StringBuilder codeLog); + + Builder putAllParameterClasses(Map> parameterClasses); + + QueryCompilerRequest build(); + } +} diff --git a/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java b/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java new file mode 100644 index 00000000000..d03a5bfdb81 --- /dev/null +++ b/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java @@ -0,0 +1,134 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.context.util; + +import javax.tools.FileObject; +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; +import java.io.IOException; +import java.util.Iterator; +import java.util.ServiceLoader; +import java.util.Set; + +public class SynchronizedJavaFileManager implements JavaFileManager { + + private final JavaFileManager delegate; + + public SynchronizedJavaFileManager(JavaFileManager delegate) { + this.delegate = delegate; + } + + @Override + public synchronized ClassLoader getClassLoader(Location location) { + return delegate.getClassLoader(location); + } + + @Override + public synchronized Iterable list( + Location location, + String packageName, + Set kinds, + boolean recurse) throws IOException { + return delegate.list(location, packageName, kinds, recurse); + } + + @Override + public synchronized String inferBinaryName(Location location, JavaFileObject file) { + return delegate.inferBinaryName(location, file); + } + + @Override + public synchronized boolean isSameFile(FileObject a, FileObject b) { + return delegate.isSameFile(a, b); + } + + @Override + public synchronized boolean handleOption(String current, Iterator remaining) { + return delegate.handleOption(current, remaining); + } + + @Override + public synchronized boolean hasLocation(Location location) { + return delegate.hasLocation(location); + } + + @Override + public synchronized JavaFileObject getJavaFileForInput( + Location location, + String className, + JavaFileObject.Kind kind) throws IOException { + return delegate.getJavaFileForInput(location, className, kind); + } + + @Override + public synchronized JavaFileObject getJavaFileForOutput( + Location location, + String className, + JavaFileObject.Kind kind, + FileObject sibling) throws IOException { + return delegate.getJavaFileForOutput(location, className, kind, sibling); + } + + @Override + public synchronized FileObject getFileForInput( + Location location, + String packageName, + String relativeName) throws IOException { + return delegate.getFileForInput(location, packageName, relativeName); + } + + @Override + public synchronized FileObject getFileForOutput( + Location location, + String packageName, + String relativeName, + FileObject sibling) throws IOException { + return delegate.getFileForOutput(location, packageName, relativeName, sibling); + } + + @Override + public synchronized void flush() throws IOException { + delegate.flush(); + } + + @Override + public synchronized void close() throws IOException { + delegate.close(); + } + + @Override + public synchronized int isSupportedOption(String option) { + return delegate.isSupportedOption(option); + } + + @Override + public synchronized Location getLocationForModule(Location location, String moduleName) throws IOException { + return delegate.getLocationForModule(location, moduleName); + } + + @Override + public synchronized Location getLocationForModule(Location location, JavaFileObject fo) throws IOException { + return delegate.getLocationForModule(location, fo); + } + + @Override + public synchronized ServiceLoader getServiceLoader(Location location, Class service) throws IOException { + return delegate.getServiceLoader(location, service); + } + + @Override + public synchronized String inferModuleName(Location location) throws IOException { + return delegate.inferModuleName(location); + } + + @Override + public synchronized Iterable> listLocationsForModules(Location location) throws IOException { + return delegate.listLocationsForModules(location); + } + + @Override + public synchronized boolean contains(Location location, FileObject fo) throws IOException { + return delegate.contains(location, fo); + } +} diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index 461e2ae232d..d9bca87a113 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -3,9 +3,12 @@ // package io.deephaven.engine.context; +import io.deephaven.UncheckedDeephavenException; +import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.time.DateTimeUtils; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.SafeCloseable; import org.junit.After; import org.junit.Before; @@ -18,8 +21,8 @@ import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; public class TestQueryCompiler { private final static int NUM_THREADS = 500; @@ -68,7 +71,8 @@ public void setUp() throws IOException { executionContextClosable = ExecutionContext.newBuilder() .captureQueryLibrary() .captureQueryScope() - .setQueryCompiler(QueryCompiler.create(folder.newFolder(), TestQueryCompiler.class.getClassLoader())) + .setQueryCompiler(QueryCompilerImpl.create( + folder.newFolder(), TestQueryCompiler.class.getClassLoader())) .build() .open(); } @@ -165,7 +169,7 @@ private void sleepIgnoringInterruptions(final long waitMillis) { } } - private void compile(boolean printDetails, final String className) throws Exception { + private void compile(boolean printDetails, final String className) { final long startMillis; if (printDetails) { startMillis = System.currentTimeMillis(); @@ -173,8 +177,13 @@ private void compile(boolean printDetails, final String className) throws Except } else { startMillis = 0; } - ExecutionContext.getContext().getQueryCompiler() - .compile(className, CLASS_CODE, "io.deephaven.temp"); + ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className(className) + .classBody(CLASS_CODE) + .packageNameRoot("io.deephaven.temp") + .build()); if (printDetails) { final long endMillis = System.currentTimeMillis(); System.out.println(printMillis(endMillis) + ": Thread 0 ending compile: (" + (endMillis - startMillis) @@ -201,8 +210,14 @@ public void testSimpleCompile() throws Exception { "}"); StringBuilder codeLog = new StringBuilder(); - final Class clazz1 = ExecutionContext.getContext().getQueryCompiler() - .compile("Test", program1Text, "com.deephaven.test", codeLog, Collections.emptyMap()); + final Class clazz1 = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className("Test") + .classBody(program1Text) + .packageNameRoot("com.deephaven.test") + .codeLog(codeLog) + .build()); final Method m1 = clazz1.getMethod("main", String[].class); Object[] args1 = new Object[] {new String[] {"hello", "there"}}; m1.invoke(null, args1); @@ -224,21 +239,75 @@ public void testCollidingCompile() throws Exception { Thread t = new Thread(() -> { StringBuilder codeLog = new StringBuilder(); try { - final Class clazz1 = ExecutionContext.getContext().getQueryCompiler() - .compile("Test", program1Text, "com.deephaven.test", codeLog, - Collections.emptyMap()); + final Class clazz1 = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className("Test") + .classBody(program1Text) + .packageNameRoot("com.deephaven.test") + .codeLog(codeLog) + .build()); final Method m1 = clazz1.getMethod("main", String[].class); Object[] args1 = new Object[] {new String[] {"hello", "there"}}; m1.invoke(null, args1); } catch (Exception e) { - throw new RuntimeException(e); + throw new UncheckedDeephavenException(e); } }); t.start(); threads.add(t); } - for (int i = 0; i < threads.size(); ++i) { - threads.get(i).join(); + for (final Thread thread : threads) { + thread.join(); } } + + @Test + public void testMultiCompileWithFailure() throws ExecutionException, InterruptedException { + final String goodProgram = String.join( + "\n", + "public class GoodTest {", + " public static void main (String [] args) {", + " }", + "}"); + final String badProgram = String.join( + "\n", + "public class BadTest {", + " public static void main (String [] args) {", + " }", + "}}"); + + QueryCompilerRequest[] requests = new QueryCompilerRequest[] { + QueryCompilerRequest.builder() + .description("Test Bad Compile") + .className("BadTest") + .classBody(badProgram) + .packageNameRoot("com.deephaven.test") + .build(), + QueryCompilerRequest.builder() + .description("Test Good Compile") + .className("GoodTest") + .classBody(goodProgram) + .packageNameRoot("com.deephaven.test") + .build(), + }; + + // noinspection unchecked + CompletionStageFuture.Resolver>[] resolvers = + (CompletionStageFuture.Resolver>[]) new CompletionStageFuture.Resolver[] { + CompletionStageFuture.make(), + CompletionStageFuture.make(), + }; + + try { + ExecutionContext.getContext().getQueryCompiler().compile(requests, resolvers); + // noinspection DataFlowIssue + throw Assert.statementNeverExecuted(); + } catch (Exception ignored) { + } + + Assert.eqTrue(resolvers[0].getFuture().isDone(), "resolvers[0].getFuture().isDone()"); + Assert.eqTrue(resolvers[1].getFuture().isDone(), "resolvers[0].getFuture().isDone()"); + Assert.neqNull(resolvers[1].getFuture().get(), "resolvers[1].getFuture().get()"); + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java new file mode 100644 index 00000000000..535b8acc573 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -0,0 +1,1084 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.context; + +import io.deephaven.UncheckedDeephavenException; +import io.deephaven.base.FileUtils; +import io.deephaven.base.log.LogOutput; +import io.deephaven.base.log.LogOutputAppendable; +import io.deephaven.base.verify.Assert; +import io.deephaven.configuration.Configuration; +import io.deephaven.configuration.DataDir; +import io.deephaven.datastructures.util.CollectionUtil; +import io.deephaven.engine.context.util.SynchronizedJavaFileManager; +import io.deephaven.engine.table.impl.perf.BasePerformanceEntry; +import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; +import io.deephaven.engine.table.impl.util.ImmediateJobScheduler; +import io.deephaven.engine.table.impl.util.JobScheduler; +import io.deephaven.engine.table.impl.util.OperationInitializerJobScheduler; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.log.impl.LogOutputStringImpl; +import io.deephaven.io.logger.Logger; +import io.deephaven.util.ByteUtils; +import io.deephaven.util.CompletionStageFuture; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.commons.text.StringEscapeUtils; +import org.jetbrains.annotations.NotNull; + +import javax.tools.*; +import java.io.*; +import java.lang.reflect.Field; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.jar.Attributes; +import java.util.jar.Manifest; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class QueryCompilerImpl implements QueryCompiler, LogOutputAppendable { + + private static final Logger log = LoggerFactory.getLogger(QueryCompilerImpl.class); + /** + * We pick a number just shy of 65536, leaving a little elbow room for good luck. + */ + private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; + + private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); + private static final int MAX_CLASS_COLLISIONS = 128; + + private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; + + private static final String CODEGEN_TIMEOUT_PROP = "QueryCompiler.codegen.timeoutMs"; + private static final long CODEGEN_TIMEOUT_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); // 10 seconds + private static final String CODEGEN_LOOP_DELAY_PROP = "QueryCompiler.codegen.retry.delay"; + private static final long CODEGEN_LOOP_DELAY_MS_DEFAULT = 100; + private static final long CODEGEN_TIMEOUT_MS = + Configuration.getInstance().getLongWithDefault(CODEGEN_TIMEOUT_PROP, CODEGEN_TIMEOUT_MS_DEFAULT); + private static final long CODEGEN_LOOP_DELAY_MS = + Configuration.getInstance().getLongWithDefault(CODEGEN_LOOP_DELAY_PROP, CODEGEN_LOOP_DELAY_MS_DEFAULT); + + private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); + + private static JavaCompiler compiler; + private static final AtomicReference fileManagerCache = new AtomicReference<>(); + + private static void ensureJavaCompiler() { + synchronized (QueryCompilerImpl.class) { + if (compiler == null) { + compiler = ToolProvider.getSystemJavaCompiler(); + if (compiler == null) { + throw new UncheckedDeephavenException( + "No Java compiler provided - are you using a JRE instead of a JDK?"); + } + } + } + } + + private static JavaFileManager acquireFileManager() { + JavaFileManager fileManager = fileManagerCache.getAndSet(null); + if (fileManager == null) { + fileManager = new SynchronizedJavaFileManager(compiler.getStandardFileManager(null, null, null)); + } + return fileManager; + } + + private static void releaseFileManager(@NotNull final JavaFileManager fileManager) { + // Reusing the file manager saves a lot of the time in the compilation process. However, we need to be careful + // to avoid keeping too many file handles open so we'll limit ourselves to just one outstanding file manager. + if (!fileManagerCache.compareAndSet(null, fileManager)) { + try { + fileManager.close(); + } catch (final IOException err) { + throw new UncheckedIOException("Could not close JavaFileManager", err); + } + } + } + + public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; + public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; + + public static QueryCompilerImpl create(File cacheDirectory, ClassLoader classLoader) { + return new QueryCompilerImpl(cacheDirectory, classLoader, true); + } + + static QueryCompilerImpl createForUnitTests() { + final Path queryCompilerDir = DataDir.get() + .resolve("io.deephaven.engine.context.QueryCompiler.createForUnitTests"); + return new QueryCompilerImpl(queryCompilerDir.toFile(), QueryCompilerImpl.class.getClassLoader(), false); + } + + private final Map>> knownClasses = new HashMap<>(); + + private final String[] dynamicPatterns = new String[] {DYNAMIC_CLASS_PREFIX, FORMULA_CLASS_PREFIX}; + + private final File classDestination; + private final Set additionalClassLocations; + private final WritableURLClassLoader ucl; + + private QueryCompilerImpl( + @NotNull final File classDestination, + @NotNull final ClassLoader parentClassLoader, + boolean classDestinationIsAlsoClassSource) { + ensureJavaCompiler(); + + this.classDestination = classDestination; + ensureDirectories(this.classDestination, () -> "Failed to create missing class destination directory " + + classDestination.getAbsolutePath()); + additionalClassLocations = new LinkedHashSet<>(); + + URL[] urls = new URL[1]; + try { + urls[0] = (classDestination.toURI().toURL()); + } catch (MalformedURLException e) { + throw new UncheckedDeephavenException(e); + } + this.ucl = new WritableURLClassLoader(urls, parentClassLoader); + + if (classDestinationIsAlsoClassSource) { + addClassSource(classDestination); + } + } + + @Override + public LogOutput append(LogOutput logOutput) { + return logOutput.append("QueryCompiler{classDestination=").append(classDestination.getAbsolutePath()) + .append("}"); + } + + @Override + public String toString() { + return new LogOutputStringImpl().append(this).toString(); + } + + /** + * Enables or disables compilation logging. + * + * @param logEnabled Whether logging should be enabled + * @return The value of {@code logEnabled} before calling this method. + */ + public static boolean setLogEnabled(boolean logEnabled) { + boolean original = QueryCompilerImpl.logEnabled; + QueryCompilerImpl.logEnabled = logEnabled; + return original; + } + + /* + * NB: This is (obviously) not thread safe if code tries to write the same className to the same + * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving + * synchronization as an external concern. + */ + public static void writeClass(final File destinationDirectory, final String className, final byte[] data) + throws IOException { + writeClass(destinationDirectory, className, data, null); + } + + /* + * NB: This is (obviously) not thread safe if code tries to write the same className to the same + * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving + * synchronization as an external concern. + */ + public static void writeClass(final File destinationDirectory, final String className, final byte[] data, + final String message) throws IOException { + final File destinationFile = new File(destinationDirectory, + className.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); + + if (destinationFile.exists()) { + final byte[] existingBytes = Files.readAllBytes(destinationFile.toPath()); + if (Arrays.equals(existingBytes, data)) { + if (message == null) { + log.info().append("Ignoring pushed class ").append(className) + .append(" because it already exists in this context!").endl(); + } else { + log.info().append("Ignoring pushed class ").append(className).append(message) + .append(" because it already exists in this context!").endl(); + } + return; + } else { + if (message == null) { + log.info().append("Pushed class ").append(className) + .append(" already exists in this context, but has changed!").endl(); + } else { + log.info().append("Pushed class ").append(className).append(message) + .append(" already exists in this context, but has changed!").endl(); + } + if (!destinationFile.delete()) { + throw new IOException("Could not delete existing class file: " + destinationFile); + } + } + } + + final File parentDir = destinationFile.getParentFile(); + ensureDirectories(parentDir, + () -> "Unable to create missing destination directory " + parentDir.getAbsolutePath()); + if (!destinationFile.createNewFile()) { + throw new UncheckedDeephavenException( + "Unable to create destination file " + destinationFile.getAbsolutePath()); + } + final ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(data.length); + byteOutStream.write(data, 0, data.length); + final FileOutputStream fileOutStream = new FileOutputStream(destinationFile); + byteOutStream.writeTo(fileOutStream); + fileOutStream.close(); + } + + @Override + public void compile( + @NotNull final QueryCompilerRequest[] requests, + @NotNull final CompletionStageFuture.Resolver>[] resolvers) { + if (requests.length == 0) { + return; + } + if (requests.length != resolvers.length) { + throw new IllegalArgumentException("Requests and resolvers must be the same length"); + } + + // noinspection unchecked + final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; + + final List newRequests = new ArrayList<>(); + final List>> newResolvers = new ArrayList<>(); + + synchronized (this) { + for (int ii = 0; ii < requests.length; ++ii) { + final QueryCompilerRequest request = requests[ii]; + final CompletionStageFuture.Resolver> resolver = resolvers[ii]; + + CompletionStageFuture> future = + knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); + if (future == null) { + newRequests.add(request); + newResolvers.add(resolver); + future = resolver.getFuture(); + } + allFutures[ii] = future; + } + } + + if (!newResolvers.isEmpty()) { + // It's my job to fulfill these futures. + try { + compileHelper(newRequests, newResolvers); + } catch (RuntimeException e) { + // These failures are not applicable to a single request, so we can't just complete the future and + // leave the failure in the cache. + synchronized (this) { + for (int ii = 0; ii < newRequests.size(); ++ii) { + if (newResolvers.get(ii).completeExceptionally(e)) { + knownClasses.remove(newRequests.get(ii).classBody()); + } + } + } + throw e; + } + } + + for (int ii = 0; ii < requests.length; ++ii) { + try { + resolvers[ii].complete(allFutures[ii].get()); + } catch (ExecutionException err) { + resolvers[ii].completeExceptionally(err.getCause()); + } catch (InterruptedException err) { + // This can only occur if we are interrupted while waiting for the future to complete from another + // compilation request. + Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures[ii], "allFutures[ii]"); + resolvers[ii].completeExceptionally(err); + } catch (Throwable err) { + resolvers[ii].completeExceptionally(err); + } + } + } + + private static void ensureDirectories(final File file, final Supplier runtimeErrMsg) { + // File.mkdirs() checks for existence on entry, in which case it returns false. + // It may also return false on a failure to create. + // Also note, two separate threads or JVMs may be running this code in parallel. It's possible that we could + // lose the race + // (and therefore mkdirs() would return false), but still get the directory we need (and therefore exists() + // would return true) + if (!file.mkdirs() && !file.isDirectory()) { + throw new UncheckedDeephavenException(runtimeErrMsg.get()); + } + } + + private ClassLoader getClassLoaderForFormula(final Map> parameterClasses) { + return new URLClassLoader(ucl.getURLs(), ucl) { + // Once we find a class that is missing, we should not attempt to load it again, + // otherwise we can end up with a StackOverflow Exception + final HashSet missingClasses = new HashSet<>(); + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + // If we have a parameter that uses this class, return it + final Class paramClass = parameterClasses.get(name); + if (paramClass != null) { + return paramClass; + } + + // Unless we are looking for a formula or Groovy class, we should use the default behavior + if (!isFormulaClass(name)) { + return super.findClass(name); + } + + // if it is a groovy class, always try to use the instance in the shell + if (name.startsWith(DYNAMIC_CLASS_PREFIX)) { + try { + return ucl.getParent().loadClass(name); + } catch (final ClassNotFoundException ignored) { + // we'll try to load it otherwise + } + } + + // We've already not found this class, so we should not try to search again + if (missingClasses.contains(name)) { + return super.findClass(name); + } + + final byte[] bytes; + try { + bytes = loadClassData(name); + } catch (IOException ioe) { + missingClasses.add(name); + return super.loadClass(name); + } + return defineClass(name, bytes, 0, bytes.length); + } + + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + private boolean isFormulaClass(String name) { + return Arrays.stream(dynamicPatterns).anyMatch(name::startsWith); + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + if (!isFormulaClass(name)) { + return super.loadClass(name); + } + return findClass(name); + } + + private byte[] loadClassData(String name) throws IOException { + final File destFile = new File(classDestination, + name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); + if (destFile.exists()) { + return Files.readAllBytes(destFile.toPath()); + } + + for (File location : additionalClassLocations) { + final File checkFile = new File(location, + name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); + if (checkFile.exists()) { + return Files.readAllBytes(checkFile.toPath()); + } + } + + throw new FileNotFoundException(name); + } + }; + } + + private static class WritableURLClassLoader extends URLClassLoader { + private WritableURLClassLoader(URL[] urls, ClassLoader parent) { + super(urls, parent); + } + + @Override + protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + Class clazz = findLoadedClass(name); + if (clazz != null) { + return clazz; + } + + try { + clazz = findClass(name); + } catch (ClassNotFoundException e) { + if (getParent() != null) { + clazz = getParent().loadClass(name); + } + } + + if (resolve) { + resolveClass(clazz); + } + return clazz; + } + + @Override + public synchronized void addURL(URL url) { + super.addURL(url); + } + } + + private void addClassSource(File classSourceDirectory) { + synchronized (additionalClassLocations) { + if (additionalClassLocations.contains(classSourceDirectory)) { + return; + } + additionalClassLocations.add(classSourceDirectory); + } + try { + ucl.addURL(classSourceDirectory.toURI().toURL()); + } catch (MalformedURLException e) { + throw new UncheckedDeephavenException(e); + } + } + + private File getClassDestination() { + return classDestination; + } + + private String getClassPath() { + StringBuilder sb = new StringBuilder(); + sb.append(classDestination.getAbsolutePath()); + synchronized (additionalClassLocations) { + for (File classLoc : additionalClassLocations) { + sb.append(File.pathSeparatorChar).append(classLoc.getAbsolutePath()); + } + } + return sb.toString(); + } + + private static class CompilationState { + int nextProbeIndex; + boolean complete; + String packageName; + String fqClassName; + } + + private void compileHelper( + @NotNull final List requests, + @NotNull final List>> resolvers) { + final MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); + } + + final String[] basicHashText = new String[requests.size()]; + for (int ii = 0; ii < requests.size(); ++ii) { + basicHashText[ii] = ByteUtils.byteArrToHex(digest.digest( + requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); + } + + int numComplete = 0; + final CompilationState[] states = new CompilationState[requests.size()]; + for (int ii = 0; ii < requests.size(); ++ii) { + states[ii] = new CompilationState(); + } + + /* + * @formatter:off + * 1. try to resolve without compiling; retain next hash to try + * 2. compile all remaining with a single compilation task + * 3. goto step 1 if any are unresolved + * @formatter:on + */ + + while (numComplete < requests.size()) { + for (int ii = 0; ii < requests.size(); ++ii) { + final CompilationState state = states[ii]; + if (state.complete) { + continue; + } + + next_probe: while (true) { + final int pi = state.nextProbeIndex++; + final String packageNameSuffix = "c_" + basicHashText[ii] + + (pi == 0 ? "" : ("p" + pi)) + + "v" + JAVA_CLASS_VERSION; + + final QueryCompilerRequest request = requests.get(ii); + if (pi >= MAX_CLASS_COLLISIONS) { + Exception err = new IllegalStateException("Found too many collisions for package name root " + + request.packageNameRoot() + ", class name=" + request.className() + ", class body " + + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); + resolvers.get(ii).completeExceptionally(err); + state.complete = true; + ++numComplete; + break; + } + + state.packageName = request.getPackageName(packageNameSuffix); + state.fqClassName = state.packageName + "." + request.className(); + + for (int jj = 0; jj < ii; ++jj) { + if (states[jj].fqClassName.equals(state.fqClassName)) { + // collision within batch + continue next_probe; + } + } + + // Ask the classloader to load an existing class with this name. This might: + // 1. Fail to find a class (returning null) + // 2. Find a class whose body has the formula we are looking for + // 3. Find a class whose body has a different formula (hash collision) + Class result = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); + if (result == null) { + break; // we'll try to compile it + } + + if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolvers.get(ii), + result)) { + state.complete = true; + ++numComplete; + break; + } + } + } + + if (numComplete == requests.size()) { + return; + } + + // Couldn't resolve at least one of the requests, so try a round of compilation. + final List compilationRequestAttempts = new ArrayList<>(); + for (int ii = 0; ii < requests.size(); ++ii) { + final CompilationState state = states[ii]; + if (!state.complete) { + final QueryCompilerRequest request = requests.get(ii); + compilationRequestAttempts.add(new CompilationRequestAttempt( + request, + state.packageName, + state.fqClassName, + resolvers.get(ii))); + } + } + + maybeCreateClasses(compilationRequestAttempts); + + // We could be running on a screwy filesystem that is slow (e.g. NFS). If we wrote a file and can't load it + // ... then give the filesystem some time. All requests should use the same deadline. + final long deadline = System.currentTimeMillis() + CODEGEN_TIMEOUT_MS - CODEGEN_LOOP_DELAY_MS; + for (int ii = 0; ii < requests.size(); ++ii) { + final CompilationState state = states[ii]; + if (state.complete) { + continue; + } + + final QueryCompilerRequest request = requests.get(ii); + final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); + if (resolver.getFuture().isDone()) { + state.complete = true; + ++numComplete; + continue; + } + + // This request may have: + // A. succeeded + // B. Lost a race to another process on the same file system which is compiling the identical formula + // C. Lost a race to another process on the same file system compiling a different formula that collides + + Class clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); + try { + while (clazz == null && System.currentTimeMillis() < deadline) { + // noinspection BusyWait + Thread.sleep(CODEGEN_LOOP_DELAY_MS); + clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); + } + } catch (final InterruptedException ie) { + throw new UncheckedDeephavenException("Interrupted while waiting for codegen", ie); + } + + // However, regardless of A-C, there will be *some* class being found + if (clazz == null) { + throw new IllegalStateException("Should have been able to load *some* class here"); + } + + if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolver, clazz)) { + state.complete = true; + ++numComplete; + } + } + } + } + + private boolean completeIfResultMatchesQueryCompilerRequest( + final String packageName, + final QueryCompilerRequest request, + final CompletionStageFuture.Resolver> resolver, + final Class result) { + final String identifyingFieldValue = loadIdentifyingField(result); + if (!request.classBody().equals(identifyingFieldValue)) { + return false; + } + + // If the caller wants a textual copy of the code we either made, or just found in the cache. + request.codeLog() + .ifPresent(sb -> sb.append(makeFinalCode(request.className(), request.classBody(), packageName))); + + // If the class we found was indeed the class we were looking for, then complete the future and return it. + resolver.complete(result); + + synchronized (this) { + // Note we are doing something kind of subtle here. We are removing an entry whose key was matched + // by value equality and replacing it with a value-equal but reference-different string that is a + // static member of the class we just loaded. This should be easier on the garbage collector because + // we are replacing a calculated value with a classloaded value and so in effect we are + // "canonicalizing" the string. This is important because these long strings stay in knownClasses + // forever. + knownClasses.remove(identifyingFieldValue); + knownClasses.put(identifyingFieldValue, resolver.getFuture()); + } + + return true; + } + + private Class tryLoadClassByFqName(String fqClassName, Map> parameterClasses) { + try { + return getClassLoaderForFormula(parameterClasses).loadClass(fqClassName); + } catch (ClassNotFoundException cnfe) { + return null; + } + } + + private static String loadIdentifyingField(Class c) { + try { + final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); + return (String) field.get(null); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalStateException("Malformed class in cache", e); + } + } + + private static String makeFinalCode(String className, String classBody, String packageName) { + final String joinedEscapedBody = createEscapedJoinedString(classBody); + classBody = classBody.replaceAll("\\$CLASSNAME\\$", className); + classBody = classBody.substring(0, classBody.lastIndexOf("}")); + classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; + return "package " + packageName + ";\n" + classBody; + } + + /** + * Transform a string into the corresponding Java source code that compiles into that string. This involves escaping + * special characters, surrounding it with quotes, and (if the string is larger than the max string length for Java + * literals), splitting it into substrings and constructing a call to String.join() that combines those substrings. + */ + public static String createEscapedJoinedString(final String originalString) { + return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); + } + + public static String createEscapedJoinedString(final String originalString, int maxStringLength) { + final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); + + // Turn each split into a Java source string by escaping it and surrounding it with " + for (int ii = 0; ii < splits.length; ++ii) { + final String escaped = StringEscapeUtils.escapeJava(splits[ii]); + splits[ii] = "\"" + escaped + "\""; + + } + assert splits.length > 0; + if (splits.length == 1) { + return splits[0]; + } + final String formattedInnards = String.join(",\n", splits); + return "String.join(\"\", " + formattedInnards + ")"; + } + + private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { + final List splits = new ArrayList<>(); + // exclusive end position of the previous substring. + int previousEnd = 0; + // Number of bytes in the "modified UTF-8" representation of the substring we are currently scanning. + int currentByteCount = 0; + for (int ii = 0; ii < originalString.length(); ++ii) { + final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); + if (currentByteCount + bytesConsumed > maxBytes) { + // This character won't fit in this string, so we flush the buffer. + splits.add(originalString.substring(previousEnd, ii)); + previousEnd = ii; + currentByteCount = 0; + } + currentByteCount += bytesConsumed; + } + // At the end of the loop, either + // 1. there are one or more characters that still need to be added to splits + // 2. originalString was empty and so splits is empty and we need to add a single empty string to splits + splits.add(originalString.substring(previousEnd)); + return splits.toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY); + } + + private static int calcBytesConsumed(final char ch) { + if (ch == 0) { + return 2; + } + if (ch <= 0x7f) { + return 1; + } + if (ch <= 0x7ff) { + return 2; + } + return 3; + } + + private static class JavaSourceFromString extends SimpleJavaFileObject { + final String description; + final String code; + final CompletionStageFuture.Resolver> resolver; + + JavaSourceFromString( + final String description, + final String name, + final String code, + final CompletionStageFuture.Resolver> resolver) { + super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); + this.description = description; + this.code = code; + this.resolver = resolver; + } + + public CharSequence getCharContent(boolean ignoreEncodingErrors) { + return code; + } + } + + private static class CompilationRequestAttempt { + final String description; + final String fqClassName; + final String finalCode; + final String packageName; + final String[] splitPackageName; + final QueryCompilerRequest request; + final CompletionStageFuture.Resolver> resolver; + + private CompilationRequestAttempt( + @NotNull final QueryCompilerRequest request, + @NotNull final String packageName, + @NotNull final String fqClassName, + @NotNull final CompletionStageFuture.Resolver> resolver) { + this.description = request.description(); + this.fqClassName = fqClassName; + this.resolver = resolver; + this.packageName = packageName; + this.request = request; + + finalCode = makeFinalCode(request.className(), request.classBody(), packageName); + + if (logEnabled) { + log.info().append("Generating code ").append(finalCode).endl(); + } + + splitPackageName = packageName.split("\\."); + if (splitPackageName.length == 0) { + final Exception err = new UncheckedDeephavenException(String.format( + "packageName %s expected to have at least one .", packageName)); + resolver.completeExceptionally(err); + } + } + + public void ensureDirectories(@NotNull final String rootPath) { + if (splitPackageName.length == 0) { + // we've already failed + return; + } + + final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); + final Path rootPathWithPackage = Paths.get(rootPath, truncatedSplitPackageName); + final File rpf = rootPathWithPackage.toFile(); + QueryCompilerImpl.ensureDirectories(rpf, + () -> "Couldn't create package directories: " + rootPathWithPackage); + } + + public JavaSourceFromString makeSource() { + return new JavaSourceFromString(description, fqClassName, finalCode, resolver); + } + } + + private void maybeCreateClasses( + @NotNull final List requests) { + // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package + // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. + // Also create a temp directory e.g. /tmp/workspace/cache/classes/temporaryCompilationDirectory12345 + // This temp directory will be where the compiler drops files into, e.g. + // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/Formula.class + // Foreshadowing: we will eventually atomically move cm12862183232603186v52_0 from the above to + // /tmp/workspace/cache/classes/io/deephaven/test + // Note: for this atomic move to work, this temp directory must be on the same file system as the destination + // directory. + final String rootPathAsString; + final String tempDirAsString; + try { + rootPathAsString = getClassDestination().getAbsolutePath(); + final Path tempPath = + Files.createTempDirectory(Paths.get(rootPathAsString), "temporaryCompilationDirectory"); + tempDirAsString = tempPath.toFile().getAbsolutePath(); + + for (final CompilationRequestAttempt request : requests) { + request.ensureDirectories(rootPathAsString); + } + } catch (IOException ioe) { + Exception err = new UncheckedIOException(ioe); + for (final CompilationRequestAttempt request : requests) { + request.resolver.completeExceptionally(err); + } + return; + } + + final ExecutionContext executionContext = ExecutionContext.getContext(); + final int parallelismFactor = executionContext.getOperationInitializer().parallelismFactor(); + + final int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); + + final int numTasks; + final JobScheduler jobScheduler; + + final boolean canParallelize = executionContext.getOperationInitializer().canParallelize(); + if (!canParallelize || parallelismFactor == 1 || requestsPerTask >= requests.size()) { + numTasks = 1; + jobScheduler = new ImmediateJobScheduler(); + } else { + numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; + jobScheduler = new OperationInitializerJobScheduler(); + } + + final AtomicBoolean cleanupAlreadyRun = new AtomicBoolean(); + final JavaFileManager fileManager = acquireFileManager(); + final AtomicReference exception = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(1); + final Runnable cleanup = () -> { + if (!cleanupAlreadyRun.compareAndSet(false, true)) { + // onError could be run after cleanup if cleanup throws an exception + return; + } + + try { + try { + FileUtils.deleteRecursively(new File(tempDirAsString)); + } catch (Exception e) { + // ignore errors here + } + try { + releaseFileManager(fileManager); + } catch (Exception e) { + // ignore errors here + } + } finally { + latch.countDown(); + } + }; + + final Consumer onError = err -> { + if (err instanceof RuntimeException) { + exception.set((RuntimeException) err); + } else { + exception.set(new UncheckedDeephavenException("Error during compilation", err)); + } + cleanup.run(); + }; + + jobScheduler.iterateParallel(executionContext, null, JobScheduler.DEFAULT_CONTEXT_FACTORY, + 0, numTasks, (context, jobId, nestedErrorConsumer) -> { + final int startInclusive = jobId * requestsPerTask; + final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); + doCreateClasses( + fileManager, requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive); + }, cleanup, onError); + + try { + latch.await(); + final BasePerformanceEntry perfEntry = jobScheduler.getAccumulatedPerformance(); + if (perfEntry != null) { + QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate(perfEntry); + } + final RuntimeException err = exception.get(); + if (err != null) { + throw err; + } + } catch (final InterruptedException e) { + throw new CancellationException("interrupted while compiling"); + } + } + + private void doCreateClasses( + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, + @NotNull final String rootPathAsString, + @NotNull final String tempDirAsString, + final int startInclusive, + final int endExclusive) { + final List toRetry = new ArrayList<>(); + // If any of our requests fail to compile then the JavaCompiler will not write any class files at all. The + // non-failing requests will be retried in a second pass that is expected to succeed. This enables us to + // fulfill futures independent of each other; otherwise a single failure would taint all requests in a batch. + final boolean wantRetry = doCreateClassesSingleRound(fileManager, requests, rootPathAsString, tempDirAsString, + startInclusive, endExclusive, toRetry); + if (!wantRetry) { + return; + } + + final List ignored = new ArrayList<>(); + if (doCreateClassesSingleRound(fileManager, toRetry, rootPathAsString, tempDirAsString, 0, toRetry.size(), + ignored)) { + // We only retried compilation units that did not fail on the first pass, so we should not have any failures + // on the second pass. + throw new IllegalStateException("Unexpected failure during second pass of compilation"); + } + } + + private boolean doCreateClassesSingleRound( + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, + @NotNull final String rootPathAsString, + @NotNull final String tempDirAsString, + final int startInclusive, + final int endExclusive, + List toRetry) { + final StringWriter compilerOutput = new StringWriter(); + + final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); + final List compilerOptions = Arrays.asList( + "-d", tempDirAsString, + "-cp", classPathAsString, + // this option allows the compiler to attempt to process all source files even if some of them fail + "--should-stop=ifError=GENERATE"); + + final MutableInt numFailures = new MutableInt(0); + compiler.getTask(compilerOutput, + fileManager, + diagnostic -> { + if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { + return; + } + + final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); + final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " + + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); + if (source.resolver.completeExceptionally(err)) { + // only count the first failure for each source + numFailures.increment(); + } + }, + compilerOptions, + null, + requests.subList(startInclusive, endExclusive).stream() + .map(CompilationRequestAttempt::makeSource) + .collect(Collectors.toList())) + .call(); + + final boolean wantRetry = numFailures.intValue() > 0 && numFailures.intValue() != endExclusive - startInclusive; + + // The above has compiled into e.g. + // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various + // class files} + // We want to atomically move it to e.g. + // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} + requests.subList(startInclusive, endExclusive).forEach(request -> { + final Path srcDir = Paths.get(tempDirAsString, request.splitPackageName); + final Path destDir = Paths.get(rootPathAsString, request.splitPackageName); + try { + Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException ioe) { + // The name "isDone" might be misleading here. We haven't called "complete" on the successful + // futures yet, so the only way they would be "done" at this point is if they completed + // exceptionally. + final boolean hasException = request.resolver.getFuture().isDone(); + + if (wantRetry && !Files.exists(srcDir) && !hasException) { + // The move failed, the source directory does not exist, and this compilation unit actually + // succeeded. However, it was not written because some other compilation unit failed. Let's schedule + // this work to try again. + toRetry.add(request); + return; + } + + if (!Files.exists(destDir) && !hasException) { + // Propagate an error here only if the destination does not exist; ignoring issues related to + // collisions with another process. + request.resolver.completeExceptionally(new UncheckedIOException( + "Move failed for some reason other than destination already existing", ioe)); + } + } + }); + + return wantRetry; + } + + /** + * @return the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables + */ + private static String getJavaClassPath() { + String javaClasspath; + { + final StringBuilder javaClasspathBuilder = new StringBuilder(System.getProperty("java.class.path")); + + final String teamCityWorkDir = System.getProperty("teamcity.build.workingDir"); + if (teamCityWorkDir != null) { + // We are running in TeamCity, get the classpath differently + final File[] classDirs = new File(teamCityWorkDir + "/_out_/classes").listFiles(); + if (classDirs != null) { + for (File f : classDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } + } + + final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); + if (testDirs != null) { + for (File f : testDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } + } + + final File[] jars = FileUtils.findAllFiles(new File(teamCityWorkDir + "/lib")); + for (File f : jars) { + if (f.getName().endsWith(".jar")) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } + } + } + javaClasspath = javaClasspathBuilder.toString(); + } + + // IntelliJ will bundle a very large class path into an empty jar with a Manifest that will define the full + // class path. Look for this being used during compile time, so the full class path can be sent into the compile + // call. + final String intellijClassPathJarRegex = ".*classpath[0-9]*\\.jar.*"; + if (javaClasspath.matches(intellijClassPathJarRegex)) { + try { + final Enumeration resources = + QueryCompilerImpl.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); + final Attributes.Name createdByAttribute = new Attributes.Name("Created-By"); + final Attributes.Name classPathAttribute = new Attributes.Name("Class-Path"); + while (resources.hasMoreElements()) { + // Check all manifests -- looking for the Intellij created one + final Manifest manifest = new Manifest(resources.nextElement().openStream()); + final Attributes attributes = manifest.getMainAttributes(); + final Object createdBy = attributes.get(createdByAttribute); + if ("IntelliJ IDEA".equals(createdBy)) { + final String extendedClassPath = (String) attributes.get(classPathAttribute); + if (extendedClassPath != null) { + // Parses the files in the manifest description an changes their format to drop the "file:/" + // and use the default path separator + final String filePaths = Stream.of(extendedClassPath.split("file:/")) + .map(String::trim) + .filter(fileName -> !fileName.isEmpty()) + .collect(Collectors.joining(File.pathSeparator)); + + // Remove the classpath jar in question, and expand it with the files from the manifest + javaClasspath = Stream.of(javaClasspath.split(File.pathSeparator)) + .map(cp -> cp.matches(intellijClassPathJarRegex) ? filePaths : cp) + .collect(Collectors.joining(File.pathSeparator)); + } + } + } + } catch (IOException e) { + throw new UncheckedIOException("Error extract manifest file from " + javaClasspath + ".\n", e); + } + } + return javaClasspath; + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java index 1645a88ab3a..b2328798e78 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java @@ -46,16 +46,18 @@ public DeferredViewTable(@NotNull final TableDefinition definition, this.deferredViewColumns = deferredViewColumns == null ? SelectColumn.ZERO_LENGTH_SELECT_COLUMN_ARRAY : deferredViewColumns; final TableDefinition parentDefinition = tableReference.getDefinition(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); SelectAndViewAnalyzer.initializeSelectColumns( - parentDefinition.getColumnNameMap(), this.deferredViewColumns); + parentDefinition.getColumnNameMap(), this.deferredViewColumns, compilationProcessor); this.deferredFilters = deferredFilters == null ? WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY : deferredFilters; for (final WhereFilter sf : this.deferredFilters) { - sf.init(parentDefinition); + sf.init(parentDefinition, compilationProcessor); if (sf instanceof LivenessReferent && sf.isRefreshing()) { manage((LivenessReferent) sf); setRefreshing(true); } } + compilationProcessor.compile(); // we really only expect one of these things to be set! final boolean haveDrop = this.deferredDropColumns.length > 0; @@ -78,9 +80,11 @@ public DeferredViewTable(@NotNull final TableDefinition definition, @Override public Table where(Filter filter) { final WhereFilter[] whereFilters = WhereFilter.fromInternal(filter); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (WhereFilter f : whereFilters) { - f.init(definition); + f.init(definition, compilationProcessor); } + compilationProcessor.compile(); return getResultTableWithWhere(whereFilters); } @@ -189,8 +193,9 @@ private PreAndPostFilters applyFilterRenamings(WhereFilter[] filters) { } } + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (final WhereFilter filter : filters) { - filter.init(definition); + filter.init(definition, compilationProcessor); final boolean isPostView = Stream.of(filter.getColumns(), filter.getColumnArrays()) .flatMap(Collection::stream) @@ -220,6 +225,7 @@ private PreAndPostFilters applyFilterRenamings(WhereFilter[] filters) { postViewFilters.add(filter); } } + compilationProcessor.compile(); return new PreAndPostFilters(preViewFilters.toArray(WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY), postViewFilters.toArray(WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY)); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java index 8409f97616c..0dd5e9e1627 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java @@ -7,6 +7,7 @@ import io.deephaven.api.filter.Filter; import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.updategraph.UpdateSourceRegistrar; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.util.TableTools; @@ -131,16 +132,20 @@ protected TableAndRemainingFilters getWithWhere(WhereFilter... whereFilters) { @Override public Table selectDistinctInternal(Collection columns) { final List selectColumns = Arrays.asList(SelectColumn.from(columns)); + try { + SelectAndViewAnalyzer.initializeSelectColumns(table.getDefinition().getColumnNameMap(), + selectColumns.toArray(SelectColumn[]::new)); + } catch (Exception e) { + return null; + } + + final Set partitioningDerivedColumnNames = new HashSet<>(); for (final SelectColumn selectColumn : selectColumns) { - try { - selectColumn.initDef(getDefinition().getColumnNameMap()); - } catch (Exception e) { - return null; - } if (!((PartitionAwareSourceTable) table).isValidAgainstColumnPartitionTable( - selectColumn.getColumns(), selectColumn.getColumnArrays())) { + selectColumn.getColumns(), selectColumn.getColumnArrays(), partitioningDerivedColumnNames)) { return null; } + partitioningDerivedColumnNames.add(selectColumn.getName()); } return table.selectDistinct(selectColumns); } @@ -253,10 +258,11 @@ private Table whereImpl(final WhereFilter[] whereFilters) { return prepareReturnThis(); } + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); final List partitionFilters = new ArrayList<>(); final List deferredFilters = new ArrayList<>(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition); + whereFilter.init(definition, compilationProcessor); if (!(whereFilter instanceof ReindexingFilter) && isValidAgainstColumnPartitionTable(whereFilter.getColumns(), whereFilter.getColumnArrays())) { partitionFilters.add(whereFilter); @@ -264,6 +270,7 @@ && isValidAgainstColumnPartitionTable(whereFilter.getColumns(), whereFilter.getC deferredFilters.add(whereFilter); } } + compilationProcessor.compile(); final PartitionAwareSourceTable withPartitionsFiltered = partitionFilters.isEmpty() ? this @@ -280,13 +287,18 @@ && isValidAgainstColumnPartitionTable(whereFilter.getColumns(), whereFilter.getC @Override public final Table selectDistinct(@NotNull final Collection columns) { final List selectColumns = Arrays.asList(SelectColumn.from(columns)); + SelectAndViewAnalyzer.initializeSelectColumns( + definition.getColumnNameMap(), selectColumns.toArray(SelectColumn[]::new)); + + final Set partitioningDerivedColumnNames = new HashSet<>(); for (final SelectColumn selectColumn : selectColumns) { - selectColumn.initDef(definition.getColumnNameMap()); - if (!isValidAgainstColumnPartitionTable(selectColumn.getColumns(), selectColumn.getColumnArrays())) { + if (!isValidAgainstColumnPartitionTable( + selectColumn.getColumns(), selectColumn.getColumnArrays(), partitioningDerivedColumnNames)) { // Be sure to invoke the super-class version of this method, rather than the array-based one that // delegates to this method. return super.selectDistinct(selectColumns); } + partitioningDerivedColumnNames.add(selectColumn.getName()); } // Ensure that the location table is available and populated with non-null, non-empty locations. @@ -299,9 +311,18 @@ public final Table selectDistinct(@NotNull final Collection columnNames, @NotNull final Collection columnArrayNames) { + return isValidAgainstColumnPartitionTable(columnNames, columnArrayNames, Collections.emptySet()); + } + + private boolean isValidAgainstColumnPartitionTable( + @NotNull final Collection columnNames, + @NotNull final Collection columnArrayNames, + @NotNull final Collection partitioningDerivedColumnNames) { if (!columnArrayNames.isEmpty()) { return false; } - return columnNames.stream().allMatch(partitioningColumnDefinitions::containsKey); + return columnNames.stream().allMatch( + columnName -> partitioningColumnDefinitions.containsKey(columnName) + || partitioningDerivedColumnNames.contains(columnName)); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java new file mode 100644 index 00000000000..ae321271be4 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java @@ -0,0 +1,178 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.table.impl; + +import io.deephaven.UncheckedDeephavenException; +import io.deephaven.api.util.NameValidator; +import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.context.QueryScope; +import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; +import io.deephaven.util.MultiException; +import io.deephaven.util.SafeCloseable; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.datastructures.CachingSupplier; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.VisibleForTesting; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public interface QueryCompilerRequestProcessor { + + /** + * @return An immediate QueryCompilerRequestProcessor + */ + static QueryCompilerRequestProcessor.ImmediateProcessor immediate() { + return new ImmediateProcessor(); + } + + /** + * @return A batch QueryCompilerRequestProcessor + */ + static QueryCompilerRequestProcessor.BatchProcessor batch() { + return new BatchProcessor(); + } + + /** + * @return a CachingSupplier that supplies a snapshot of the current query scope variables + */ + @VisibleForTesting + static CachingSupplier> newQueryScopeVariableSupplier() { + final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); + return new CachingSupplier<>(() -> Collections.unmodifiableMap( + queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name)))); + } + + /** + * @return a lazily cached snapshot of the current query scope variables + */ + Map getQueryScopeVariables(); + + /** + * Submit a request for compilation. The QueryCompilerRequestProcessor is not required to immediately compile this + * request. + * + * @param request the request to compile + */ + CompletionStageFuture> submit(@NotNull QueryCompilerRequest request); + + /** + * A QueryCompilerRequestProcessor that immediately compiles requests. + */ + class ImmediateProcessor implements QueryCompilerRequestProcessor { + + private final CachingSupplier> queryScopeVariableSupplier = newQueryScopeVariableSupplier(); + + private ImmediateProcessor() { + // force use of static factory method + } + + @Override + public Map getQueryScopeVariables() { + return queryScopeVariableSupplier.get(); + } + + @Override + public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { + final String desc = "Compile: " + request.description(); + final CompletionStageFuture.Resolver> resolver = CompletionStageFuture.make(); + try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(desc)) { + ExecutionContext.getContext().getQueryCompiler().compile(request, resolver); + } + + // The earlier we validate the future, the better. + final CompletionStageFuture> future = resolver.getFuture(); + try { + future.get(0, TimeUnit.SECONDS); + } catch (ExecutionException err) { + throw new UncheckedDeephavenException("Compilation failed", err.getCause()); + } catch (InterruptedException | TimeoutException err) { + // This should never happen since the future is already completed. + throw new UncheckedDeephavenException("Caught unexpected exception", err); + } + + return future; + } + } + + /** + * A QueryCompilerRequestProcessor that batches requests and compiles them all at once. + *

+ * The compile method must be called to actually compile the requests. + */ + class BatchProcessor implements QueryCompilerRequestProcessor { + private final List requests = new ArrayList<>(); + private final List>> resolvers = new ArrayList<>(); + private final CachingSupplier> queryScopeVariableSupplier = newQueryScopeVariableSupplier(); + + private BatchProcessor() { + // force use of static factory method + } + + @Override + public Map getQueryScopeVariables() { + return queryScopeVariableSupplier.get(); + } + + @Override + public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { + final CompletionStageFuture.Resolver> resolver = CompletionStageFuture.make(); + requests.add(request); + resolvers.add(resolver); + return resolver.getFuture(); + } + + /** + * Compile all the requests that have been submitted. + */ + public void compile() { + if (requests.isEmpty()) { + return; + } + + final String desc; + if (requests.size() == 1) { + desc = "Compile: " + requests.get(0).description(); + } else { + final StringBuilder descriptionBuilder = new StringBuilder(); + descriptionBuilder.append("Batch Compile of ").append(requests.size()).append(" requests:\n"); + for (final QueryCompilerRequest request : requests) { + descriptionBuilder.append('\t').append(request.description()).append('\n'); + } + desc = descriptionBuilder.toString(); + } + + try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(desc)) { + final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); + // noinspection unchecked + compiler.compile( + requests.toArray(QueryCompilerRequest[]::new), + resolvers.toArray(CompletionStageFuture.Resolver[]::new)); + + final List exceptions = new ArrayList<>(); + for (CompletionStageFuture.Resolver> resolver : resolvers) { + try { + Object ignored2 = resolver.getFuture().get(); + } catch (ExecutionException err) { + exceptions.add(err.getCause()); + } catch (InterruptedException err) { + exceptions.add(err); + } + } + if (!exceptions.isEmpty()) { + throw new UncheckedDeephavenException("Compilation failed", + MultiException.maybeWrapInMultiException("Compilation exceptions for multiple requests", + exceptions)); + } + } + } + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java index 14ce8368704..139f58d196b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java @@ -1167,13 +1167,18 @@ private void initializeAndPrioritizeFilters(@NotNull final WhereFilter... filter final DataIndexer dataIndexer = DataIndexer.existingOf(rowSet); final int numFilters = filters.length; final BitSet priorityFilterIndexes = new BitSet(numFilters); + + final QueryCompilerRequestProcessor.BatchProcessor compilationProcesor = QueryCompilerRequestProcessor.batch(); + // Initialize our filters immediately so we can examine the columns they use. Note that filter + // initialization is safe to invoke repeatedly. + for (final WhereFilter filter : filters) { + filter.init(getDefinition(), compilationProcesor); + } + compilationProcesor.compile(); + for (int fi = 0; fi < numFilters; ++fi) { final WhereFilter filter = filters[fi]; - // Initialize our filters immediately so we can examine the columns they use. Note that filter - // initialization is safe to invoke repeatedly. - filter.init(getDefinition()); - // Simple filters against indexed columns get priority if (dataIndexer != null && !(filter instanceof ReindexingFilter) @@ -1246,22 +1251,20 @@ private QueryTable whereInternal(final WhereFilter... filters) { return result; } - { - final List whereFilters = new LinkedList<>(); - final List>>> shiftColPairs = - new LinkedList<>(); - for (final WhereFilter filter : filters) { - if (filter instanceof AbstractConditionFilter - && ((AbstractConditionFilter) filter).hasConstantArrayAccess()) { - shiftColPairs.add(((AbstractConditionFilter) filter).getFormulaShiftColPair()); - } else { - whereFilters.add(filter); - } - } - if (!shiftColPairs.isEmpty()) { - return (QueryTable) ShiftedColumnsFactory.where(this, shiftColPairs, whereFilters); + final List whereFilters = new LinkedList<>(); + final List>>> shiftColPairs = + new LinkedList<>(); + for (final WhereFilter filter : filters) { + if (filter instanceof AbstractConditionFilter + && ((AbstractConditionFilter) filter).hasConstantArrayAccess()) { + shiftColPairs.add(((AbstractConditionFilter) filter).getFormulaShiftColPair()); + } else { + whereFilters.add(filter); } } + if (!shiftColPairs.isEmpty()) { + return (QueryTable) ShiftedColumnsFactory.where(this, shiftColPairs, whereFilters); + } return memoizeResult(MemoizedOperationKey.filter(filters), () -> { try (final SafeCloseable ignored = Arrays.stream(filters) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java index 24c36fb5733..fe479e6ea17 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java @@ -49,8 +49,11 @@ private Table viewInternal(Collection selectables, boolean final Map> resultColumnsExternal = new LinkedHashMap<>(); final Map> allColumns = new HashMap<>(definition.getColumnNameMap()); boolean simpleRetain = true; + + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (final SelectColumn selectColumn : columns) { - final List usedColumnNames = new ArrayList<>(selectColumn.initDef(allColumns)); + final List usedColumnNames = new ArrayList<>( + selectColumn.initDef(allColumns, compilationProcessor)); usedColumnNames.addAll(selectColumn.getColumnArrays()); resultColumnsInternal.addAll(usedColumnNames.stream() .filter(usedColumnName -> !resultColumnsExternal.containsKey(usedColumnName)) @@ -65,6 +68,7 @@ private Table viewInternal(Collection selectables, boolean resultColumnsExternal.put(selectColumn.getName(), columnDef); allColumns.put(selectColumn.getName(), columnDef); } + compilationProcessor.compile(); TableDefinition newDefExternal = TableDefinition.of( resultColumnsExternal.values().toArray(ColumnDefinition.ZERO_LENGTH_COLUMN_DEFINITION_ARRAY)); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java index 13c3c2d7bad..05ef6d8acd2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java @@ -93,9 +93,12 @@ public String getLogPrefix() { @Override public SafeCloseable beginOperation(@NotNull final QueryTable parent) { + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); + Arrays.stream(whereFilters).forEach(filter -> filter.init(parent.getDefinition(), compilationProcessor)); + compilationProcessor.compile(); + return Arrays.stream(whereFilters) .map((final WhereFilter filter) -> { - filter.init(parent.getDefinition()); // Ensure we gather the correct dependencies when building a snapshot control. return filter.beginOperation(parent); }).collect(SafeCloseableList.COLLECTOR); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java index ad7dfa44d3f..7bf2a38008f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java @@ -38,11 +38,8 @@ import io.deephaven.engine.table.ChunkSource; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.impl.BaseTable; -import io.deephaven.engine.table.impl.QueryTable; -import io.deephaven.engine.table.impl.TupleSourceFactory; import io.deephaven.engine.table.impl.by.rollup.NullColumns; import io.deephaven.engine.table.impl.by.rollup.RollupAggregation; import io.deephaven.engine.table.impl.by.rollup.RollupAggregationOutputs; @@ -338,7 +335,7 @@ private Converter( isBlink = this.table.isBlink(); } - final AggregationContext build() { + AggregationContext build() { walkAllAggregations(); transformers.add(new RowLookupAttributeSetter()); return makeAggregationContext(); @@ -664,12 +661,21 @@ final void addWeightedAvgOrSumOperator( * {@link AggregationContext} for standard aggregations. Accumulates state by visiting each aggregation. */ private final class NormalConverter extends Converter { + private final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor; private NormalConverter( @NotNull final Table table, final boolean requireStateChangeRecorder, @NotNull final String... groupByColumnNames) { super(table, requireStateChangeRecorder, groupByColumnNames); + this.compilationProcessor = QueryCompilerRequestProcessor.batch(); + } + + @Override + AggregationContext build() { + final AggregationContext resultContext = super.build(); + compilationProcessor.compile(); + return resultContext; } // ------------------------------------------------------------------------------------------------------------- @@ -744,7 +750,8 @@ public void visit(@NotNull final AggSpecFormula formula) { final GroupByChunkedOperator groupByChunkedOperator = new GroupByChunkedOperator(table, false, null, resultPairs.stream().map(pair -> MatchPair.of((Pair) pair.input())).toArray(MatchPair[]::new)); final FormulaChunkedOperator formulaChunkedOperator = new FormulaChunkedOperator(groupByChunkedOperator, - true, formula.formula(), formula.paramToken(), MatchPair.fromPairs(resultPairs)); + true, formula.formula(), formula.paramToken(), compilationProcessor, + MatchPair.fromPairs(resultPairs)); addNoInputOperator(formulaChunkedOperator); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java index 3bf31cb5e8b..bdf69ed3a97 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.*; import io.deephaven.engine.table.ChunkSource.GetContext; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaUtil; import io.deephaven.engine.liveness.LivenessReferent; import io.deephaven.engine.table.ModifiedColumnSet; @@ -69,10 +70,12 @@ class FormulaChunkedOperator implements IterativeChunkedAggregationOperator { * @param columnParamName The token to substitute column names for * @param resultColumnPairs The names for formula input and result columns */ - FormulaChunkedOperator(@NotNull final GroupByChunkedOperator groupBy, + FormulaChunkedOperator( + @NotNull final GroupByChunkedOperator groupBy, final boolean delegateToBy, @NotNull final String formula, @NotNull final String columnParamName, + @NotNull final QueryCompilerRequestProcessor compilationProcessor, @NotNull final MatchPair... resultColumnPairs) { this.groupBy = groupBy; this.delegateToBy = delegateToBy; @@ -95,9 +98,10 @@ class FormulaChunkedOperator implements IterativeChunkedAggregationOperator { final ColumnDefinition inputColumnDefinition = ColumnDefinition .fromGenericType(inputColumnName, inputColumnSource.getType(), inputColumnSource.getComponentType()); - formulaColumn.initDef(Collections.singletonMap(inputColumnName, inputColumnDefinition)); - // noinspection unchecked - resultColumns[ci] = ArrayBackedColumnSource.getMemoryColumnSource(0, formulaColumn.getReturnedType()); + formulaColumn.initDef(Collections.singletonMap(inputColumnName, inputColumnDefinition), + compilationProcessor); + resultColumns[ci] = ArrayBackedColumnSource.getMemoryColumnSource( + 0, formulaColumn.getReturnedType(), formulaColumn.getReturnedComponentType()); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java index 6ff7f0460fe..bc007401aec 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java @@ -11,6 +11,7 @@ import io.deephaven.chunk.util.hashing.CharChunkHasher; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetBuilderRandom; @@ -19,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.impl.MultiJoinModifiedSlotTracker; import io.deephaven.engine.table.impl.NaturalJoinModifiedSlotTracker; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.asofjoin.RightIncrementalAsOfJoinStateManagerTypedBase; import io.deephaven.engine.table.impl.asofjoin.StaticAsOfJoinStateManagerTypedBase; import io.deephaven.engine.table.impl.asofjoin.TypedAsOfJoinFactory; @@ -566,12 +568,18 @@ public static T make(HasherConfig hasherConfig, ColumnSource[] tableKe final String javaString = Arrays.stream(javaStrings).filter(s -> !s.startsWith("package ")).collect(Collectors.joining("\n")); - final Class clazz = ExecutionContext.getContext().getQueryCompiler().compile(className, javaString, - "io.deephaven.engine.table.impl.by.typed." + hasherConfig.packageMiddle + ".gen"); + final Class clazz = ExecutionContext.getContext().getQueryCompiler().compile(QueryCompilerRequest.builder() + .description("TypedHasherFactory: " + className) + .className(className) + .classBody(javaString) + .packageNameRoot("io.deephaven.engine.table.impl.by.typed." + hasherConfig.packageMiddle + ".gen") + .build()); + if (!hasherConfig.baseClass.isAssignableFrom(clazz)) { throw new IllegalStateException("Generated class is not a " + hasherConfig.baseClass.getCanonicalName()); } + // noinspection unchecked final Class castedClass = (Class) clazz; T retVal; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java index 9b943837bbe..8602715b72c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java @@ -12,10 +12,7 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.hierarchical.RollupTable; import io.deephaven.engine.table.hierarchical.TreeTable; -import io.deephaven.engine.table.impl.AbsoluteSortColumnConventions; -import io.deephaven.engine.table.impl.NoSuchColumnException; -import io.deephaven.engine.table.impl.QueryTable; -import io.deephaven.engine.table.impl.TableAdapter; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; @@ -284,16 +281,24 @@ private Stream absoluteSelectColumns() { // custom columns in the future. For now, we've plumbed absolute column value sorting via naming // conventions. Note that we simply avoid telling the client about these columns when sending schemas, so we // have no need to drop them post-sort. - return sortColumns.stream() + + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + QueryCompilerRequestProcessor.batch(); + + final SelectColumn[] columns = sortColumns.stream() .map(sc -> sc.column().name()) .filter(AbsoluteSortColumnConventions::isAbsoluteColumnName) .map(cn -> { final String baseColumnName = AbsoluteSortColumnConventions.absoluteColumnNameToBaseName(cn); final Selectable selectable = AbsoluteSortColumnConventions.makeSelectable(cn, baseColumnName); final SelectColumn selectColumn = SelectColumn.of(selectable); - selectColumn.initDef(Map.of(baseColumnName, getDefinition().getColumn(baseColumnName))); + selectColumn.initDef(Map.of(baseColumnName, getDefinition().getColumn(baseColumnName)), + compilationProcessor); return selectColumn; - }); + }).toArray(SelectColumn[]::new); + + compilationProcessor.compile(); + return Stream.of(columns); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java index 03c7d193274..26a77957a08 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java @@ -19,6 +19,7 @@ import io.deephaven.engine.table.hierarchical.RollupTable; import io.deephaven.engine.table.impl.BaseTable.CopyAttributeOperation; import io.deephaven.engine.table.impl.NotificationStepSource; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.SortOperation; import io.deephaven.engine.table.impl.by.AggregationProcessor; @@ -275,8 +276,9 @@ public static WhereFilter[] initializeAndValidateFilters( @NotNull final Collection filters, @NotNull final Function exceptionFactory) { final WhereFilter[] whereFilters = WhereFilter.from(filters); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (final WhereFilter whereFilter : whereFilters) { - whereFilter.init(source.getDefinition()); + whereFilter.init(source.getDefinition(), compilationProcessor); final List invalidColumnsUsed = whereFilter.getColumns().stream().map(ColumnName::of) .filter(cn -> !groupByColumns.contains(cn)).map(ColumnName::name).collect(Collectors.toList()); if (!invalidColumnsUsed.isEmpty()) { @@ -290,6 +292,8 @@ public static WhereFilter[] initializeAndValidateFilters( + " may not use column arrays, but uses column arrays from " + whereFilter.getColumnArrays()); } } + compilationProcessor.compile(); + return whereFilters; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java index 9fdfc15138a..a40d60d69c5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java @@ -9,12 +9,12 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.hierarchical.TreeTable; import io.deephaven.engine.table.hierarchical.TreeTable.NodeOperationsRecorder; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.WhereFilter; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -137,7 +137,14 @@ public Table where(Filter filter) { } private Stream whereFilters() { - return Stream.of(WhereFilter.fromInternal(filter)).peek(wf -> wf.init(getDefinition())); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + QueryCompilerRequestProcessor.batch(); + final WhereFilter[] filters = WhereFilter.fromInternal(filter); + for (final WhereFilter filter : filters) { + filter.init(getDefinition(), compilationProcessor); + } + compilationProcessor.compile(); + return Stream.of(filters); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java index e40cd841662..53f5eaadb85 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java @@ -151,7 +151,10 @@ private TreeTableFilter(@NotNull final TreeTableImpl tree, @NotNull final WhereF parentIdColumnName = tree.getParentIdentifierColumn(); sourceRowLookup = tree.getSourceRowLookup(); this.filters = filters; - Arrays.stream(filters).forEach((final WhereFilter filter) -> filter.init(source.getDefinition())); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); + Arrays.stream(filters) + .forEach((final WhereFilter filter) -> filter.init(source.getDefinition(), compilationProcessor)); + compilationProcessor.compile(); idSource = source.getColumnSource(tree.getIdentifierColumn().name()); parentIdSource = source.getColumnSource(tree.getParentIdentifierColumn().name()); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java index dfb0130594c..680b20c07a6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java @@ -106,7 +106,7 @@ private static Table makeNullSingleColumnTable( @NotNull final Table source, @NotNull final ColumnName column, final long size) { - final ColumnDefinition columnDefinition = source.getDefinition().getColumn(column.name()); + final ColumnDefinition columnDefinition = source.getDefinition().getColumn(column.name()); final TableDefinition columnOnlyTableDefinition = TableDefinition.of(columnDefinition); // noinspection resource return new QueryTable(columnOnlyTableDefinition, RowSetFactory.flat(size).toTracking(), @@ -154,13 +154,16 @@ public TreeTable withFilter(@NotNull Filter filter) { if (whereFilters.length == 0) { return noopResult(); } + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); final Map> nodeSuitabilityToFilters = Stream.of(whereFilters) - .peek(wf -> wf.init(source.getDefinition())) + .peek(wf -> wf.init(source.getDefinition(), compilationProcessor)) .collect(Collectors.partitioningBy(wf -> { // Node-level filters have only node-filter columns and use no column arrays return wf.getColumns().stream().map(ColumnName::of).allMatch(nodeFilterColumns::contains) && wf.getColumnArrays().isEmpty(); })); + compilationProcessor.compile(); + final List nodeFilters = nodeSuitabilityToFilters.get(true); final List sourceFilters = nodeSuitabilityToFilters.get(false); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java index ae037b986d6..36b10f914fa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java @@ -94,6 +94,7 @@ import io.deephaven.engine.util.PyCallableWrapperJpyImpl; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; +import io.deephaven.time.TimeLiteralReplacedExpression; import io.deephaven.util.annotations.TestUseOnly; import io.deephaven.util.type.TypeUtils; import io.deephaven.vector.ByteVector; @@ -109,6 +110,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.VisibleForTesting; import org.jpy.PyObject; import java.lang.reflect.Array; @@ -188,32 +190,7 @@ public final class QueryLanguageParser extends GenericVisitorAdapter, Q * imported. * @param variables A map of the names of scope variables to their types * @param variableTypeArguments A map of the names of scope variables to their type arguments - * @throws QueryLanguageParseException If any exception or error is encountered - */ - @TestUseOnly - QueryLanguageParser( - String expression, - Collection packageImports, - Collection> classImports, - Collection> staticImports, - Map> variables, - Map[]> variableTypeArguments) throws QueryLanguageParseException { - this(expression, packageImports, classImports, staticImports, variables, - variableTypeArguments, null, null, true); - } - - /** - * Create a QueryLanguageParser and parse the given {@code expression}. After construction, the - * {@link QueryLanguageParser.Result result} of parsing the {@code expression} is available with the - * {@link #getResult()}} method. - * - * @param expression The query language expression to parse - * @param packageImports Wildcard package imports - * @param classImports Individual class imports - * @param staticImports Wildcard static imports. All static variables and methods for the given classes are - * imported. - * @param variables A map of the names of scope variables to their types - * @param variableTypeArguments A map of the names of scope variables to their type arguments + * @param unboxArguments If true it will unbox the query scope arguments * @param queryScopeVariables A mutable map of the names of query scope variables to their values * @param columnVariables A set of column variable names * @throws QueryLanguageParseException If any exception or error is encountered @@ -226,9 +203,22 @@ public QueryLanguageParser( Map> variables, Map[]> variableTypeArguments, @Nullable Map queryScopeVariables, - @Nullable Set columnVariables) throws QueryLanguageParseException { - this(expression, packageImports, classImports, staticImports, variables, - variableTypeArguments, queryScopeVariables, columnVariables, true); + @Nullable Set columnVariables, + boolean unboxArguments, + @Nullable TimeLiteralReplacedExpression timeConversionResult) throws QueryLanguageParseException { + this( + expression, + packageImports, + classImports, + staticImports, + variables, + variableTypeArguments, + queryScopeVariables, + columnVariables, + unboxArguments, + false, + PyCallableWrapperJpyImpl.class.getName(), + timeConversionResult); } /** @@ -243,36 +233,21 @@ public QueryLanguageParser( * imported. * @param variables A map of the names of scope variables to their types * @param variableTypeArguments A map of the names of scope variables to their type arguments - * @param unboxArguments If true it will unbox the query scope arguments - * @param queryScopeVariables A mutable map of the names of query scope variables to their values - * @param columnVariables A set of column variable names * @throws QueryLanguageParseException If any exception or error is encountered */ - public QueryLanguageParser( + @TestUseOnly + QueryLanguageParser( String expression, Collection packageImports, Collection> classImports, Collection> staticImports, Map> variables, - Map[]> variableTypeArguments, - @Nullable Map queryScopeVariables, - @Nullable Set columnVariables, - boolean unboxArguments) - throws QueryLanguageParseException { - this( - expression, - packageImports, - classImports, - staticImports, - variables, - variableTypeArguments, - queryScopeVariables, - columnVariables, - unboxArguments, - false, - PyCallableWrapperJpyImpl.class.getName()); + Map[]> variableTypeArguments) throws QueryLanguageParseException { + this(expression, packageImports, classImports, staticImports, variables, + variableTypeArguments, null, null, true, null); } + @VisibleForTesting QueryLanguageParser( String expression, final Collection packageImports, @@ -284,7 +259,8 @@ public QueryLanguageParser( @Nullable final Set columnVariables, final boolean unboxArguments, final boolean verifyIdempotence, - @NotNull final String pyCallableWrapperImplName) throws QueryLanguageParseException { + @NotNull final String pyCallableWrapperImplName, + @Nullable final TimeLiteralReplacedExpression timeConversionResult) throws QueryLanguageParseException { this.packageImports = packageImports == null ? Collections.emptySet() : Set.copyOf(packageImports); this.classImports = classImports == null ? Collections.emptySet() : Set.copyOf(classImports); this.staticImports = staticImports == null ? Collections.emptySet() : Set.copyOf(staticImports); @@ -346,7 +322,7 @@ public QueryLanguageParser( final QueryLanguageParser validationQueryLanguageParser = new QueryLanguageParser( printedSource, packageImports, classImports, staticImports, variables, variableTypeArguments, queryScopeVariables, columnVariables, false, false, - pyCallableWrapperImplName); + pyCallableWrapperImplName, timeConversionResult); final String reparsedSource = validationQueryLanguageParser.result.source; Assert.equals( @@ -364,7 +340,7 @@ public QueryLanguageParser( } result = new Result(type, printer.builder.toString(), variablesUsed, this.queryScopeVariables, - isConstantValueExpression, formulaShiftColPair); + isConstantValueExpression, formulaShiftColPair, timeConversionResult); } catch (Throwable e) { // need to catch it and make a new one because it contains unserializable variables... final StringBuilder exceptionMessageBuilder = new StringBuilder(1024) @@ -3245,6 +3221,7 @@ public static class Result { private final Map possibleParams; private final boolean isConstantValueExpression; private final Pair>> formulaShiftColPair; + private final TimeLiteralReplacedExpression timeConversionResult; Result( Class type, @@ -3252,13 +3229,15 @@ public static class Result { HashSet variablesUsed, Map possibleParams, boolean isConstantValueExpression, - Pair>> formulaShiftColPair) { + Pair>> formulaShiftColPair, + TimeLiteralReplacedExpression timeConversionResult) { this.type = Objects.requireNonNull(type, "type"); this.source = source; this.variablesUsed = variablesUsed; this.possibleParams = possibleParams; this.isConstantValueExpression = isConstantValueExpression; this.formulaShiftColPair = formulaShiftColPair; + this.timeConversionResult = timeConversionResult; } public Class getType() { @@ -3284,6 +3263,10 @@ public Map getPossibleParams() { public Pair>> getFormulaShiftColPair() { return formulaShiftColPair; } + + public TimeLiteralReplacedExpression getTimeConversionResult() { + return timeConversionResult; + } } public static class VisitArgs { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java index 507f27a88fd..22c1fc2e8d5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java @@ -28,6 +28,11 @@ public final Class getReturnedType() { return Table.class; } + @Override + public Class getReturnedComponentType() { + return null; + } + @Override public final List getColumnArrays() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java index 453be979a3d..449488af308 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java @@ -74,6 +74,12 @@ public final Class getReturnedType() { return long.class; } + @Override + public Class getReturnedComponentType() { + // long does not have a component type + return null; + } + @Override public final List getColumnArrays() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java index 95751fba30e..e93342b475a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java @@ -19,10 +19,7 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.*; -import io.deephaven.engine.table.impl.BaseTable; -import io.deephaven.engine.table.impl.MatchPair; -import io.deephaven.engine.table.impl.MemoizedOperationKey; -import io.deephaven.engine.table.impl.QueryTable; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.engine.table.impl.select.MatchFilter; import io.deephaven.engine.table.impl.select.WhereFilter; @@ -235,10 +232,12 @@ private Map computeSharedAttributes(@NotNull final Iterator filters) { final WhereFilter[] whereFilters = WhereFilter.from(filters); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); final boolean invalidFilter = Arrays.stream(whereFilters).flatMap((final WhereFilter filter) -> { - filter.init(table.getDefinition()); + filter.init(table.getDefinition(), compilationProcessor); return Stream.concat(filter.getColumns().stream(), filter.getColumnArrays().stream()); }).anyMatch((final String columnName) -> columnName.equals(constituentColumnName)); + compilationProcessor.compile(); if (invalidFilter) { throw new IllegalArgumentException("Unsupported filter against constituent column " + constituentColumnName + " found in filters: " + filters); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java index 1ae21286f1b..ef7644f1fbb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java @@ -464,9 +464,11 @@ public PartitionedTable.Proxy sort(Collection columnsToSortBy) { public PartitionedTable.Proxy where(Filter filter) { final WhereFilter[] whereFilters = WhereFilter.fromInternal(filter); final TableDefinition definition = target.constituentDefinition(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition); + whereFilter.init(definition, compilationProcessor); } + compilationProcessor.compile(); return basicTransform(ct -> ct.where(Filter.and(WhereFilter.copyFrom(whereFilters)))); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java index 481d25065ee..e96bb529f1e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java @@ -3,35 +3,29 @@ // package io.deephaven.engine.table.impl.select; -import io.deephaven.api.util.NameValidator; import io.deephaven.base.Pair; -import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.WritableRowSet; -import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; +import io.deephaven.engine.table.impl.select.codegen.FormulaAnalyzer; import io.deephaven.engine.table.impl.select.python.ArgumentsChunked; import io.deephaven.engine.table.impl.select.python.DeephavenCompatibleFunction; import io.deephaven.engine.util.PyCallableWrapperJpyImpl; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.vector.ObjectVector; import org.jetbrains.annotations.NotNull; import org.jpy.PyObject; import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; import java.net.MalformedURLException; import java.util.*; -import java.util.function.BiConsumer; import java.util.stream.Collectors; import static io.deephaven.engine.table.impl.select.DhFormulaColumn.COLUMN_SUFFIX; @@ -80,89 +74,23 @@ public List getColumnArrays() { } @Override - public synchronized void init(TableDefinition tableDefinition) { + public void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, QueryCompilerRequestProcessor.immediate()); + } + + @Override + public synchronized void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (initialized) { return; } - final Map> possibleVariables = new HashMap<>(); - possibleVariables.put("i", int.class); - possibleVariables.put("ii", long.class); - possibleVariables.put("k", long.class); - - final Map[]> possibleVariableParameterizedTypes = new HashMap<>(); - try { - final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); - final Map queryScopeVariables = queryScope.toMap( - (name, value) -> NameValidator.isValidQueryParameterName(name)); - for (Map.Entry param : queryScopeVariables.entrySet()) { - possibleVariables.put(param.getKey(), QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); - Type declaredType = QueryScopeParamTypeUtil.getDeclaredType(param.getValue()); - if (declaredType instanceof ParameterizedType) { - ParameterizedType pt = (ParameterizedType) declaredType; - Class[] paramTypes = Arrays.stream(pt.getActualTypeArguments()) - .map(QueryScopeParamTypeUtil::classFromType) - .toArray(Class[]::new); - possibleVariableParameterizedTypes.put(param.getKey(), paramTypes); - } - } - - final Set columnVariables = new HashSet<>(); - columnVariables.add("i"); - columnVariables.add("ii"); - columnVariables.add("k"); - - final BiConsumer> createColumnMappings = (columnName, column) -> { - final Class vectorType = DhFormulaColumn.getVectorType(column.getDataType()); - - columnVariables.add(columnName); - if (possibleVariables.put(columnName, column.getDataType()) != null) { - possibleVariableParameterizedTypes.remove(columnName); - } - columnVariables.add(columnName + COLUMN_SUFFIX); - if (possibleVariables.put(columnName + COLUMN_SUFFIX, vectorType) != null) { - possibleVariableParameterizedTypes.remove(columnName + COLUMN_SUFFIX); - } - - final Class compType = column.getComponentType(); - if (compType != null && !compType.isPrimitive()) { - possibleVariableParameterizedTypes.put(columnName, new Class[] {compType}); - } - if (vectorType == ObjectVector.class) { - possibleVariableParameterizedTypes.put(columnName + COLUMN_SUFFIX, - new Class[] {column.getDataType()}); - } - }; - - // By default all columns are available to the formula - for (final ColumnDefinition column : tableDefinition.getColumns()) { - createColumnMappings.accept(column.getName(), column); - } - // Overwrite any existing column mapping using the provided renames. - for (final Map.Entry entry : outerToInnerNames.entrySet()) { - final String columnName = entry.getKey(); - final ColumnDefinition column = tableDefinition.getColumn(entry.getValue()); - createColumnMappings.accept(columnName, column); - } - - log.debug("Expression (before) : " + formula); - - final TimeLiteralReplacedExpression timeConversionResult = - TimeLiteralReplacedExpression.convertExpression(formula); - - log.debug("Expression (after time conversion) : " + timeConversionResult.getConvertedFormula()); + final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( + formula, tableDefinition.getColumnNameMap(), outerToInnerNames, + compilationProcessor.getQueryScopeVariables(), unboxArguments); - possibleVariables.putAll(timeConversionResult.getNewVariables()); - - final QueryLanguageParser.Result result = new QueryLanguageParser( - timeConversionResult.getConvertedFormula(), - ExecutionContext.getContext().getQueryLibrary().getPackageImports(), - ExecutionContext.getContext().getQueryLibrary().getClassImports(), - ExecutionContext.getContext().getQueryLibrary().getStaticImports(), - possibleVariables, possibleVariableParameterizedTypes, queryScopeVariables, columnVariables, - unboxArguments) - .getResult(); formulaShiftColPair = result.getFormulaShiftColPair(); if (formulaShiftColPair != null) { log.debug("Formula (after shift conversion) : " + formulaShiftColPair.getFirst()); @@ -225,7 +153,7 @@ public synchronized void init(TableDefinition tableDefinition) { final Class resultType = result.getType(); checkReturnType(result, resultType); - generateFilterCode(tableDefinition, timeConversionResult, result); + generateFilterCode(tableDefinition, result.getTimeConversionResult(), result, compilationProcessor); initialized = true; } } catch (Exception e) { @@ -270,15 +198,19 @@ private void checkAndInitializeVectorization(QueryLanguageParser.Result result, checkReturnType(result, pyCallableWrapper.getSignature().getReturnType()); for (String variable : result.getVariablesUsed()) { - if (variable.equals("i")) { - usesI = true; - usedColumns.add("i"); - } else if (variable.equals("ii")) { - usesII = true; - usedColumns.add("ii"); - } else if (variable.equals("k")) { - usesK = true; - usedColumns.add("k"); + switch (variable) { + case "i": + usesI = true; + usedColumns.add("i"); + break; + case "ii": + usesII = true; + usedColumns.add("ii"); + break; + case "k": + usesK = true; + usedColumns.add("k"); + break; } } ArgumentsChunked argumentsChunked = pyCallableWrapper.buildArgumentsChunked(usedColumns); @@ -303,9 +235,12 @@ private void checkReturnType(QueryLanguageParser.Result result, Class resultT } } - protected abstract void generateFilterCode(TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) throws MalformedURLException, ClassNotFoundException; + protected abstract void generateFilterCode( + @NotNull TableDefinition tableDefinition, + @NotNull TimeLiteralReplacedExpression timeConversionResult, + @NotNull QueryLanguageParser.Result result, + @NotNull QueryCompilerRequestProcessor compilationProcessor) + throws MalformedURLException, ClassNotFoundException; @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java index 52111727915..03fd59fdd77 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java @@ -3,12 +3,15 @@ // package io.deephaven.engine.table.impl.select; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.verify.Require; import io.deephaven.configuration.Configuration; import io.deephaven.engine.table.*; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.table.impl.vector.*; import io.deephaven.engine.table.impl.select.formula.*; @@ -18,9 +21,14 @@ import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.api.util.NameValidator; +import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Consumer; /** @@ -34,11 +42,12 @@ public abstract class AbstractFormulaColumn implements FormulaColumn { protected String formulaString; + protected final String originalFormulaString; protected List usedColumns; @NotNull protected final String columnName; - protected FormulaFactory formulaFactory; + protected Future formulaFactoryFuture; private Formula formula; protected QueryScopeParam[] params; protected Map> columnSources; @@ -61,6 +70,7 @@ public abstract class AbstractFormulaColumn implements FormulaColumn { */ protected AbstractFormulaColumn(String columnName, String formulaString) { this.formulaString = Require.neqNull(formulaString, "formulaString"); + this.originalFormulaString = formulaString; this.columnName = NameValidator.validateColumnName(columnName); } @@ -69,6 +79,11 @@ public Class getReturnedType() { return returnedType; } + @Override + public Class getReturnedComponentType() { + return returnedType.getComponentType(); + } + @Override public List initInputs( @NotNull final TrackingRowSet rowSet, @@ -79,7 +94,8 @@ public List initInputs( if (usedColumns != null) { return usedColumns; } - return initDef(extractDefinitions(columnsOfInterest)); + + return initDef(extractDefinitions(columnsOfInterest), QueryCompilerRequestProcessor.immediate()); } @Override @@ -136,7 +152,7 @@ protected void applyUsedVariables( } protected void onCopy(final AbstractFormulaColumn copy) { - copy.formulaFactory = formulaFactory; + copy.formulaFactoryFuture = formulaFactoryFuture; copy.columnDefinitions = columnDefinitions; copy.params = params; copy.usedColumns = usedColumns; @@ -220,7 +236,15 @@ private ColumnSource getViewColumnSource(boolean lazy) { private Formula getFormula(boolean initLazyMap, Map> columnsToData, QueryScopeParam... params) { - formula = formulaFactory.createFormula(rowSet, initLazyMap, columnsToData, params); + try { + // the future must already be completed or else it is an error + formula = formulaFactoryFuture.get(0, TimeUnit.SECONDS).createFormula( + StringEscapeUtils.escapeJava(columnName), rowSet, initLazyMap, columnsToData, params); + } catch (InterruptedException | TimeoutException e) { + throw new IllegalStateException("Formula factory not already compiled!"); + } catch (ExecutionException e) { + throw new UncheckedDeephavenException("Error creating formula for " + columnName, e.getCause()); + } return formula; } @@ -254,26 +278,28 @@ private static Vector makeAppropriateVectorWrapper(ColumnSource cs, RowSet return new ObjectVectorColumnWrapper<>((ColumnSource) cs, rowSet); } - protected FormulaFactory createKernelFormulaFactory(final FormulaKernelFactory formulaKernelFactory) { + protected Future createKernelFormulaFactory( + @NotNull final CompletionStageFuture formulaKernelFactoryFuture) { final FormulaSourceDescriptor sd = getSourceDescriptor(); - return (rowSet, lazy, columnsToData, params) -> { - // Maybe warn that we ignore "lazy". By the way, "lazy" is the wrong term anyway. "lazy" doesn't mean - // "cached", which is how we are using it. - final Map> netColumnSources = new HashMap<>(); - for (final String columnName : sd.sources) { - final ColumnSource columnSourceToUse = columnsToData.get(columnName); - netColumnSources.put(columnName, columnSourceToUse); - } - - final Vector[] vectors = new Vector[sd.arrays.length]; - for (int ii = 0; ii < sd.arrays.length; ++ii) { - final ColumnSource cs = columnsToData.get(sd.arrays[ii]); - vectors[ii] = makeAppropriateVectorWrapper(cs, rowSet); - } - final FormulaKernel fk = formulaKernelFactory.createInstance(vectors, params); - return new FormulaKernelAdapter(rowSet, sd, netColumnSources, fk); - }; + return formulaKernelFactoryFuture + .thenApply(formulaKernelFactory -> (columnName, rowSet, lazy, columnsToData, params) -> { + // Maybe warn that we ignore "lazy". By the way, "lazy" is the wrong term anyway. "lazy" doesn't + // mean "cached", which is how we are using it. + final Map> netColumnSources = new HashMap<>(); + for (final String sourceColumnName : sd.sources) { + final ColumnSource columnSourceToUse = columnsToData.get(sourceColumnName); + netColumnSources.put(sourceColumnName, columnSourceToUse); + } + + final Vector[] vectors = new Vector[sd.arrays.length]; + for (int ii = 0; ii < sd.arrays.length; ++ii) { + final ColumnSource cs = columnsToData.get(sd.arrays[ii]); + vectors[ii] = makeAppropriateVectorWrapper(cs, rowSet); + } + final FormulaKernel fk = formulaKernelFactory.createInstance(vectors, params); + return new FormulaKernelAdapter(rowSet, sd, netColumnSources, fk); + }); } protected abstract FormulaSourceDescriptor getSourceDescriptor(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java index a967463f1bd..513dafc7d36 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java @@ -32,13 +32,13 @@ public abstract class AbstractRangeFilter extends WhereFilterImpl { /** * The chunkFilter can be applied to the columns native type. - * + *

* In practice, this is for non-reinterpretable DateTimes. */ ChunkFilter chunkFilter; /** - * If the column can be be reinterpreted to a long, then we should prefer to use the longFilter instead. - * + * If the column can be reinterpreted to a long, then we should prefer to use the longFilter instead. + *

* In practice, this is used for reinterpretable DateTimes. */ ChunkFilter longFilter; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java index 78d8c588417..887f5a221fa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java @@ -19,7 +19,7 @@ /** * Base class for filters that will release more rows of a table on each UGP cycle. - * + *

* The use case is for benchmarks that want to replay a table in order to better understand incremental processing * capacity. */ @@ -67,7 +67,7 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { initialized = true; if (!started) { return; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java index c80de6413d8..d506eee8bc5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java @@ -67,7 +67,7 @@ static WhereFilter makeByteRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java index 7c22ea023ce..f5d640d13f5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java @@ -63,7 +63,7 @@ static WhereFilter makeCharRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java index c5ab78e9baa..3050b8d13a3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java @@ -41,7 +41,7 @@ public static ComparableRangeFilter makeForTest(String columnName, Comparable } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java index 4bc67ce3e8b..d7cd648006d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java @@ -5,6 +5,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.liveness.LivenessArtifact; @@ -61,9 +62,18 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull TableDefinition tableDefinition) { + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); + init(tableDefinition, compilationProcessor); + compilationProcessor.compile(); + } + + @Override + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { for (WhereFilter filter : componentFilters) { - filter.init(tableDefinition); + filter.init(tableDefinition, compilationProcessor); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java index 8728d9173bc..f08995981e0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java @@ -5,8 +5,9 @@ import io.deephaven.base.Pair; import io.deephaven.chunk.attributes.Any; -import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Context; import io.deephaven.engine.table.SharedContext; @@ -14,11 +15,11 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; import io.deephaven.engine.table.impl.util.codegen.CodeGenerator; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.ColumnSource; import io.deephaven.chunk.*; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; @@ -34,6 +35,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import static io.deephaven.engine.table.impl.select.DhFormulaColumn.COLUMN_SUFFIX; @@ -44,7 +49,7 @@ public class ConditionFilter extends AbstractConditionFilter { public static final int CHUNK_SIZE = 4096; - private Class filterKernelClass = null; + private Future> filterKernelClassFuture = null; private List>> usedInputs; // that is columns and special variables private String classBody; private Filter filter = null; @@ -65,7 +70,7 @@ public static WhereFilter createConditionFilter(@NotNull String formula, Formula case Numba: throw new UnsupportedOperationException("Python condition filter should be created from python"); default: - throw new UnsupportedOperationException("Unknow parser type " + parser); + throw new UnsupportedOperationException("Unknown parser type " + parser); } } @@ -378,47 +383,54 @@ private static String toTitleCase(String input) { @Override protected void generateFilterCode( - TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) { + @NotNull final TableDefinition tableDefinition, + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final QueryLanguageParser.Result result, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { final StringBuilder classBody = getClassBody(tableDefinition, timeConversionResult, result); - if (classBody == null) + if (classBody == null) { return; - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(formula)) { - final List> paramClasses = new ArrayList<>(); - final Consumer> addParamClass = (cls) -> { - if (cls != null) { - paramClasses.add(cls); - } - }; - for (String usedColumn : usedColumns) { - usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); - final ColumnDefinition column = tableDefinition.getColumn(usedColumn); - addParamClass.accept(column.getDataType()); - addParamClass.accept(column.getComponentType()); - } - for (String usedColumn : usedColumnArrays) { - usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); - final ColumnDefinition column = tableDefinition.getColumn(usedColumn); - addParamClass.accept(column.getDataType()); - addParamClass.accept(column.getComponentType()); - } - for (final QueryScopeParam param : params) { - addParamClass.accept(QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); - } - - filterKernelClass = ExecutionContext.getContext().getQueryCompiler() - .compile("GeneratedFilterKernel", this.classBody = classBody.toString(), - QueryCompiler.FORMULA_PREFIX, QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)); } + + final List> paramClasses = new ArrayList<>(); + final Consumer> addParamClass = (cls) -> { + if (cls != null) { + paramClasses.add(cls); + } + }; + for (String usedColumn : usedColumns) { + usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); + final ColumnDefinition column = tableDefinition.getColumn(usedColumn); + addParamClass.accept(column.getDataType()); + addParamClass.accept(column.getComponentType()); + } + for (String usedColumn : usedColumnArrays) { + usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); + final ColumnDefinition column = tableDefinition.getColumn(usedColumn); + addParamClass.accept(column.getDataType()); + addParamClass.accept(column.getComponentType()); + } + for (final QueryScopeParam param : params) { + addParamClass.accept(QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); + } + + this.classBody = classBody.toString(); + + filterKernelClassFuture = compilationProcessor.submit(QueryCompilerRequest.builder() + .description("Filter Expression: " + formula) + .className("GeneratedFilterKernel") + .classBody(this.classBody) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) + .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) + .build()); } @Nullable private StringBuilder getClassBody( - TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) { - if (filterKernelClass != null) { + @NotNull final TableDefinition tableDefinition, + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final QueryLanguageParser.Result result) { + if (filterKernelClassFuture != null) { return null; } usedInputs = new ArrayList<>(); @@ -577,15 +589,23 @@ private StringBuilder getClassBody( protected Filter getFilter(Table table, RowSet fullSet) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { if (filter == null) { - final FilterKernel filterKernel = (FilterKernel) filterKernelClass - .getConstructor(Table.class, RowSet.class, QueryScopeParam[].class) - .newInstance(table, fullSet, (Object) params); - final String[] columnNames = usedInputs.stream() - .map(p -> outerToInnerNames.getOrDefault(p.first, p.first)) - .toArray(String[]::new); - filter = new ChunkFilter(filterKernel, columnNames, CHUNK_SIZE); - // note this filter is not valid for use in other contexts, as it captures references from the source table - filterValidForCopy = false; + try { + final FilterKernel filterKernel = (FilterKernel) filterKernelClassFuture + .get(0, TimeUnit.SECONDS) + .getConstructor(Table.class, RowSet.class, QueryScopeParam[].class) + .newInstance(table, fullSet, (Object) params); + final String[] columnNames = usedInputs.stream() + .map(p -> outerToInnerNames.getOrDefault(p.first, p.first)) + .toArray(String[]::new); + filter = new ChunkFilter(filterKernel, columnNames, CHUNK_SIZE); + // note this filter is not valid for use in other contexts, as it captures references from the source + // table + filterValidForCopy = false; + } catch (InterruptedException | TimeoutException e) { + throw new IllegalStateException("Formula factory not already compiled!"); + } catch (ExecutionException e) { + throw new FormulaCompilationException("Formula compilation error for: " + formula, e.getCause()); + } } return filter; } @@ -600,7 +620,7 @@ public ConditionFilter copy() { final ConditionFilter copy = new ConditionFilter(formula, outerToInnerNames); onCopy(copy); if (initialized) { - copy.filterKernelClass = filterKernelClass; + copy.filterKernelClassFuture = filterKernelClassFuture; copy.usedInputs = usedInputs; copy.classBody = classBody; if (filterValidForCopy) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index e1babe4bec3..31a80d3a4e2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -3,18 +3,20 @@ // package io.deephaven.engine.table.impl.select; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.Pair; import io.deephaven.chunk.ChunkType; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.Table; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.select.codegen.FormulaAnalyzer; import io.deephaven.engine.table.impl.select.codegen.JavaKernelBuilder; import io.deephaven.engine.table.impl.select.codegen.RichType; @@ -30,12 +32,10 @@ import io.deephaven.engine.util.caching.C14nUtil; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; -import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.util.SafeCloseable; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.type.TypeUtils; import io.deephaven.vector.ObjectVector; import io.deephaven.vector.Vector; -import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; import org.jpy.PyObject; @@ -43,11 +43,8 @@ import java.math.BigInteger; import java.time.Instant; import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.Function; @@ -182,19 +179,24 @@ public static Class getVectorType(Class declaredType) { } @Override - public List initDef(Map> columnDefinitionMap) { - if (formulaFactory != null) { + public List initDef(@NotNull final Map> columnDefinitionMap) { + return initDef(columnDefinitionMap, QueryCompilerRequestProcessor.immediate()); + } + + @Override + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + if (formulaFactoryFuture != null) { validateColumnDefinition(columnDefinitionMap); return formulaColumnPython != null ? formulaColumnPython.usedColumns : usedColumns; } try { - final TimeLiteralReplacedExpression timeConversionResult = - TimeLiteralReplacedExpression.convertExpression(formulaString); - final QueryLanguageParser.Result result = FormulaAnalyzer.getCompiledFormula(columnDefinitionMap, - timeConversionResult); - analyzedFormula = FormulaAnalyzer.analyze(formulaString, columnDefinitionMap, - timeConversionResult, result); + final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( + formulaString, columnDefinitionMap, Collections.emptyMap(), + compilationRequestProcessor.getQueryScopeVariables()); + analyzedFormula = FormulaAnalyzer.analyze(formulaString, columnDefinitionMap, result); hasConstantValue = result.isConstantValueExpression(); formulaShiftColPair = result.getFormulaShiftColPair(); @@ -209,18 +211,22 @@ public List initDef(Map> columnDefinitionMap formulaString = result.getConvertedExpression(); // check if this is a column to be created with a Python vectorizable function - checkAndInitializeVectorization(columnDefinitionMap); + checkAndInitializeVectorization(columnDefinitionMap, compilationRequestProcessor); } catch (Exception e) { - throw new FormulaCompilationException("Formula compilation error for: " + formulaString, e); + throw new FormulaCompilationException("Formula compilation error for: " + originalFormulaString, e); } - formulaFactory = useKernelFormulasProperty - ? createKernelFormulaFactory(getFormulaKernelFactory()) - : createFormulaFactory(); + if (useKernelFormulasProperty) { + formulaFactoryFuture = createKernelFormulaFactory(getFormulaKernelFactory(compilationRequestProcessor)); + } else { + compileFormula(compilationRequestProcessor); + } return formulaColumnPython != null ? formulaColumnPython.usedColumns : usedColumns; } - private void checkAndInitializeVectorization(Map> columnDefinitionMap) { + private void checkAndInitializeVectorization( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection SuspiciousToArrayCall final PyCallableWrapperJpyImpl[] cws = Arrays.stream(params) .filter(p -> p.getValue() instanceof PyCallableWrapperJpyImpl) @@ -242,7 +248,7 @@ private void checkAndInitializeVectorization(Map> co this.analyzedFormula.sourceDescriptor.sources, argumentsChunked, true)); - formulaColumnPython.initDef(columnDefinitionMap); + formulaColumnPython.initDef(columnDefinitionMap, compilationRequestProcessor); } } @@ -254,6 +260,7 @@ String generateClassBody() { CodeGenerator.create(ExecutionContext.getContext().getQueryLibrary().getImportStrings().toArray()), "", "public class $CLASSNAME$ extends [[FORMULA_CLASS_NAME]]", CodeGenerator.block( generateFormulaFactoryLambda(), "", + "private final String __columnName;", CodeGenerator.repeated("instanceVar", "private final [[TYPE]] [[NAME]];"), "private final Map [[LAZY_RESULT_CACHE_NAME]];", analyzedFormula.timeInstanceVariables, "", @@ -305,12 +312,14 @@ private CodeGenerator generateFormulaFactoryLambda() { private CodeGenerator generateConstructor() { final CodeGenerator g = CodeGenerator.create( - "public $CLASSNAME$(final TrackingRowSet __rowSet,", CodeGenerator.indent( + "public $CLASSNAME$(final String __columnName,", CodeGenerator.indent( + "final TrackingRowSet __rowSet,", "final boolean __lazy,", "final java.util.Map __columnsToData,", "final [[PARAM_CLASSNAME]]... __params)"), CodeGenerator.block( "super(__rowSet);", + "this.__columnName = __columnName;", CodeGenerator.repeated("initColumn", "[[COLUMN_NAME]] = __columnsToData.get(\"[[COLUMN_NAME]]\");"), CodeGenerator.repeated("initNormalColumnArray", @@ -356,7 +365,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { "try", CodeGenerator.block( "return [[FORMULA_STRING]];"), CodeGenerator.samelineBlock("catch (java.lang.Exception __e)", - "throw new [[EXCEPTION_TYPE]](\"In formula: [[COLUMN_NAME]] = \" + [[JOINED_FORMULA_STRING]], __e);"))); + "throw new [[EXCEPTION_TYPE]](\"In formula: \" + __columnName + \" = \" + [[JOINED_FORMULA_STRING]], __e);"))); g.replace("RETURN_TYPE", ta.typeString); final List args = visitFormulaParameters(n -> n.typeString + " " + n.name, n -> n.typeString + " " + n.name, @@ -364,8 +373,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(formulaString)); - g.replace("COLUMN_NAME", StringEscapeUtils.escapeJava(columnName)); - final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(formulaString); + final String joinedFormulaString = QueryCompilerImpl.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", EVALUATION_EXCEPTION_CLASSNAME); return g.freeze(); @@ -706,11 +714,13 @@ protected FormulaSourceDescriptor getSourceDescriptor() { return analyzedFormula.sourceDescriptor; } - protected FormulaKernelFactory getFormulaKernelFactory() { - return invokeKernelBuilder().formulaKernelFactory; + protected CompletionStageFuture getFormulaKernelFactory( + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + return invokeKernelBuilder(compilationRequestProcessor).thenApply(result -> result.formulaKernelFactory); } - private JavaKernelBuilder.Result invokeKernelBuilder() { + private CompletionStageFuture invokeKernelBuilder( + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { final FormulaAnalyzer.Result af = analyzedFormula; final FormulaSourceDescriptor sd = af.sourceDescriptor; final Map columnDict = makeNameToRichTypeDict(sd.sources, columnDefinitions); @@ -723,8 +733,15 @@ private JavaKernelBuilder.Result invokeKernelBuilder() { for (final String p : sd.params) { paramDict.put(p, allParamDict.get(p)); } - return JavaKernelBuilder.create(af.cookedFormulaString, sd.returnType, af.timeInstanceVariables, columnDict, - arrayDict, paramDict); + return JavaKernelBuilder.create( + originalFormulaString, + af.cookedFormulaString, + sd.returnType, + af.timeInstanceVariables, + columnDict, + arrayDict, + paramDict, + compilationRequestProcessor); } /** @@ -732,13 +749,17 @@ private JavaKernelBuilder.Result invokeKernelBuilder() { */ @NotNull String generateKernelClassBody() { - return invokeKernelBuilder().classBody; + try { + return invokeKernelBuilder(QueryCompilerRequestProcessor.immediate()).get().classBody; + } catch (InterruptedException | ExecutionException e) { + throw new UncheckedDeephavenException("Failed to compile formula: ", e); + } } @Override public SelectColumn copy() { final DhFormulaColumn copy = new DhFormulaColumn(columnName, formulaString); - if (formulaFactory != null) { + if (formulaFactoryFuture != null) { copy.analyzedFormula = analyzedFormula; copy.hasConstantValue = hasConstantValue; copy.returnedType = returnedType; @@ -759,48 +780,46 @@ public Pair>> getFormulaShiftColPair() { return formulaShiftColPair; } - private FormulaFactory createFormulaFactory() { - final String classBody = generateClassBody(); + private void compileFormula(@NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { final String what = "Compile regular formula: " + formulaString; - final Class clazz = compileFormula(what, classBody, "Formula"); - try { - return (FormulaFactory) clazz.getField(FORMULA_FACTORY_NAME).get(null); - } catch (ReflectiveOperationException e) { - throw new FormulaCompilationException("Formula compilation error for: " + what, e); - } - } + final String className = "Formula"; + final String classBody = generateClassBody(); - @SuppressWarnings("SameParameterValue") - private Class compileFormula(final String what, final String classBody, final String className) { - // System.out.printf("compileFormula: what is %s. Code is...%n%s%n", what, classBody); - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(what)) { - // Compilation needs to take place with elevated privileges, but the created object should not have them. + final List> paramClasses = new ArrayList<>(); + final Consumer> addParamClass = (cls) -> { + if (cls != null) { + paramClasses.add(cls); + } + }; + visitFormulaParameters(null, + csp -> { + addParamClass.accept(csp.type); + addParamClass.accept(csp.columnDefinition.getComponentType()); + return null; + }, + cap -> { + addParamClass.accept(cap.dataType); + addParamClass.accept(cap.columnDefinition.getComponentType()); + return null; + }, + p -> { + addParamClass.accept(p.type); + return null; + }); - final List> paramClasses = new ArrayList<>(); - final Consumer> addParamClass = (cls) -> { - if (cls != null) { - paramClasses.add(cls); - } - }; - visitFormulaParameters(null, - csp -> { - addParamClass.accept(csp.type); - addParamClass.accept(csp.columnDefinition.getComponentType()); - return null; - }, - cap -> { - addParamClass.accept(cap.dataType); - addParamClass.accept(cap.columnDefinition.getComponentType()); - return null; - }, - p -> { - addParamClass.accept(p.type); - return null; - }); - final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); - return compiler.compile(className, classBody, QueryCompiler.FORMULA_PREFIX, - QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)); - } + formulaFactoryFuture = compilationRequestProcessor.submit(QueryCompilerRequest.builder() + .description("Formula Expression: " + formulaString) + .className(className) + .classBody(classBody) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) + .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) + .build()).thenApply(clazz -> { + try { + return (FormulaFactory) clazz.getField(FORMULA_FACTORY_NAME).get(null); + } catch (ReflectiveOperationException e) { + throw new FormulaCompilationException("Formula compilation error for: " + what, e); + } + }); } private static class IndexParameter { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java index 3c4cc277672..1b195e01b0b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java @@ -79,7 +79,7 @@ static WhereFilter makeDoubleRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java index 3d45c6c6edf..d917756a289 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java @@ -85,7 +85,7 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java index 53527caf51c..2cedb5ea274 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java @@ -403,7 +403,7 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java index d5f89f0fbe7..2cdebf0bde7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java @@ -75,7 +75,7 @@ static WhereFilter makeFloatRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java index e7651213c6b..7f48ca55598 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java @@ -114,7 +114,7 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef(@NotNull final Map> columnDefinitionMap) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { @@ -133,6 +133,11 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + return componentType; + } + @Override public List getColumns() { return List.of(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumnLong.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumnLong.java index ca72e42b00c..071858b3f92 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumnLong.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumnLong.java @@ -87,7 +87,7 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef(@NotNull final Map> columnDefinitionMap) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { @@ -106,6 +106,11 @@ public Class getReturnedType() { return long.class; } + @Override + public Class getReturnedComponentType() { + return null; + } + @Override public List getColumns() { return List.of(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java index 1089a546ad1..7d45254db75 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java @@ -37,7 +37,7 @@ public InstantRangeFilter( } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java index e730768584c..ee4a345e2a2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java @@ -67,7 +67,7 @@ static WhereFilter makeIntRangeFilter(String columnName, Condition condition, St } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java index a412b35b613..d0718eab953 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java @@ -67,7 +67,7 @@ static WhereFilter makeLongRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 421ac67feeb..0ad27701356 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -5,8 +5,6 @@ import io.deephaven.api.literal.Literal; import io.deephaven.base.string.cache.CompressedString; -import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.liveness.LivenessScopeStack; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.WritableRowSet; @@ -15,9 +13,10 @@ import io.deephaven.engine.table.DataIndex; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.engine.table.impl.preview.DisplayWrapper; import io.deephaven.engine.table.impl.DependencyStreamProvider; import io.deephaven.engine.table.impl.indexer.DataIndexer; -import io.deephaven.engine.table.impl.preview.DisplayWrapper; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.time.DateTimeUtils; import io.deephaven.util.SafeCloseable; @@ -155,7 +154,14 @@ public List getColumnArrays() { } @Override - public synchronized void init(TableDefinition tableDefinition) { + public void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, QueryCompilerRequestProcessor.immediate()); + } + + @Override + public synchronized void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (initialized) { return; } @@ -169,12 +175,12 @@ public synchronized void init(TableDefinition tableDefinition) { return; } final List valueList = new ArrayList<>(); - final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); + final Map queryScopeVariables = compilationProcessor.getQueryScopeVariables(); final ColumnTypeConvertor convertor = ColumnTypeConvertorFactory.getConvertor(column.getDataType(), column.getName()); for (String strValue : strValues) { - if (queryScope != null && queryScope.hasParamName(strValue)) { - Object paramValue = queryScope.readParamValue(strValue); + if (queryScopeVariables.containsKey(strValue)) { + Object paramValue = queryScopeVariables.get(strValue); if (paramValue != null && paramValue.getClass().isArray()) { ArrayTypeUtils.ArrayAccessor accessor = ArrayTypeUtils.getArrayAccessor(paramValue); for (int ai = 0; ai < accessor.length(); ++ai) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java index e96fbf758e3..4dd09a88eaa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java @@ -100,7 +100,7 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef(@NotNull final Map> columnDefinitionMap) { NoSuchColumnException.throwIf(columnDefinitionMap.keySet(), sourceNames); return getColumns(); } @@ -110,6 +110,11 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + return componentType; + } + @Override public List getColumns() { return Collections.unmodifiableList(sourceNames); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java index 3cdb2939328..c897dcb7daf 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java @@ -32,7 +32,7 @@ public List initInputs(final TrackingRowSet rowSet, } @Override - public List initDef(final Map> columnDefinitionMap) { + public List initDef(@NotNull final Map> columnDefinitionMap) { return Collections.emptyList(); } @@ -41,6 +41,11 @@ public Class getReturnedType() { return nvcs.getType(); } + @Override + public Class getReturnedComponentType() { + return nvcs.getComponentType(); + } + @Override public List getColumns() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java index eec62883efc..24cb8332a46 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java @@ -4,7 +4,7 @@ package io.deephaven.engine.table.impl.select; import groovy.lang.Closure; -import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.util.type.TypeUtils; import java.lang.reflect.Modifier; @@ -48,7 +48,7 @@ private static void visitParameterClass(final Map> found, Class } final String name = cls.getName(); - if (!name.startsWith(QueryCompiler.DYNAMIC_GROOVY_CLASS_PREFIX)) { + if (!name.startsWith(QueryCompilerImpl.DYNAMIC_CLASS_PREFIX)) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java index 374b0a4f0df..f7cdfaa570a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java @@ -7,6 +7,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.time.DateTimeUtils; import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.rowset.RowSet; @@ -129,18 +130,25 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, QueryCompilerRequestProcessor.immediate()); + } + + @Override + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (filter != null) { return; } - final ColumnDefinition def = tableDefinition.getColumn(columnName); + final ColumnDefinition def = tableDefinition.getColumn(columnName); if (def == null) { throw new RuntimeException("Column \"" + columnName + "\" doesn't exist in this table, available columns: " + tableDefinition.getColumnNames()); } - final Class colClass = def.getDataType(); + final Class colClass = def.getDataType(); if (colClass == double.class || colClass == Double.class) { filter = DoubleRangeFilter.makeDoubleRangeFilter(columnName, condition, value); @@ -179,7 +187,7 @@ public void init(TableDefinition tableDefinition) { } } - filter.init(tableDefinition); + filter.init(tableDefinition, compilationProcessor); } public static char parseCharFilter(String value) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java index 0d258a9072a..c4ecb9a0394 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java @@ -38,13 +38,13 @@ /** * Allows {@link ColumnSource} reinterpretation via view-type ({@link Table#view} and {@link Table#updateView}) * {@link Table} operations. - * + *

* TODO: If we come up with other valid, useful reinterpretations, it would be trivial to create a general purpose * syntax for use in view()/updateView() column expressions. - * + *

* The syntax I have in mind is: "<ColumnNameB>=<ColumnNameA>.as(<ClassName>)" * "<ColumnName>.as(<ClassName>)" - * + *

* Making this work would consist of any one of: 1. Adding a V1 version and updating SelectColumnFactory and * SelectColumnAdaptor 2. Adding the appropriate if-regex-matches to realColumn selection in V2 SwitchColumn 3. Creating * a V2-native SelectColumnFactory @@ -151,7 +151,7 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef(@NotNull final Map> columnDefinitionMap) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { @@ -169,6 +169,12 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + // we don't support reinterpreting column types with components + return null; + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java index fdd46b07550..6459c65c704 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java @@ -45,7 +45,7 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java index 480e4d8e10f..fe05a0675ca 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java @@ -12,6 +12,7 @@ import io.deephaven.api.expression.Method; import io.deephaven.api.filter.Filter; import io.deephaven.api.literal.Literal; +import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.ColumnDefinition; @@ -19,6 +20,7 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import org.jetbrains.annotations.NotNull; import java.util.Arrays; @@ -71,7 +73,8 @@ static Collection copyFrom(Collection selectColumns) List initInputs(TrackingRowSet rowSet, Map> columnsOfInterest); /** - * Initialize any internal column definitions from the provided initial. + * Initialize any internal column definitions from the provided initial. Any formulae will be compiled immediately + * using the {@link QueryCompiler} in the current {@link ExecutionContext}. * * @param columnDefinitionMap the starting set of column definitions; valid for this call only * @@ -80,7 +83,27 @@ static Collection copyFrom(Collection selectColumns) * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. */ - List initDef(Map> columnDefinitionMap); + List initDef(@NotNull Map> columnDefinitionMap); + + /** + * Initialize any internal column definitions from the provided initial. A compilation request consumer is provided + * to allow for deferred compilation of expressions that belong to the same query. + *

+ * Compilations must be resolved before using this {@code SelectColumn}. + * + * @param columnDefinitionMap the starting set of column definitions; valid for this call only + * @param compilationRequestProcessor a consumer to submit compilation requests; valid for this call only + * + * @return a list of columns on which the result of this is dependent + * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or + * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. + * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. + */ + default List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + return initDef(columnDefinitionMap); + } /** * Get the data type stored in the resultant column. @@ -89,6 +112,13 @@ static Collection copyFrom(Collection selectColumns) */ Class getReturnedType(); + /** + * Get the data component type stored in the resultant column. + * + * @return the component type + */ + Class getReturnedComponentType(); + /** * Get a list of the names of columns used in this SelectColumn. Behavior is undefined if none of the init* methods * have been called yet. diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java index 737f8cf0f3e..461707e7b7f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java @@ -67,7 +67,7 @@ static WhereFilter makeShortRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java index 01b04d4561a..d9a54a7758b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java @@ -34,7 +34,7 @@ public static SingleSidedComparableRangeFilter makeForTest(String columnName, Co } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java index a0f53b462ea..fcaa5d30bd2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java @@ -46,7 +46,7 @@ public SourceColumn(String sourceName, String destName) { this(NameValidator.validateColumnName(sourceName), NameValidator.validateColumnName(destName), true); } - private SourceColumn(String sourceName, String destName, boolean unused) { + private SourceColumn(@NotNull final String sourceName, @NotNull final String destName, boolean unused) { this.sourceName = sourceName; this.destName = destName; } @@ -61,7 +61,7 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef(@NotNull final Map> columnDefinitionMap) { sourceDefinition = columnDefinitionMap.get(sourceName); if (sourceDefinition == null) { throw new NoSuchColumnException(columnDefinitionMap.keySet(), sourceName); @@ -78,6 +78,15 @@ public Class getReturnedType() { return sourceColumn.getType(); } + @Override + public Class getReturnedComponentType() { + // Try to be a little flexible, depending on whether initInputs or initDef was called. + if (sourceDefinition != null) { + return sourceDefinition.getComponentType(); + } + return sourceColumn.getComponentType(); + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java index 4e44a7bd3a2..d380d98ed36 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java @@ -8,6 +8,7 @@ import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.python.FormulaColumnPython; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.rowset.TrackingRowSet; @@ -45,7 +46,14 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef(@NotNull Map> columnDefinitionMap) { + return initDef(columnDefinitionMap, QueryCompilerRequestProcessor.immediate()); + } + + @Override + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (realColumn == null) { if (columnDefinitionMap.get(expression) != null) { realColumn = new SourceColumn(expression, columnName); @@ -53,7 +61,7 @@ public List initDef(Map> columnDefinitionMap realColumn = FormulaColumn.createFormulaColumn(columnName, expression, parser); } } - List usedColumns = realColumn.initDef(columnDefinitionMap); + final List usedColumns = realColumn.initDef(columnDefinitionMap, compilationRequestProcessor); if (realColumn instanceof DhFormulaColumn) { FormulaColumnPython formulaColumnPython = ((DhFormulaColumn) realColumn).getFormulaColumnPython(); realColumn = formulaColumnPython != null ? formulaColumnPython : realColumn; @@ -66,6 +74,11 @@ public Class getReturnedType() { return getRealColumn().getReturnedType(); } + @Override + public Class getReturnedComponentType() { + return getRealColumn().getReturnedComponentType(); + } + @Override public List getColumns() { return getRealColumn().getColumns(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java index 4ca30178f64..497957f4e45 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java @@ -59,7 +59,7 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java index b6486527259..88e40930ba7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java @@ -12,6 +12,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.util.SafeCloseable; @@ -106,7 +107,22 @@ interface RecomputeListener { * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or * {@link QueryCompiler} usage needs to be resolved within init. Implementations must be idempotent. */ - void init(TableDefinition tableDefinition); + void init(@NotNull TableDefinition tableDefinition); + + /** + * Initialize this select filter given the table definition + * + * @param tableDefinition the definition of the table that will be filtered + * @param compilationProcessor the processor to use for compilation + * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or + * {@link QueryCompiler} usage needs to be resolved within init. Implementations must be idempotent. + */ + @SuppressWarnings("unused") + default void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + init(tableDefinition); + } /** * Perform any operation-level initialization necessary using the {@link Table} that will be filtered with this diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java index 97f4e46c1dd..eae7fa10a70 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java @@ -10,6 +10,7 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.DependencyStreamProvider; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.util.SafeCloseable; import io.deephaven.util.annotations.VisibleForTesting; @@ -57,8 +58,15 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { - filter.init(tableDefinition); + public void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, QueryCompilerRequestProcessor.immediate()); + } + + @Override + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + filter.init(tableDefinition, compilationProcessor); } @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java index f3cf1df29b6..de19278857d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java @@ -43,7 +43,7 @@ private WhereFilterPatternImpl(FilterPattern filterPattern) { } @Override - public void init(TableDefinition tableDefinition) { + public void init(@NotNull final TableDefinition tableDefinition) { final String columnName = columnName(); final ColumnDefinition column = tableDefinition.getColumn(columnName); if (column == null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java index 4495af78636..7a8a686ba9e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java @@ -33,7 +33,7 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java index ecb4207fc4a..0d9c0892c4d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java @@ -59,16 +59,6 @@ final Map> getColumnSourcesRecurse(GetMode mode) { return result; } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - for (Map.Entry> entry : sources.entrySet()) { - final String name = entry.getKey(); - final ColumnSource cs = entry.getValue(); - final ColumnDefinition cd = ColumnDefinition.fromGenericType(name, cs.getType(), cs.getComponentType()); - columnDefinitions.put(name, cd); - } - } - @Override public void applyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, JobScheduler jobScheduler, @Nullable LivenessNode liveResultOwner, SelectLayerCompletionHandler onCompletion) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java index 4ad846ceba3..67fa89b424a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java @@ -36,14 +36,6 @@ public abstract class DependencyLayerBase extends SelectAndViewAnalyzer { this.myModifiedColumnSet = mcsBuilder; } - - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - final ColumnDefinition cd = - ColumnDefinition.fromGenericType(name, columnSource.getType(), columnSource.getComponentType()); - columnDefinitions.put(name, cd); - } - @Override void populateModifiedColumnSetRecurse(ModifiedColumnSet mcsBuilder, Set remainingDepsToSatisfy) { // Later-defined columns override earlier-defined columns. So we satisfy column dependencies "on the way diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java index f9f9b44ffb9..59963f1df13 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java @@ -164,11 +164,6 @@ public SelectAndViewAnalyzer getInner() { return inner; } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - inner.updateColumnDefinitionsFromTopLayer(columnDefinitions); - } - @Override public void startTrackingPrev() { rowRedirection.startTrackingPrevValues(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java index 42eeb4b933c..b2a5907e32e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java @@ -12,6 +12,7 @@ import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.select.SelectColumn; @@ -47,11 +48,20 @@ public enum Mode { public static void initializeSelectColumns( final Map> parentColumnMap, final SelectColumn[] selectColumns) { + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); + initializeSelectColumns(parentColumnMap, selectColumns, compilationProcessor); + compilationProcessor.compile(); + } + + public static void initializeSelectColumns( + final Map> parentColumnMap, + final SelectColumn[] selectColumns, + final QueryCompilerRequestProcessor compilationProcessor) { final Map> targetColumnMap = new HashMap<>(parentColumnMap); for (SelectColumn column : selectColumns) { - column.initDef(targetColumnMap); - final ColumnDefinition columnDefinition = - ColumnDefinition.fromGenericType(column.getName(), column.getReturnedType()); + column.initDef(targetColumnMap, compilationProcessor); + final ColumnDefinition columnDefinition = ColumnDefinition.fromGenericType( + column.getName(), column.getReturnedType(), column.getReturnedComponentType()); targetColumnMap.put(column.getName(), columnDefinition); } } @@ -89,30 +99,60 @@ public static SelectAndViewAnalyzerWrapper create( rowRedirection = null; } - final TrackingRowSet originalRowSet = rowSet; - boolean flatResult = rowSet.isFlat(); - // if we preserve a column, we set this to false - boolean flattenedResult = !flatResult - && allowInternalFlatten - && (columnSources.isEmpty() || !publishTheseSources) - && mode == Mode.SELECT_STATIC; - int numberOfInternallyFlattenedColumns = 0; - List processedCols = new LinkedList<>(); List remainingCols = null; FormulaColumn shiftColumn = null; boolean shiftColumnHasPositiveOffset = false; final HashSet resultColumns = new HashSet<>(); - final HashMap> resultAlias = new HashMap<>(); + + // First pass to initialize all columns and to compile formulas in one batch. + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); + for (Map.Entry> entry : columnSources.entrySet()) { + final String name = entry.getKey(); + final ColumnSource cs = entry.getValue(); + final ColumnDefinition cd = ColumnDefinition.fromGenericType(name, cs.getType(), cs.getComponentType()); + columnDefinitions.put(name, cd); + } + for (final SelectColumn sc : selectColumns) { if (remainingCols != null) { remainingCols.add(sc); continue; } - analyzer.updateColumnDefinitionsFromTopLayer(columnDefinitions); - sc.initDef(columnDefinitions); + sc.initDef(columnDefinitions, compilationProcessor); + final ColumnDefinition cd = ColumnDefinition.fromGenericType( + sc.getName(), sc.getReturnedType(), sc.getReturnedComponentType()); + columnDefinitions.put(sc.getName(), cd); + + if (useShiftedColumns && hasConstantArrayAccess(sc)) { + remainingCols = new LinkedList<>(); + shiftColumn = sc instanceof FormulaColumn + ? (FormulaColumn) sc + : (FormulaColumn) ((SwitchColumn) sc).getRealColumn(); + shiftColumnHasPositiveOffset = hasPositiveOffsetConstantArrayAccess(sc); + continue; + } + + processedCols.add(sc); + } + + compilationProcessor.compile(); + + // Second pass builds the analyzer and destination columns + final TrackingRowSet originalRowSet = rowSet; + boolean flatResult = rowSet.isFlat(); + // if we preserve a column, we set this to false + boolean flattenedResult = !flatResult + && allowInternalFlatten + && (columnSources.isEmpty() || !publishTheseSources) + && mode == Mode.SELECT_STATIC; + int numberOfInternallyFlattenedColumns = 0; + + final HashMap> resultAlias = new HashMap<>(); + for (final SelectColumn sc : processedCols) { + sc.initInputs(rowSet, analyzer.getAllColumnSources()); // When flattening the result, intermediate columns generate results in position space. When we discover @@ -138,12 +178,8 @@ public static SelectAndViewAnalyzerWrapper create( final ModifiedColumnSet mcsBuilder = new ModifiedColumnSet(parentMcs); if (useShiftedColumns && hasConstantArrayAccess(sc)) { - remainingCols = new LinkedList<>(); - shiftColumn = sc instanceof FormulaColumn - ? (FormulaColumn) sc - : (FormulaColumn) ((SwitchColumn) sc).getRealColumn(); - shiftColumnHasPositiveOffset = hasPositiveOffsetConstantArrayAccess(sc); - continue; + // we use the first shifted column to split between processed columns and remaining columns + throw new IllegalStateException("Found ShiftedColumn in processed column list"); } // shifted columns appear to not be safe for refresh, so we do not validate them until they are rewritten @@ -152,8 +188,6 @@ public static SelectAndViewAnalyzerWrapper create( sc.validateSafeForRefresh(sourceTable); } - processedCols.add(sc); - if (hasConstantValue(sc)) { final WritableColumnSource constViewSource = SingleValueColumnSource.getSingleValueColumnSource(sc.getReturnedType()); @@ -177,8 +211,7 @@ public static SelectAndViewAnalyzerWrapper create( if (!sourceIsNew) { if (numberOfInternallyFlattenedColumns > 0) { // we must preserve this column, but have already created an analyzer for the internally - // flattened - // column, therefore must start over without permitting internal flattening + // flattened column, therefore must start over without permitting internal flattening return create(sourceTable, mode, columnSources, originalRowSet, parentMcs, publishTheseSources, useShiftedColumns, false, selectColumns); } else { @@ -248,7 +281,7 @@ public static SelectAndViewAnalyzerWrapper create( case SELECT_REFRESHING: { // We need to call newDestInstance because only newDestInstance has the knowledge to endow our // created array with the proper componentType (in the case of Vectors). - // TODO(kosak): use DeltaAwareColumnSource + // TODO: use DeltaAwareColumnSource WritableColumnSource scs = sc.newDestInstance(targetDestinationCapacity); WritableColumnSource underlyingSource = null; if (rowRedirection != null) { @@ -265,6 +298,7 @@ public static SelectAndViewAnalyzerWrapper create( throw new UnsupportedOperationException("Unsupported case " + mode); } } + return new SelectAndViewAnalyzerWrapper(analyzer, shiftColumn, shiftColumnHasPositiveOffset, remainingCols, processedCols); } @@ -522,8 +556,6 @@ public final Map calcEffects(boolean forcePublishAllResources) public abstract SelectAndViewAnalyzer getInner(); - public abstract void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions); - public abstract void startTrackingPrev(); /** @@ -562,8 +594,8 @@ public boolean alreadyFlattenedSources() { abstract public boolean allowCrossColumnParallelization(); /** - * A class that handles the completion of one select column. The handlers are chained together so that when a column - * completes all of the downstream dependencies may execute. + * A class that handles the completion of one select column. The handlers are chained together; all downstream + * dependencies may execute when a column completes. */ public static abstract class SelectLayerCompletionHandler { /** @@ -591,7 +623,7 @@ public static abstract class SelectLayerCompletionHandler { * Create the final completion handler, which has no next handler. * * @param requiredColumns the columns required for this handler to fire - * @param completedColumns the set of completed columns, shared with all of the other handlers + * @param completedColumns the set of completed columns, shared with all the other handlers */ public SelectLayerCompletionHandler(BitSet requiredColumns, BitSet completedColumns) { this.requiredColumns = requiredColumns; @@ -601,9 +633,9 @@ public SelectLayerCompletionHandler(BitSet requiredColumns, BitSet completedColu /** * Called when a single column is completed. - * + *

* If we are ready, then we call {@link #onAllRequiredColumnsCompleted()}. - * + *

* We may not be ready, but other columns downstream of us may be ready, so they are also notified (the * nextHandler). * @@ -639,7 +671,7 @@ protected void onError(Exception error) { } /** - * Called when all of the required columns are completed. + * Called when all required columns are completed. */ protected abstract void onAllRequiredColumnsCompleted(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java index 9a0cf12a180..25827b2ca19 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java @@ -117,11 +117,6 @@ int getLayerIndexFor(String column) { return inner.getLayerIndexFor(column); } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - inner.updateColumnDefinitionsFromTopLayer(columnDefinitions); - } - @Override public void startTrackingPrev() { throw new UnsupportedOperationException("StaticFlattenLayer is used in only non-refreshing scenarios"); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java index 7dbd37baa25..eecd97ae009 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java @@ -3,9 +3,9 @@ // package io.deephaven.engine.table.impl.select.codegen; -import io.deephaven.api.util.NameValidator; import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; import io.deephaven.engine.table.impl.select.QueryScopeParamTypeUtil; @@ -17,17 +17,20 @@ import io.deephaven.engine.rowset.TrackingWritableRowSet; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; +import org.jetbrains.annotations.NotNull; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.*; +import java.util.function.BiConsumer; + +import static io.deephaven.engine.table.impl.select.AbstractFormulaColumn.COLUMN_SUFFIX; public class FormulaAnalyzer { private static final Logger log = LoggerFactory.getLogger(FormulaAnalyzer.class); public static Result analyze(final String rawFormulaString, final Map> columnDefinitionMap, - final TimeLiteralReplacedExpression timeConversionResult, final QueryLanguageParser.Result queryLanguageResult) throws Exception { log.debug().append("Expression (after language conversion) : ") @@ -58,7 +61,7 @@ public static Result analyze(final String rawFormulaString, returnedType = Boolean.class; } final String cookedFormulaString = queryLanguageResult.getConvertedExpression(); - final String timeInstanceVariables = timeConversionResult.getInstanceVariablesString(); + final String timeInstanceVariables = queryLanguageResult.getTimeConversionResult().getInstanceVariablesString(); return new Result(returnedType, usedColumns.toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY), usedColumnArrays.toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY), @@ -67,8 +70,45 @@ public static Result analyze(final String rawFormulaString, queryLanguageResult.isConstantValueExpression()); } - public static QueryLanguageParser.Result getCompiledFormula(Map> availableColumns, - TimeLiteralReplacedExpression timeConversionResult) throws Exception { + /** + * Get the compiled formula for a given formula string. + * + * @param formulaString The raw formula string + * @param availableColumns The columns available for use in the formula + * @param columnRenames Outer to inner column name mapping + * @param queryScopeVariables The query scope variables + * @return The parsed formula {@link QueryLanguageParser.Result result} + * @throws Exception If the formula cannot be parsed + */ + public static QueryLanguageParser.Result parseFormula( + @NotNull final String formulaString, + @NotNull final Map> availableColumns, + @NotNull final Map columnRenames, + @NotNull final Map queryScopeVariables) throws Exception { + return parseFormula(formulaString, availableColumns, columnRenames, queryScopeVariables, true); + } + + /** + * Get the compiled formula for a given formula string. + * + * @param formulaString The raw formula string + * @param availableColumns The columns available for use in the formula + * @param columnRenames Outer to inner column name mapping + * @param queryScopeVariables The query scope variables + * @param unboxArguments If true it will unbox the query scope arguments + * @return The parsed formula {@link QueryLanguageParser.Result result} + * @throws Exception If the formula cannot be parsed + */ + public static QueryLanguageParser.Result parseFormula( + @NotNull final String formulaString, + @NotNull final Map> availableColumns, + @NotNull final Map columnRenames, + @NotNull final Map queryScopeVariables, + final boolean unboxArguments) throws Exception { + + final TimeLiteralReplacedExpression timeConversionResult = + TimeLiteralReplacedExpression.convertExpression(formulaString); + final Map> possibleVariables = new HashMap<>(); possibleVariables.put("i", int.class); possibleVariables.put("ii", long.class); @@ -81,34 +121,66 @@ public static QueryLanguageParser.Result getCompiledFormula(Map[]> possibleVariableParameterizedTypes = new HashMap<>(); + // Column names get the highest priority. + final BiConsumer> processColumn = (columnName, column) -> { + if (!columnVariables.add(columnName)) { + // this column was renamed + return; + } + + possibleVariables.put(columnName, column.getDataType()); + + final Class compType = column.getComponentType(); + if (compType != null && !compType.isPrimitive()) { + possibleVariableParameterizedTypes.put(columnName, new Class[] {compType}); + } + }; + + // Renames trump the original columns; so they go first. + for (Map.Entry columnRename : columnRenames.entrySet()) { + final String columnName = columnRename.getKey(); + final ColumnDefinition column = availableColumns.get(columnRename.getValue()); + processColumn.accept(columnName, column); + } + + // Now process the original columns. for (ColumnDefinition columnDefinition : availableColumns.values()) { - // add column-vectors - final String columnSuffix = DhFormulaColumn.COLUMN_SUFFIX; - final Class vectorType = DhFormulaColumn.getVectorType(columnDefinition.getDataType()); + processColumn.accept(columnDefinition.getName(), columnDefinition); + } - possibleVariables.put(columnDefinition.getName() + columnSuffix, vectorType); - columnVariables.add(columnDefinition.getName() + columnSuffix); + // Column arrays come between columns and parameters. + final BiConsumer> processColumnArray = (columnName, column) -> { + final String columnArrayName = columnName + COLUMN_SUFFIX; - if (vectorType == ObjectVector.class) { - possibleVariableParameterizedTypes.put(columnDefinition.getName() + columnSuffix, - new Class[] {columnDefinition.getDataType()}); + if (!columnVariables.add(columnArrayName)) { + // Either this is a rename or overloads an existing column name. + return; } - // add columns - columnVariables.add(columnDefinition.getName()); - possibleVariables.put(columnDefinition.getName(), columnDefinition.getDataType()); - final Class compType = columnDefinition.getComponentType(); - if (compType != null && !compType.isPrimitive()) { - possibleVariableParameterizedTypes.put(columnDefinition.getName(), new Class[] {compType}); + final Class vectorType = DhFormulaColumn.getVectorType(column.getDataType()); + possibleVariables.put(columnArrayName, vectorType); + + if (vectorType == ObjectVector.class) { + possibleVariableParameterizedTypes.put(columnArrayName, new Class[] {column.getDataType()}); } + }; + + // Renames still trump the original columns; so they go first. + for (Map.Entry columnRename : columnRenames.entrySet()) { + final String columnName = columnRename.getKey(); + final ColumnDefinition column = availableColumns.get(columnRename.getValue()); + processColumnArray.accept(columnName, column); } - final ExecutionContext context = ExecutionContext.getContext(); - final Map queryScopeVariables = context.getQueryScope().toMap( - (name, value) -> NameValidator.isValidQueryParameterName(name)); + // Now process the original columns. + for (ColumnDefinition columnDefinition : availableColumns.values()) { + processColumnArray.accept(columnDefinition.getName(), columnDefinition); + } + + // Parameters come last. for (Map.Entry param : queryScopeVariables.entrySet()) { if (possibleVariables.containsKey(param.getKey())) { - // skip any existing matches + // Columns and column arrays take precedence over parameters. continue; } @@ -124,22 +196,18 @@ public static QueryLanguageParser.Result getCompiledFormula(Map> classImports = - new HashSet<>(context.getQueryLibrary().getClassImports()); + final QueryLibrary queryLibrary = ExecutionContext.getContext().getQueryLibrary(); + final Set> classImports = new HashSet<>(queryLibrary.getClassImports()); classImports.add(TrackingWritableRowSet.class); classImports.add(WritableColumnSource.class); - return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), - context.getQueryLibrary().getPackageImports(), - classImports, context.getQueryLibrary().getStaticImports(), possibleVariables, - possibleVariableParameterizedTypes, queryScopeVariables, columnVariables) - .getResult(); + return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), queryLibrary.getPackageImports(), + classImports, queryLibrary.getStaticImports(), possibleVariables, possibleVariableParameterizedTypes, + queryScopeVariables, columnVariables, unboxArguments, timeConversionResult).getResult(); } public static class Result { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java index e46f6edb9f4..6ad2eb4d31e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java @@ -3,12 +3,13 @@ // package io.deephaven.engine.table.impl.select.codegen; -import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.util.SafeCloseable; +import io.deephaven.engine.context.QueryCompilerImpl; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.context.QueryScopeParam; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.DhFormulaColumn; import io.deephaven.engine.table.impl.select.FormulaCompilationException; @@ -30,21 +31,38 @@ public class JavaKernelBuilder { private static final String FORMULA_KERNEL_FACTORY_NAME = "__FORMULA_KERNEL_FACTORY"; - public static Result create(String cookedFormulaString, Class returnedType, String timeInstanceVariables, - Map columns, Map> arrays, Map> params) { - final JavaKernelBuilder jkf = new JavaKernelBuilder(cookedFormulaString, returnedType, timeInstanceVariables, - columns, arrays, params); + public static CompletionStageFuture create( + @NotNull final String originalFormulaString, + @NotNull final String cookedFormulaString, + @NotNull final Class returnedType, + @NotNull final String timeInstanceVariables, + @NotNull final Map columns, + @NotNull final Map> arrays, + @NotNull final Map> params, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + final JavaKernelBuilder jkf = new JavaKernelBuilder( + originalFormulaString, cookedFormulaString, returnedType, timeInstanceVariables, columns, arrays, + params); final String classBody = jkf.generateKernelClassBody(); - final Class clazz = compileFormula(cookedFormulaString, classBody, "Formula"); - final FormulaKernelFactory fkf; - try { - fkf = (FormulaKernelFactory) clazz.getField(FORMULA_KERNEL_FACTORY_NAME).get(null); - } catch (ReflectiveOperationException e) { - throw new FormulaCompilationException("Formula compilation error for: " + cookedFormulaString, e); - } - return new Result(classBody, clazz, fkf); + + return compilationRequestProcessor.submit(QueryCompilerRequest.builder() + .description("FormulaKernel: " + originalFormulaString) + .className("Formula") + .classBody(classBody) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) + .build()).thenApply(clazz -> { + final FormulaKernelFactory fkf; + try { + fkf = (FormulaKernelFactory) clazz.getField(FORMULA_KERNEL_FACTORY_NAME).get(null); + } catch (ReflectiveOperationException e) { + throw new FormulaCompilationException( + "Formula compilation error for: " + cookedFormulaString, e); + } + return new Result(classBody, clazz, fkf); + }); } + private final String originalFormulaString; private final String cookedFormulaString; private final Class returnedType; private final String timeInstanceVariables; @@ -62,8 +80,15 @@ public static Result create(String cookedFormulaString, Class returnedType, S */ private final Map> params; - private JavaKernelBuilder(String cookedFormulaString, Class returnedType, String timeInstanceVariables, - Map columns, Map> arrays, Map> params) { + private JavaKernelBuilder( + @NotNull final String originalFormulaString, + @NotNull final String cookedFormulaString, + @NotNull final Class returnedType, + @NotNull final String timeInstanceVariables, + @NotNull final Map columns, + @NotNull final Map> arrays, + @NotNull final Map> params) { + this.originalFormulaString = originalFormulaString; this.cookedFormulaString = cookedFormulaString; this.returnedType = returnedType; this.timeInstanceVariables = timeInstanceVariables; @@ -214,7 +239,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(cookedFormulaString)); - final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(cookedFormulaString); + final String joinedFormulaString = QueryCompilerImpl.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", FormulaEvaluationException.class.getCanonicalName()); return g.freeze(); @@ -257,23 +282,16 @@ private List visitFormulaParameters( return results; } - @SuppressWarnings("SameParameterValue") - private static Class compileFormula(final String what, final String classBody, final String className) { - // System.out.printf("compileFormula: formulaString is %s. Code is...%n%s%n", what, classBody); - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(what)) { - // Compilation needs to take place with elevated privileges, but the created object should not have them. - final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); - return compiler.compile(className, classBody, QueryCompiler.FORMULA_PREFIX); - } - } - public static class Result { public final String classBody; public final Class clazz; public final FormulaKernelFactory formulaKernelFactory; - public Result(String classBody, Class clazz, FormulaKernelFactory formulaKernelFactory) { + public Result( + @NotNull final String classBody, + @NotNull final Class clazz, + @NotNull final FormulaKernelFactory formulaKernelFactory) { this.classBody = classBody; this.clazz = clazz; this.formulaKernelFactory = formulaKernelFactory; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java index 3fa8eb25bf5..8c32326dc5d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java @@ -12,6 +12,7 @@ public interface FormulaFactory { Formula createFormula( + String columnName, TrackingRowSet rowSet, boolean initLazyMap, Map columnsToData, QueryScopeParam... params); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java index 2b181753891..298abcf1702 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java @@ -5,12 +5,14 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.context.QueryScopeParam; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.table.impl.select.AbstractFormulaColumn; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.formula.FormulaKernel; import io.deephaven.engine.table.impl.select.formula.FormulaKernelFactory; import io.deephaven.engine.table.impl.select.formula.FormulaSourceDescriptor; +import org.jetbrains.annotations.NotNull; import java.util.LinkedHashSet; import java.util.List; @@ -25,8 +27,7 @@ public class FormulaColumnPython extends AbstractFormulaColumn implements FormulaKernelFactory { @SuppressWarnings("unused") // called from python - public static FormulaColumnPython create(String columnName, - DeephavenCompatibleFunction dcf) { + public static FormulaColumnPython create(String columnName, DeephavenCompatibleFunction dcf) { return new FormulaColumnPython(columnName, dcf); } @@ -39,13 +40,13 @@ private FormulaColumnPython(String columnName, } @Override - public final List initDef(Map> columnDefinitionMap) { - if (formulaFactory != null) { + public final List initDef(@NotNull final Map> columnDefinitionMap) { + if (formulaFactoryFuture != null) { validateColumnDefinition(columnDefinitionMap); } else { returnedType = dcf.getReturnedType(); applyUsedVariables(columnDefinitionMap, new LinkedHashSet<>(dcf.getColumnNames()), Map.of()); - formulaFactory = createKernelFormulaFactory(this); + formulaFactoryFuture = createKernelFormulaFactory(CompletionStageFuture.completedFuture(this)); } return usedColumns; @@ -69,7 +70,7 @@ protected final FormulaSourceDescriptor getSourceDescriptor() { @Override public final SelectColumn copy() { final FormulaColumnPython copy = new FormulaColumnPython(columnName, dcf); - if (formulaFactory != null) { + if (formulaFactoryFuture != null) { // copy all initDef state copy.returnedType = returnedType; onCopy(copy); @@ -79,7 +80,7 @@ public final SelectColumn copy() { @Override public final FormulaKernel createInstance(Vector[] arrays, QueryScopeParam[] params) { - if (formulaFactory == null) { + if (formulaFactoryFuture == null) { throw new IllegalStateException("Must be initialized first"); } return dcf.toFormulaKernel(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java index 45e594997ce..6e551476520 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java @@ -12,6 +12,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.updateby.delta.*; import io.deephaven.engine.table.impl.updateby.em.*; @@ -85,7 +86,9 @@ final Collection getOutputColumns(@NotNull final Collection getOperators(@NotNull final Collection specs) { - final OperationVisitor v = new OperationVisitor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); + + final OperationVisitor v = new OperationVisitor(compilationProcessor); specs.forEach(s -> s.walk(v)); // Do we have a combined rolling group operator to create? @@ -93,6 +96,8 @@ final Collection getOperators(@NotNull final Collection, UpdateByOperation.Visitor { + private final QueryCompilerRequestProcessor compilationProcessor; private final List ops = new ArrayList<>(); private MatchPair[] pairs; @@ -289,6 +295,11 @@ private class OperationVisitor implements UpdateBySpec.Visitor, UpdateByOp RollingGroupSpec rollingGroupSpec; MatchPair[] rollingGroupPairs; + OperationVisitor( + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + this.compilationProcessor = compilationProcessor; + } + /** * Check if the supplied type is one of the supported time types. * @@ -1364,47 +1375,47 @@ private UpdateByOperator makeRollingFormulaOperator(@NotNull final MatchPair pai return new BooleanRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == byte.class || csType == Byte.class) { return new ByteRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == char.class || csType == Character.class) { return new CharRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == short.class || csType == Short.class) { return new ShortRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == int.class || csType == Integer.class) { return new IntRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == long.class || csType == Long.class) { return new LongRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == float.class || csType == Float.class) { return new FloatRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == double.class || csType == Double.class) { return new DoubleRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, compilationProcessor); } return new ObjectRollingFormulaOperator<>(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, compilationProcessor); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java index 32c662fe872..b5e0a9560a9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java @@ -10,6 +10,7 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.select.FormulaUtil; import io.deephaven.engine.table.impl.sources.ArrayBackedColumnSource; @@ -95,7 +96,8 @@ public BaseRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef) { + @NotNull final TableDefinition tableDef, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, true); this.formulaColumnMap = formulaColumnMap; this.tableDef = tableDef; @@ -113,7 +115,7 @@ public BaseRollingFormulaOperator( final ColumnDefinition inputColumnDefinition = ColumnDefinition .fromGenericType(PARAM_COLUMN_NAME, inputVectorType, inputColumnType); - tmp.initDef(Collections.singletonMap(PARAM_COLUMN_NAME, inputColumnDefinition)); + tmp.initDef(Collections.singletonMap(PARAM_COLUMN_NAME, inputColumnDefinition), compilationProcessor); return tmp; }); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java index b463a44a0f1..b54d4dcc02d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; import io.deephaven.engine.table.impl.updateby.UpdateByOperator; @@ -160,9 +161,10 @@ public BooleanRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef) { + @NotNull final TableDefinition tableDef, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, compilationProcessor); } protected BooleanRollingFormulaOperator( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java index af6029a58aa..4cb47097afa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -163,12 +164,13 @@ public ByteRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java index 205ee8f2bfa..7018c431482 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -159,12 +160,13 @@ public CharRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java index 5bc233a533d..bb0c449e394 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -163,12 +164,13 @@ public DoubleRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java index 459fa465bff..66a65898978 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -163,12 +164,13 @@ public FloatRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java index 194d1dc7063..c0b07b52af9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -163,12 +164,13 @@ public IntRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java index 1171ffccaf6..9b3eb63de7b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -163,12 +164,13 @@ public LongRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java index 849a7a1de28..8b062025e52 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; import io.deephaven.engine.table.impl.updateby.UpdateByOperator; @@ -159,9 +160,10 @@ public ObjectRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef) { + @NotNull final TableDefinition tableDef, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, compilationProcessor); } protected ObjectRollingFormulaOperator( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java index c0a6978a129..7cce65c7cb5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -163,12 +164,13 @@ public ShortRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java index 26388fe5546..51640f22d05 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java @@ -41,6 +41,14 @@ public abstract class AbstractScriptSession Supplier compileSimpleFunction(final Class res public static Supplier compileSimpleStatement(final Class resultType, final String code, final String... imports) { - final List importClasses = new ArrayList<>(); + final List> importClasses = new ArrayList<>(); for (final String importString : imports) { try { importClasses.add(Class.forName(importString)); @@ -33,14 +34,14 @@ public static Supplier compileSimpleStatement(final Class re } public static Supplier compileSimpleFunction(final Class resultType, final String code, - final Collection imports, final Collection staticImports) { + final Collection> imports, final Collection> staticImports) { final StringBuilder classBody = new StringBuilder(); classBody.append("import ").append(resultType.getName()).append(";\n"); - for (final Class im : imports) { + for (final Class im : imports) { classBody.append("import ").append(im.getName()).append(";\n"); } - for (final Class sim : staticImports) { + for (final Class sim : staticImports) { classBody.append("import static ").append(sim.getName()).append(".*;\n"); } @@ -52,8 +53,13 @@ public static Supplier compileSimpleFunction(final Class res classBody.append(" }\n"); classBody.append("}\n"); - final Class partitionClass = ExecutionContext.getContext().getQueryCompiler() - .compile("Function", classBody.toString(), QueryCompiler.FORMULA_PREFIX); + final Class partitionClass = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Simple Function: " + code) + .className("Function") + .classBody(classBody.toString()) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) + .build()); try { // noinspection unchecked @@ -63,7 +69,7 @@ public static Supplier compileSimpleFunction(final Class res } } - public static Class getClassThroughCompilation(final String object) { + public static Class getClassThroughCompilation(final String object) { final StringBuilder classBody = new StringBuilder(); classBody.append("public class $CLASSNAME$ implements ").append(Supplier.class.getCanonicalName()) .append("{ \n"); @@ -71,12 +77,17 @@ public static Class getClassThroughCompilation(final String object) { classBody.append(" public Class get() { return ").append(object).append(".class; }\n"); classBody.append("}\n"); - final Class partitionClass = ExecutionContext.getContext().getQueryCompiler() - .compile("Function", classBody.toString(), QueryCompiler.FORMULA_PREFIX); + final Class partitionClass = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Formula: return " + object + ".class") + .className("Function") + .classBody(classBody.toString()) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) + .build()); try { // noinspection unchecked - return ((Supplier) partitionClass.newInstance()).get(); + return ((Supplier>) partitionClass.newInstance()).get(); } catch (InstantiationException | IllegalAccessException e) { throw new RuntimeException("Could not instantiate function.", e); } diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 83a3694c410..e6291a639e9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -37,6 +37,7 @@ import io.deephaven.time.DateTimeUtils; import io.deephaven.time.calendar.StaticCalendarMethods; import io.deephaven.util.QueryConstants; +import io.deephaven.util.SafeCloseable; import io.deephaven.util.annotations.VisibleForTesting; import io.deephaven.util.type.ArrayTypeUtils; import io.deephaven.util.type.TypeUtils; @@ -51,7 +52,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import javax.tools.JavaFileObject; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -60,8 +60,6 @@ import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; @@ -69,6 +67,7 @@ import java.time.ZonedDateTime; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -82,11 +81,12 @@ public class GroovyDeephavenSession extends AbstractScriptSession loadClass(String name, boolean resolve) throws ClassNotFoundE private final ScriptFinder scriptFinder; /** Contains imports to be applied to commands run in the console */ - private final ImportCustomizer consoleImports = new ImportCustomizer(); + private final ImportCustomizer consoleImports; /** Contains imports to be applied to .groovy files loaded from the classpath */ - private final ImportCustomizer loadedGroovyScriptImports = new ImportCustomizer(); + private final ImportCustomizer loadedGroovyScriptImports; - private final Set dynamicClasses = new HashSet<>(); - private final Map bindingBackingMap = Collections.synchronizedMap(new LinkedHashMap<>()); - private final GroovyShell groovyShell; + private final Set dynamicClasses; + private final Map bindingBackingMap; - private int counter; - private String script = "Script"; + private static class DeephavenGroovyShell extends GroovyShell { + private final AtomicInteger counter = new AtomicInteger(); + private volatile String scriptPrefix = DEFAULT_SCRIPT_PREFIX; - private String getNextScriptClassName() { - return script + "_" + (counter + 1); + DeephavenGroovyShell( + final GroovyClassLoader loader, + final Binding binding, + final CompilerConfiguration config) { + super(loader, binding, config); + } + + @Override + protected String generateScriptName() { + return scriptPrefix + "_" + (counter.incrementAndGet()) + ".groovy"; + } + + private String getNextScriptClassName() { + return scriptPrefix + "_" + (counter.get() + 1); + } + + public SafeCloseable setScriptPrefix(final String newPrefix) { + scriptPrefix = newPrefix; + return () -> { + scriptPrefix = newPrefix; + }; + } } - public GroovyDeephavenSession( + private final DeephavenGroovyShell groovyShell; + + public static GroovyDeephavenSession of( final UpdateGraph updateGraph, final OperationInitializer operationInitializer, final ObjectTypeLookup objectTypeLookup, final RunScripts runScripts) throws IOException { - this(updateGraph, operationInitializer, objectTypeLookup, null, runScripts); + return GroovyDeephavenSession.of(updateGraph, operationInitializer, objectTypeLookup, null, runScripts); } - public GroovyDeephavenSession( + public static GroovyDeephavenSession of( final UpdateGraph updateGraph, final OperationInitializer operationInitializer, ObjectTypeLookup objectTypeLookup, @Nullable final Listener changeListener, - final RunScripts runScripts) - throws IOException { - super(updateGraph, operationInitializer, objectTypeLookup, changeListener); + final RunScripts runScripts) throws IOException { + + final ImportCustomizer consoleImports = new ImportCustomizer(); + final ImportCustomizer loadedGroovyScriptImports = new ImportCustomizer(); addDefaultImports(consoleImports); if (INCLUDE_DEFAULT_IMPORTS_IN_LOADED_GROOVY) { - addDefaultImports(this.loadedGroovyScriptImports); + addDefaultImports(loadedGroovyScriptImports); } + final File classCacheDirectory = AbstractScriptSession.newClassCacheLocation(); + // Specify a classloader to read from the classpath, with script imports CompilerConfiguration scriptConfig = new CompilerConfiguration(); scriptConfig.getCompilationCustomizers().add(loadedGroovyScriptImports); - scriptConfig.setTargetDirectory(executionContext.getQueryCompiler().getFakeClassDestination()); + scriptConfig.setTargetDirectory(classCacheDirectory); GroovyClassLoader scriptClassLoader = new GroovyClassLoader(STATIC_LOADER, scriptConfig); // Specify a configuration for compiling/running console commands for custom imports CompilerConfiguration consoleConfig = new CompilerConfiguration(); consoleConfig.getCompilationCustomizers().add(consoleImports); - consoleConfig.setTargetDirectory(executionContext.getQueryCompiler().getFakeClassDestination()); + consoleConfig.setTargetDirectory(classCacheDirectory); + Map bindingBackingMap = Collections.synchronizedMap(new LinkedHashMap<>()); Binding binding = new Binding(bindingBackingMap); - groovyShell = new GroovyShell(scriptClassLoader, binding, consoleConfig) { - protected synchronized String generateScriptName() { - return GroovyDeephavenSession.this.generateScriptName(); - } - }; + DeephavenGroovyShell groovyShell = new DeephavenGroovyShell(scriptClassLoader, binding, consoleConfig); + + + return new GroovyDeephavenSession(updateGraph, operationInitializer, objectTypeLookup, changeListener, + runScripts, classCacheDirectory, consoleImports, loadedGroovyScriptImports, bindingBackingMap, + groovyShell); + } + + private GroovyDeephavenSession( + final UpdateGraph updateGraph, + final OperationInitializer operationInitializer, + ObjectTypeLookup objectTypeLookup, + @Nullable final Listener changeListener, + final RunScripts runScripts, + final File classCacheDirectory, + final ImportCustomizer consoleImports, + final ImportCustomizer loadedGroovyScriptImports, + final Map bindingBackingMap, + final DeephavenGroovyShell groovyShell) + throws IOException { + super(updateGraph, operationInitializer, objectTypeLookup, changeListener, classCacheDirectory, + groovyShell.getClassLoader()); this.scriptFinder = new ScriptFinder(DEFAULT_SCRIPT_PATH); + this.consoleImports = consoleImports; + this.loadedGroovyScriptImports = loadedGroovyScriptImports; + + this.dynamicClasses = new HashSet<>(); + this.bindingBackingMap = bindingBackingMap; + + this.groovyShell = groovyShell; + groovyShell.setVariable("__groovySession", this); groovyShell.setVariable("DB_SCRIPT_PATH", DEFAULT_SCRIPT_PATH); - executionContext.getQueryCompiler().setParentClassLoader(getShell().getClassLoader()); - publishInitial(); for (final String path : runScripts.paths) { @@ -199,7 +248,7 @@ protected synchronized String generateScriptName() { /** * Adds the default imports that Groovy users assume to be present. */ - private void addDefaultImports(ImportCustomizer imports) { + private static void addDefaultImports(ImportCustomizer imports) { // TODO (core#230): Remove large list of manual text-based consoleImports // NOTE: Don't add to this list without a compelling reason!!! Use the user script import if possible. imports.addImports( @@ -244,10 +293,6 @@ private void addDefaultImports(ImportCustomizer imports) { StaticCalendarMethods.class.getName()); } - private String generateScriptName() { - return script + "_" + (++counter) + ".groovy"; - } - public static InputStream findScript(String relativePath) throws IOException { return new ScriptFinder(DEFAULT_SCRIPT_PATH).findScript(relativePath); } @@ -287,10 +332,6 @@ protected T getVariable(String name) { } } - private void evaluateCommand(String command) { - groovyShell.evaluate(command); - } - @Override protected void evaluate(String command, String scriptName) { grepScriptImports(removeComments(command)); @@ -299,27 +340,22 @@ protected void evaluate(String command, String scriptName) { final String lastCommand = fc.second; final String commandPrefix = fc.first; - final String oldScriptName = script; - - try { - if (scriptName != null) { - script = scriptName.replaceAll("[^0-9A-Za-z_]", "_").replaceAll("(^[0-9])", "_$1"); - } - final String currentScriptName = script; + final String currentScriptName = scriptName == null + ? DEFAULT_SCRIPT_PREFIX + : scriptName.replaceAll("[^0-9A-Za-z_]", "_").replaceAll("(^[0-9])", "_$1"); + try (final SafeCloseable ignored = groovyShell.setScriptPrefix(currentScriptName)) { updateClassloader(lastCommand); try { ExecutionContext.getContext().getUpdateGraph().exclusiveLock() - .doLockedInterruptibly(() -> evaluateCommand(lastCommand)); + .doLockedInterruptibly(() -> groovyShell.evaluate(lastCommand)); } catch (InterruptedException e) { throw new CancellationException(e.getMessage() != null ? e.getMessage() : "Query interrupted", maybeRewriteStackTrace(scriptName, currentScriptName, e, lastCommand, commandPrefix)); } catch (Exception e) { throw wrapAndRewriteStackTrace(scriptName, currentScriptName, e, lastCommand, commandPrefix); } - } finally { - script = oldScriptName; } } @@ -617,25 +653,11 @@ private Pair fullCommand(String command) { + "\n\n// this final true prevents Groovy from interpreting a trailing class definition as something to execute\n;\ntrue;\n"); } - public static byte[] getDynamicClass(String name) { - return readClass(ExecutionContext.getContext().getQueryCompiler().getFakeClassDestination(), name); - } - - private static byte[] readClass(final File rootDirectory, final String className) { - final String resourceName = className.replace('.', '/') + JavaFileObject.Kind.CLASS.extension; - final Path path = new File(rootDirectory, resourceName).toPath(); - try { - return Files.readAllBytes(path); - } catch (IOException e) { - throw new RuntimeException("Error reading path " + path + " for className " + className, e); - } - } - private void updateClassloader(String currentCommand) { - final String name = getNextScriptClassName(); + final String name = groovyShell.getNextScriptClassName(); CompilerConfiguration config = new CompilerConfiguration(CompilerConfiguration.DEFAULT); - config.setTargetDirectory(executionContext.getQueryCompiler().getFakeClassDestination()); + config.setTargetDirectory(classCacheDirectory); config.getCompilationCustomizers().add(consoleImports); final CompilationUnit cu = new CompilationUnit(config, null, groovyShell.getClassLoader()); cu.addSource(name, currentCommand); @@ -645,8 +667,7 @@ private void updateClassloader(String currentCommand) { } catch (RuntimeException e) { throw new GroovyExceptionWrapper(e); } - final File dynamicClassDestination = ExecutionContext.getContext().getQueryCompiler().getFakeClassDestination(); - if (dynamicClassDestination == null) { + if (classCacheDirectory == null) { return; } final List classes = cu.getClasses(); @@ -672,7 +693,7 @@ && isAnInteger(aClass.getName().substring(SCRIPT_PREFIX.length()))) { } try { - QueryCompiler.writeClass(dynamicClassDestination, entry.getKey(), entry.getValue()); + QueryCompilerImpl.writeClass(classCacheDirectory, entry.getKey(), entry.getValue()); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/FuzzerTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/FuzzerTest.java index b5327549923..e673093a736 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/FuzzerTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/FuzzerTest.java @@ -76,7 +76,7 @@ private GroovyDeephavenSession getGroovySession() throws IOException { } private GroovyDeephavenSession getGroovySession(@Nullable Clock clock) throws IOException { - final GroovyDeephavenSession session = new GroovyDeephavenSession( + final GroovyDeephavenSession session = GroovyDeephavenSession.of( ExecutionContext.getContext().getUpdateGraph(), ExecutionContext.getContext().getOperationInitializer(), NoOp.INSTANCE, diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java index df1467197ac..65b27a6b380 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java @@ -57,7 +57,7 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java index 2fd1498e064..18fec0ccca1 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java @@ -4,16 +4,17 @@ package io.deephaven.engine.table.impl.lang; import groovy.lang.Closure; -import io.deephaven.api.util.NameValidator; import io.deephaven.base.Pair; import io.deephaven.base.testing.BaseArrayTestCase; import io.deephaven.base.verify.Assert; import io.deephaven.base.verify.Require; import io.deephaven.engine.context.*; import io.deephaven.engine.table.Table; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser.QueryLanguageParseException; import io.deephaven.engine.testutil.ControlledUpdateGraph; import io.deephaven.engine.util.PyCallableWrapper; +import io.deephaven.time.TimeLiteralReplacedExpression; import io.deephaven.util.QueryConstants; import io.deephaven.util.SafeCloseable; import io.deephaven.util.type.TypeUtils; @@ -3178,7 +3179,7 @@ private void check(String expression, String resultExpression, Class resultTy final Map possibleParams; final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); if (!(queryScope instanceof PoisonedQueryScope)) { - possibleParams = queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name)); + possibleParams = QueryCompilerRequestProcessor.newQueryScopeVariableSupplier().get(); } else { possibleParams = null; } @@ -3188,7 +3189,7 @@ private void check(String expression, String resultExpression, Class resultTy variables, variableParameterizedTypes, possibleParams, null, true, verifyIdempotence, - PyCallableWrapperDummyImpl.class.getName()).getResult(); + PyCallableWrapperDummyImpl.class.getName(), null).getResult(); assertEquals(resultType, result.getType()); assertEquals(resultExpression, result.getConvertedExpression()); diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java index 505bad205b0..d6a5c6abcc1 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java @@ -106,7 +106,7 @@ private long applyFormulaPerItem(long II, long ii, int I, int i) { try { return plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1)))); } catch (java.lang.Exception __e) { - throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: " + "plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1))))", __e); + throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: " + "I * II + q * ii + II_[i - 1]", __e); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java index 897f8697869..f24fd283dbc 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java @@ -73,6 +73,7 @@ public class FormulaSample extends io.deephaven.engine.table.impl.select.Formula { public static final io.deephaven.engine.table.impl.select.formula.FormulaFactory __FORMULA_FACTORY = FormulaSample::new; + private final String __columnName; private final io.deephaven.engine.table.ColumnSource II; private final io.deephaven.engine.table.ColumnSource I; private final io.deephaven.vector.LongVector II_; @@ -80,11 +81,13 @@ public class FormulaSample extends io.deephaven.engine.table.impl.select.Formula private final Map __lazyResultCache; - public FormulaSample(final TrackingRowSet __rowSet, + public FormulaSample(final String __columnName, + final TrackingRowSet __rowSet, final boolean __lazy, final java.util.Map __columnsToData, final io.deephaven.engine.context.QueryScopeParam... __params) { super(__rowSet); + this.__columnName = __columnName; II = __columnsToData.get("II"); I = __columnsToData.get("I"); II_ = new io.deephaven.engine.table.impl.vector.LongVectorColumnWrapper(__columnsToData.get("II"), __rowSet); @@ -190,7 +193,7 @@ private long applyFormulaPerItem(int i, long ii, long II, int I) { try { return plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1)))); } catch (java.lang.Exception __e) { - throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: Value = " + "plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1))))", __e); + throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: " + __columnName + " = " + "I * II + q * ii + II_[i - 1]", __e); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java b/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java index 206b6c73f16..b700c4ed006 100644 --- a/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java +++ b/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java @@ -7,6 +7,7 @@ import io.deephaven.configuration.DataDir; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.TrackingRowSet; @@ -102,7 +103,7 @@ private QueryCompiler compilerForUnitTests() { final Path queryCompilerDir = DataDir.get() .resolve("io.deephaven.engine.updategraph.impl.TestEventDrivenUpdateGraph.compilerForUnitTests"); - return QueryCompiler.create(queryCompilerDir.toFile(), getClass().getClassLoader()); + return QueryCompilerImpl.create(queryCompilerDir.toFile(), getClass().getClassLoader()); } @Test diff --git a/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java b/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java index a2e1a5c71b7..0522779ed2c 100644 --- a/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java +++ b/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java @@ -18,7 +18,7 @@ public void testNotString() { DynamicCompileUtils.compileSimpleFunction(String.class, "return 7"); TestCase.fail("Should never have reached this statement."); } catch (RuntimeException e) { - TestCase.assertTrue(e.getMessage().contains("int cannot be converted to String")); + TestCase.assertTrue(e.getMessage().contains("int cannot be converted to java.lang.String")); } } } diff --git a/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java b/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java index 365c4505eb7..ae9f80e17da 100644 --- a/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java +++ b/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java @@ -16,7 +16,6 @@ import io.deephaven.function.Sort; import io.deephaven.plugin.type.ObjectTypeLookup.NoOp; import io.deephaven.util.SafeCloseable; -import io.deephaven.util.thread.ThreadInitializationFactory; import org.apache.commons.lang3.mutable.MutableInt; import org.junit.After; import org.junit.Assert; @@ -50,8 +49,8 @@ public void setup() throws IOException { livenessScope = new LivenessScope(); LivenessScopeStack.push(livenessScope); final ExecutionContext context = ExecutionContext.getContext(); - session = new GroovyDeephavenSession( - context.getUpdateGraph(), context.getOperationInitializer(), NoOp.INSTANCE, null, + session = GroovyDeephavenSession.of( + context.getUpdateGraph(), context.getOperationInitializer(), NoOp.INSTANCE, GroovyDeephavenSession.RunScripts.none()); executionContext = session.getExecutionContext().open(); } diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java index 39ac93d077e..eb4bd23c3eb 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java @@ -20,7 +20,7 @@ public static ExecutionContext createForUnitTests() { .markSystemic() .newQueryScope() .newQueryLibrary() - .setQueryCompiler(QueryCompiler.createForUnitTests()) + .setQueryCompiler(QueryCompilerImpl.createForUnitTests()) .setUpdateGraph(UPDATE_GRAPH) .setOperationInitializer(OPERATION_INITIALIZATION) .build(); diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java index a54c3357c55..d689b823ae6 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java @@ -7,7 +7,7 @@ import io.deephaven.chunk.util.pools.ChunkPoolReleaseTracking; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.TestExecutionContext; import io.deephaven.engine.liveness.LivenessScope; import io.deephaven.engine.liveness.LivenessScopeStack; @@ -70,7 +70,7 @@ public void setUp() throws Exception { errors = null; livenessScopeCloseable = LivenessScopeStack.open(new LivenessScope(true), true); - oldLogEnabled = QueryCompiler.setLogEnabled(ENABLE_QUERY_COMPILER_LOGGING); + oldLogEnabled = QueryCompilerImpl.setLogEnabled(ENABLE_QUERY_COMPILER_LOGGING); oldSerialSafe = updateGraph.setSerialTableOperationsSafe(true); AsyncErrorLogger.init(); ChunkPoolReleaseTracking.enableStrict(); @@ -81,7 +81,7 @@ public void tearDown() throws Exception { ChunkPoolReleaseTracking.checkAndDisable(); final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast(); updateGraph.setSerialTableOperationsSafe(oldSerialSafe); - QueryCompiler.setLogEnabled(oldLogEnabled); + QueryCompilerImpl.setLogEnabled(oldLogEnabled); // reset the execution context executionContext.close(); diff --git a/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java b/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java index 521314bf70c..646f4862707 100644 --- a/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java +++ b/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java @@ -8,8 +8,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static io.deephaven.util.QueryConstants.NULL_LONG; - // TODO: Move? /** diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java index fe44da7d22b..df4339624ff 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java @@ -9,7 +9,7 @@ import io.deephaven.parquet.compress.CompressorAdapter; import io.deephaven.parquet.compress.DeephavenCompressorAdapterFactory; import io.deephaven.util.channel.SeekableChannelContext.ContextHolder; -import io.deephaven.util.datastructures.LazyCachingFunction; +import io.deephaven.util.datastructures.SoftCachingFunction; import org.apache.commons.io.FilenameUtils; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.column.ColumnDescriptor; @@ -71,7 +71,7 @@ final class ColumnChunkReaderImpl implements ColumnChunkReader { decompressor = CompressorAdapter.PASSTHRU; } this.fieldTypes = fieldTypes; - this.dictionarySupplier = new LazyCachingFunction<>(this::getDictionary); + this.dictionarySupplier = new SoftCachingFunction<>(this::getDictionary); this.nullMaterializerFactory = PageMaterializer.factoryForType(path.getPrimitiveType().getPrimitiveTypeName()); this.numRows = numRows; this.version = version; diff --git a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java index 57f01185faa..2fbef5447c8 100644 --- a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java +++ b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java @@ -5,7 +5,7 @@ import io.deephaven.base.FileUtils; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.table.Table; @@ -51,7 +51,7 @@ public void setupEnv() throws IOException { .newQueryLibrary() .newQueryScope() .setQueryCompiler( - QueryCompiler.create(rootPath.resolve("cache").toFile(), getClass().getClassLoader())) + QueryCompilerImpl.create(rootPath.resolve("cache").toFile(), getClass().getClassLoader())) .build(); exContextCloseable = context.open(); diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ChunkDictionary.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ChunkDictionary.java index 56bccb4e4a0..17e83139c31 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ChunkDictionary.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ChunkDictionary.java @@ -9,7 +9,7 @@ import io.deephaven.util.channel.SeekableChannelContext; import io.deephaven.stringset.LongBitmapStringSet; import io.deephaven.chunk.ObjectChunk; -import io.deephaven.util.datastructures.LazyCachingSupplier; +import io.deephaven.util.datastructures.SoftCachingSupplier; import org.apache.parquet.column.Dictionary; import org.jetbrains.annotations.NotNull; @@ -47,7 +47,7 @@ public interface Lookup { ChunkDictionary( @NotNull final Lookup lookup, @NotNull final Function dictionarySupplier) { - this.valuesSupplier = new LazyCachingSupplier<>(() -> { + this.valuesSupplier = new SoftCachingSupplier<>(() -> { // We use NULL channel context here and rely on materialization logic to provide the correct context final Dictionary dictionary = dictionarySupplier.apply(SeekableChannelContext.NULL); final T[] values = ObjectChunk.makeArray(dictionary.getMaxId() + 1); @@ -56,7 +56,7 @@ public interface Lookup { } return ObjectChunk.chunkWrap(values); }); - this.reverseMapSupplier = new LazyCachingSupplier<>(() -> { + this.reverseMapSupplier = new SoftCachingSupplier<>(() -> { final ObjectChunk values = getChunk(); final TObjectIntMap reverseMap = new TObjectIntHashMap<>(values.size()); for (int vi = 0; vi < values.size(); ++vi) { diff --git a/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java b/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java index dd9d19d6626..0d8c176051f 100644 --- a/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java +++ b/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java @@ -337,15 +337,13 @@ public static TreeTable queryPerformanceAsTreeTable(@NotNull final Table qpl) { public static TreeTable queryOperationPerformanceAsTreeTable( @NotNull final Table qpl, @NotNull final Table qopl) { - // TODO (https://github.com/deephaven/deephaven-core/issues/4814): use NULL_INT for ParentOperationNumber and - // Depth once we can prevent any compilation or at least reduce multiple usages to a single formula Table mergeWithAggKeys = TableTools.merge( qpl.updateView( "EvalKey = Long.toString(EvaluationNumber)", "ParentEvalKey = ParentEvaluationNumber == null ? null : (Long.toString(ParentEvaluationNumber))", "OperationNumber = NULL_INT", - "ParentOperationNumber = OperationNumber", - "Depth = OperationNumber", + "ParentOperationNumber = NULL_INT", + "Depth = NULL_INT", "CallerLine = (String) null", "IsCompilation = NULL_BOOLEAN", "InputSizeLong = NULL_LONG"), diff --git a/open-api/lang-parser/lang-parser.gradle b/open-api/lang-parser/lang-parser.gradle index 8baf8dc5632..d785dc7c187 100644 --- a/open-api/lang-parser/lang-parser.gradle +++ b/open-api/lang-parser/lang-parser.gradle @@ -10,6 +10,7 @@ plugins { apply from: "$rootDir/gradle/web-common.gradle" dependencies { + implementation project(':Util') api project(':open-api-shared-fu') implementation project(':log-factory') api project(':proto:proto-backplane-grpc') diff --git a/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/ParsedDocument.java b/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/ParsedDocument.java index 93f3269a790..a75cfb9ddfd 100644 --- a/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/ParsedDocument.java +++ b/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/ParsedDocument.java @@ -3,7 +3,6 @@ // package io.deephaven.lang.parse; -import io.deephaven.base.Lazy; import io.deephaven.base.verify.Assert; import io.deephaven.io.logger.Logger; import io.deephaven.lang.generated.*; @@ -11,6 +10,7 @@ import io.deephaven.proto.backplane.script.grpc.DocumentRange; import io.deephaven.proto.backplane.script.grpc.Position; import io.deephaven.proto.backplane.script.grpc.TextEdit; +import io.deephaven.util.datastructures.CachingSupplier; import io.deephaven.web.shared.fu.MappedIterable; import java.util.*; @@ -100,7 +100,7 @@ public boolean containsIndex(int i) { private static final Pattern NEW_LINE_PATTERN = Pattern.compile("\\r?\\n"); private final ChunkerDocument doc; - private final Lazy> statements; + private final CachingSupplier> statements; private final String src; private String errorSource; private ParseException error; @@ -112,7 +112,7 @@ public ParsedDocument(ChunkerDocument doc, String document) { this.src = document; computedPositions = new ConcurrentHashMap<>(4); assignments = new ConcurrentHashMap<>(12); - statements = new Lazy<>(() -> { + statements = new CachingSupplier<>(() -> { final LinkedHashSet stmts = new LinkedHashSet<>(); doc.childrenAccept(new ChunkerDefaultVisitor() { @Override diff --git a/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/CompletionLookups.java b/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/CompletionLookups.java index 554c043458e..e2d928cb7e7 100644 --- a/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/CompletionLookups.java +++ b/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/CompletionLookups.java @@ -3,11 +3,11 @@ // package io.deephaven.lang.completion; -import io.deephaven.base.Lazy; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.util.ScriptSession; +import io.deephaven.util.datastructures.CachingSupplier; import java.util.Collection; import java.util.Map; @@ -26,15 +26,15 @@ public class CompletionLookups { private static final WeakHashMap lookups = new WeakHashMap<>(); - private final Lazy>> statics; + private final CachingSupplier>> statics; private final Map referencedTables; - private final Lazy customCompletions; + private final CachingSupplier customCompletions; public CompletionLookups(Set customCompletionFactory) { final QueryLibrary ql = ExecutionContext.getContext().getQueryLibrary(); - statics = new Lazy<>(ql::getStaticImports); + statics = new CachingSupplier<>(ql::getStaticImports); referencedTables = new ConcurrentHashMap<>(); - customCompletions = new Lazy<>(() -> new DelegatingCustomCompletion(customCompletionFactory)); + customCompletions = new CachingSupplier<>(() -> new DelegatingCustomCompletion(customCompletionFactory)); // This can be slow, so lets start it on a background thread right away. final ForkJoinPool pool = ForkJoinPool.commonPool(); diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateCachingSupplier.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateCachingSupplier.java index ae28bf10bc5..a102cb09e85 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateCachingSupplier.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateCachingSupplier.java @@ -8,9 +8,9 @@ import static io.deephaven.replication.ReplicatePrimitiveCode.replaceAll; public class ReplicateCachingSupplier { - private static final String LAZY_CACHING_SUPPLIER_DIR = "Util/src/main/java/io/deephaven/util/datastructures/"; - private static final String LAZY_CACHING_SUPPLIER_PATH = LAZY_CACHING_SUPPLIER_DIR + "LazyCachingSupplier.java"; - private static final String LAZY_CACHING_FUNCTION_PATH = LAZY_CACHING_SUPPLIER_DIR + "LazyCachingFunction.java"; + private static final String SOFT_CACHING_SUPPLIER_DIR = "Util/src/main/java/io/deephaven/util/datastructures/"; + private static final String SOFT_CACHING_SUPPLIER_PATH = SOFT_CACHING_SUPPLIER_DIR + "SoftCachingSupplier.java"; + private static final String SOFT_CACHING_FUNCTION_PATH = SOFT_CACHING_SUPPLIER_DIR + "SoftCachingFunction.java"; private static final String[] NO_EXCEPTIONS = new String[0]; @@ -22,7 +22,7 @@ public static void main(final String[] args) throws IOException { {"Supplier", "Function"}, {"supplier", "function"}, }; - replaceAll("replicateCachingSupplier", LAZY_CACHING_SUPPLIER_PATH, LAZY_CACHING_FUNCTION_PATH, null, + replaceAll("replicateCachingSupplier", SOFT_CACHING_SUPPLIER_PATH, SOFT_CACHING_FUNCTION_PATH, null, NO_EXCEPTIONS, pairs); } } diff --git a/server/src/main/java/io/deephaven/server/console/groovy/GroovyConsoleSessionModule.java b/server/src/main/java/io/deephaven/server/console/groovy/GroovyConsoleSessionModule.java index 8497050d449..0adcd9400c2 100644 --- a/server/src/main/java/io/deephaven/server/console/groovy/GroovyConsoleSessionModule.java +++ b/server/src/main/java/io/deephaven/server/console/groovy/GroovyConsoleSessionModule.java @@ -36,7 +36,7 @@ GroovyDeephavenSession bindGroovySession( final ScriptSession.Listener listener, final RunScripts runScripts) { try { - return new GroovyDeephavenSession(updateGraph, operationInitializer, lookup, listener, runScripts); + return GroovyDeephavenSession.of(updateGraph, operationInitializer, lookup, listener, runScripts); } catch (final IOException e) { throw new UncheckedIOException(e); } diff --git a/server/src/test/java/io/deephaven/server/appmode/ApplicationTest.java b/server/src/test/java/io/deephaven/server/appmode/ApplicationTest.java index dbfd1a22029..3870fb17124 100644 --- a/server/src/test/java/io/deephaven/server/appmode/ApplicationTest.java +++ b/server/src/test/java/io/deephaven/server/appmode/ApplicationTest.java @@ -50,7 +50,7 @@ public void app00() { @Test public void app01() throws IOException { - session = new GroovyDeephavenSession( + session = GroovyDeephavenSession.of( ExecutionContext.getContext().getUpdateGraph(), ExecutionContext.getContext().getOperationInitializer(), NoOp.INSTANCE, null,