From fae704c30114e893e0dd7ac6c36c06cc1c55215c Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 22 Feb 2024 16:36:59 -0700 Subject: [PATCH 01/19] QueryCompiler Batch Formula Compilation Checkpoint --- .../deephaven/util/CompletionStageFuture.java | 201 +++++ .../util/CompletionStageFutureImpl.java | 356 +++++++++ .../util/datastructures/CachingSupplier.java | 44 ++ engine/context/build.gradle | 1 + .../engine/context/PoisonedQueryCompiler.java | 10 +- .../engine/context/QueryCompiler.java | 700 +++++++++++------- .../engine/context/QueryCompilerRequest.java | 72 ++ .../util/SynchronizedJavaFileManager.java | 132 ++++ .../engine/context/TestQueryCompiler.java | 102 ++- .../engine/table/impl/DeferredViewTable.java | 21 +- .../table/impl/PartitionAwareSourceTable.java | 54 +- .../impl/QueryCompilerRequestProcessor.java | 93 +++ .../engine/table/impl/QueryTable.java | 8 +- .../engine/table/impl/RedefinableTable.java | 8 +- .../table/impl/WouldMatchOperation.java | 19 +- .../table/impl/by/AggregationProcessor.java | 23 +- .../table/impl/by/FormulaChunkedOperator.java | 14 +- .../impl/by/typed/TypedHasherFactory.java | 12 +- .../BaseNodeOperationsRecorder.java | 19 +- .../impl/hierarchical/RollupTableImpl.java | 10 +- .../TreeNodeOperationsRecorder.java | 16 +- .../impl/hierarchical/TreeTableFilter.java | 9 +- .../impl/hierarchical/TreeTableImpl.java | 9 +- .../BaseTableTransformationColumn.java | 6 + .../BiTableTransformationColumn.java | 7 +- .../impl/partitioned/LongConstantColumn.java | 13 +- .../partitioned/PartitionedTableImpl.java | 13 +- .../PartitionedTableProxyImpl.java | 7 +- .../TableTransformationColumn.java | 7 +- .../rangejoin/ValidFloatingPointFilter.java | 8 +- .../impl/select/AbstractConditionFilter.java | 125 +--- .../impl/select/AbstractFormulaColumn.java | 68 +- .../impl/select/AbstractRangeFilter.java | 6 +- .../select/BaseIncrementalReleaseFilter.java | 10 +- .../table/impl/select/ByteRangeFilter.java | 9 +- .../table/impl/select/CharRangeFilter.java | 9 +- .../engine/table/impl/select/ClockFilter.java | 8 +- .../impl/select/ComparableRangeFilter.java | 9 +- .../table/impl/select/ComposedFilter.java | 10 +- .../table/impl/select/ConditionFilter.java | 108 +-- .../table/impl/select/DhFormulaColumn.java | 154 ++-- .../table/impl/select/DoubleRangeFilter.java | 9 +- .../impl/select/DownsampledWhereFilter.java | 8 +- .../table/impl/select/DynamicWhereFilter.java | 6 +- .../table/impl/select/FloatRangeFilter.java | 9 +- .../table/impl/select/FunctionalColumn.java | 12 +- .../table/impl/select/InstantRangeFilter.java | 8 +- .../table/impl/select/IntRangeFilter.java | 9 +- .../table/impl/select/LongRangeFilter.java | 9 +- .../engine/table/impl/select/MatchFilter.java | 13 +- .../select/MultiSourceFunctionalColumn.java | 13 +- .../table/impl/select/NullSelectColumn.java | 12 +- .../impl/select/RangeConditionFilter.java | 10 +- .../impl/select/ReinterpretedColumn.java | 19 +- .../impl/select/RollingReleaseFilter.java | 8 +- .../table/impl/select/SelectColumn.java | 39 +- .../table/impl/select/ShortRangeFilter.java | 9 +- .../SingleSidedComparableRangeFilter.java | 9 +- .../table/impl/select/SourceColumn.java | 18 +- .../table/impl/select/SwitchColumn.java | 15 +- .../table/impl/select/TimeSeriesFilter.java | 8 +- .../engine/table/impl/select/WhereFilter.java | 24 +- .../impl/select/WhereFilterInvertedImpl.java | 10 +- .../impl/select/WhereFilterPatternImpl.java | 8 +- .../table/impl/select/WhereNoneFilter.java | 8 +- .../impl/select/analyzers/BaseLayer.java | 10 - .../select/analyzers/DependencyLayerBase.java | 8 - .../select/analyzers/RedirectionLayer.java | 5 - .../analyzers/SelectAndViewAnalyzer.java | 113 ++- .../select/analyzers/StaticFlattenLayer.java | 5 - .../impl/select/codegen/FormulaAnalyzer.java | 125 +++- .../select/codegen/JavaKernelBuilder.java | 74 +- .../impl/select/formula/FormulaFactory.java | 1 + .../select/python/FormulaColumnPython.java | 14 +- .../updateby/UpdateByOperatorFactory.java | 36 +- .../BaseRollingFormulaOperator.java | 9 +- .../BooleanRollingFormulaOperator.java | 8 +- .../ByteRollingFormulaOperator.java | 12 +- .../CharRollingFormulaOperator.java | 12 +- .../DoubleRollingFormulaOperator.java | 12 +- .../FloatRollingFormulaOperator.java | 12 +- .../IntRollingFormulaOperator.java | 12 +- .../LongRollingFormulaOperator.java | 12 +- .../ObjectRollingFormulaOperator.java | 8 +- .../ShortRollingFormulaOperator.java | 12 +- .../engine/util/DynamicCompileUtils.java | 32 +- .../impl/QueryTableWhereParallelTest.java | 7 +- .../impl/lang/TestQueryLanguageParser.java | 3 +- .../impl/select/FormulaKernelSample.java | 2 +- .../table/impl/select/FormulaSample.java | 7 +- .../util/TestCompileSimpleFunction.java | 2 +- .../time/TimeLiteralReplacedExpression.java | 2 - .../impl/util/PerformanceQueriesGeneral.java | 6 +- 93 files changed, 2573 insertions(+), 791 deletions(-) create mode 100644 Util/src/main/java/io/deephaven/util/CompletionStageFuture.java create mode 100644 Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java create mode 100644 Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java create mode 100644 engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java create mode 100644 engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java new file mode 100644 index 00000000000..c410765e187 --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java @@ -0,0 +1,201 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.util; + +import org.jetbrains.annotations.NotNull; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * A safe version of CompletableFuture that does not expose the completion API. + * + * @param The result type returned by this future's {@code join} + */ +public interface CompletionStageFuture extends Future, CompletionStage { + + /** + * Returns a new CompletionStageFuture that is already completed with the given value. + * + * @param value the value + * @param the type of the value + * @return the completed CompletionStageFuture + * @see CompletableFuture#completedFuture(Object) + */ + static CompletionStageFuture completedFuture(U value) { + final CompletionStageFutureImpl.Resolver resolver = CompletionStageFutureImpl.make(); + resolver.complete(value); + return resolver.getFuture(); + } + + /** + * Returns a new CompletableFuture that is already completed exceptionally with the given exception. + * + * @param ex the exception + * @param the type of the value + * @return the exceptionally completed CompletableFuture + * @since 9 + * @see CompletableFuture#failedFuture(Throwable) + */ + static CompletionStageFuture failedFuture(Throwable ex) { + final CompletionStageFutureImpl.Resolver resolver = CompletionStageFutureImpl.make(); + resolver.completeExceptionally(ex); + return resolver.getFuture(); + } + + interface Resolver { + + /** + * If not already completed, sets the value returned by {@link #get()} and related methods to the given value. + * + * @param value the result value + * @return {@code true} if this invocation caused this SafeCompletableFuture to transition to a completed state, + * else {@code false} + * @see java.util.concurrent.CompletableFuture#complete(Object) + */ + boolean complete(T value); + + /** + * If not already completed, causes invocations of {@link #get()} and related methods to throw the given + * exception. + * + * @param ex the exception + * @return {@code true} if this invocation caused this SafeCompletableFuture to transition to a completed state, + * else {@code false} + * @see java.util.concurrent.CompletableFuture#completeExceptionally(Throwable) + */ + boolean completeExceptionally(@NotNull Throwable ex); + + /** + * @return the underlying future to provide to the recipient + */ + CompletionStageFuture getFuture(); + } + + @Override + CompletionStageFuture thenApply(Function fn); + + @Override + CompletionStageFuture thenApplyAsync(Function fn); + + @Override + CompletionStageFuture thenApplyAsync(Function fn, Executor executor); + + @Override + CompletionStageFuture thenAccept(Consumer action); + + @Override + CompletionStageFuture thenAcceptAsync(Consumer action); + + @Override + CompletionStageFuture thenAcceptAsync(Consumer action, Executor executor); + + @Override + CompletionStageFuture thenRun(Runnable action); + + @Override + CompletionStageFuture thenRunAsync(Runnable action); + + @Override + CompletionStageFuture thenRunAsync(Runnable action, Executor executor); + + @Override + CompletionStageFuture thenCombine( + CompletionStage other, BiFunction fn); + + @Override + CompletionStageFuture thenCombineAsync( + CompletionStage other, BiFunction fn); + + @Override + CompletionStageFuture thenCombineAsync( + CompletionStage other, BiFunction fn, Executor executor); + + @Override + CompletionStageFuture thenAcceptBoth( + CompletionStage other, BiConsumer action); + + @Override + CompletionStageFuture thenAcceptBothAsync( + CompletionStage other, BiConsumer action); + + @Override + CompletionStageFuture thenAcceptBothAsync( + CompletionStage other, BiConsumer action, Executor executor); + + @Override + CompletionStageFuture runAfterBoth(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterBothAsync(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterBothAsync(CompletionStage other, Runnable action, Executor executor); + + @Override + CompletionStageFuture applyToEither(CompletionStage other, Function fn); + + @Override + CompletionStageFuture applyToEitherAsync(CompletionStage other, Function fn); + + @Override + CompletionStageFuture applyToEitherAsync( + CompletionStage other, Function fn, Executor executor); + + @Override + CompletionStageFuture acceptEither(CompletionStage other, Consumer action); + + @Override + CompletionStageFuture acceptEitherAsync(CompletionStage other, Consumer action); + + @Override + CompletionStageFuture acceptEitherAsync( + CompletionStage other, Consumer action, Executor executor); + + @Override + CompletionStageFuture runAfterEither(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterEitherAsync(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterEitherAsync(CompletionStage other, Runnable action, Executor executor); + + @Override + CompletionStageFuture thenCompose(Function> fn); + + @Override + CompletionStageFuture thenComposeAsync(Function> fn); + + @Override + CompletionStageFuture thenComposeAsync( + Function> fn, Executor executor); + + @Override + CompletionStageFuture handle(BiFunction fn); + + @Override + CompletionStageFuture handleAsync(BiFunction fn); + + @Override + CompletionStageFuture handleAsync(BiFunction fn, Executor executor); + + @Override + CompletionStageFuture whenComplete(BiConsumer action); + + @Override + CompletionStageFuture whenCompleteAsync(BiConsumer action); + + @Override + CompletionStageFuture whenCompleteAsync(BiConsumer action, Executor executor); + + @Override + CompletionStageFuture exceptionally(Function fn); +} diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java new file mode 100644 index 00000000000..d0ebef551fe --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java @@ -0,0 +1,356 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.util; + +import org.jetbrains.annotations.NotNull; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.function.*; + +/** + * A {@link CompletableFuture} that prevents users from completing the future by hiding behind + * {@link CompletionStageFuture}. + * + * @param The result type returned by this future's {@code join} + */ +@SuppressWarnings("unchecked") +public class CompletionStageFutureImpl extends CompletableFuture implements CompletionStageFuture { + + /** + * Create a new incomplete future. + * + * @param The result type returned by this future's {@code join} + * @return a resolver for the future + */ + public static Resolver make() { + return new CompletionStageFutureImpl().new ResolverImpl(); + } + + /** + * A resolver for this future implementation. + */ + private class ResolverImpl implements CompletionStageFuture.Resolver { + public boolean complete(final T value) { + return safelyComplete(value); + } + + public boolean completeExceptionally(@NotNull final Throwable ex) { + return safelyCompleteExceptionally(ex); + } + + public CompletionStageFuture getFuture() { + return CompletionStageFutureImpl.this; + } + } + + private boolean safelyComplete(final T value) { + return super.complete(value); + } + + private boolean safelyCompleteExceptionally(@NotNull final Throwable ex) { + return super.completeExceptionally(ex); + } + + @Override + public CompletableFuture newIncompleteFuture() { + return new CompletionStageFutureImpl<>(); + } + + /////////////////////////// + // CompletableFuture API // + /////////////////////////// + + @Override + public boolean complete(final T value) { + throw erroneousCompletionException(); + } + + @Override + public boolean completeExceptionally(@NotNull final Throwable ex) { + throw erroneousCompletionException(); + } + + @Override + public void obtrudeValue(final T value) { + throw erroneousCompletionException(); + } + + @Override + public void obtrudeException(@NotNull final Throwable ex) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeAsync( + @NotNull final Supplier supplier, + @NotNull final Executor executor) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeAsync(@NotNull final Supplier supplier) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeOnTimeout(final T value, long timeout, TimeUnit unit) { + throw erroneousCompletionException(); + } + + private static UnsupportedOperationException erroneousCompletionException() { + return new UnsupportedOperationException("Users should not complete futures."); + } + + ///////////////////////// + // CompletionStage API // + ///////////////////////// + + @Override + public CompletionStageFutureImpl thenApply(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.thenApply(fn); + } + + @Override + public CompletionStageFutureImpl thenApplyAsync(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.thenApplyAsync(fn); + } + + @Override + public CompletionStageFutureImpl thenApplyAsync( + @NotNull final Function fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenApplyAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl thenAccept(@NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.thenAccept(action); + } + + @Override + public CompletionStageFutureImpl thenAcceptAsync(@NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.thenAcceptAsync(action); + } + + @Override + public CompletionStageFutureImpl thenAcceptAsync( + @NotNull final Consumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenAcceptAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl thenRun(@NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.thenRun(action); + } + + @Override + public CompletionStageFutureImpl thenRunAsync(@NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.thenRunAsync(action); + } + + @Override + public CompletionStageFutureImpl thenRunAsync( + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenRunAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl thenCombine( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.thenCombine(other, fn); + } + + @Override + public CompletionStageFutureImpl thenCombineAsync( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.thenCombineAsync(other, fn); + } + + @Override + public CompletionStageFutureImpl thenCombineAsync( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenCombineAsync(other, fn, executor); + } + + + @Override + public CompletionStageFutureImpl thenAcceptBoth( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.thenAcceptBoth(other, action); + } + + + @Override + public CompletionStageFutureImpl thenAcceptBothAsync( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.thenAcceptBothAsync(other, action); + } + + @Override + public CompletionStageFutureImpl thenAcceptBothAsync( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenAcceptBothAsync(other, action, executor); + } + + + @Override + public CompletionStageFutureImpl runAfterBoth( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterBoth(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterBothAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterBothAsync(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterBothAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.runAfterBothAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl applyToEither( + @NotNull final CompletionStage other, + @NotNull final Function fn) { + return (CompletionStageFutureImpl) super.applyToEither(other, fn); + } + + @Override + public CompletionStageFutureImpl applyToEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Function fn) { + return (CompletionStageFutureImpl) super.applyToEitherAsync(other, fn); + } + + @Override + public CompletionStageFutureImpl applyToEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Function fn, Executor executor) { + return (CompletionStageFutureImpl) super.applyToEitherAsync(other, fn, executor); + } + + @Override + public CompletionStageFutureImpl acceptEither( + @NotNull final CompletionStage other, + @NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.acceptEither(other, action); + } + + @Override + public CompletionStageFutureImpl acceptEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.acceptEitherAsync(other, action); + } + + @Override + public CompletionStageFutureImpl acceptEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Consumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.acceptEitherAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl runAfterEither( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterEither(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterEitherAsync(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.runAfterEitherAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl thenCompose( + @NotNull final Function> fn) { + return (CompletionStageFutureImpl) super.thenCompose(fn); + } + + @Override + public CompletionStageFutureImpl thenComposeAsync( + @NotNull final Function> fn) { + return (CompletionStageFutureImpl) super.thenComposeAsync(fn); + } + + @Override + public CompletionStageFutureImpl thenComposeAsync( + @NotNull final Function> fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenComposeAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl handle(@NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.handle(fn); + } + + @Override + public CompletionStageFutureImpl handleAsync( + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.handleAsync(fn); + } + + @Override + public CompletionStageFutureImpl handleAsync( + @NotNull final BiFunction fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.handleAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl whenComplete(@NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.whenComplete(action); + } + + @Override + public CompletionStageFutureImpl whenCompleteAsync( + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.whenCompleteAsync(action); + } + + @Override + public CompletionStageFutureImpl whenCompleteAsync( + @NotNull final BiConsumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.whenCompleteAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl exceptionally(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.exceptionally(fn); + } +} diff --git a/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java new file mode 100644 index 00000000000..43ff488da7e --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java @@ -0,0 +1,44 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.util.datastructures; + +import org.jetbrains.annotations.NotNull; + +import java.util.function.Supplier; + +/** + * {@link Supplier} wrapper that caches the result. + * + * @param the type of results supplied by this supplier + */ +public final class CachingSupplier implements Supplier { + + private final Supplier internalSupplier; + + private volatile boolean hasCachedResult; + private volatile OUTPUT_TYPE cachedResult; + + /** + * Construct a {@link Supplier} wrapper. + * + * @param internalSupplier The {@link Supplier} to wrap. + */ + public CachingSupplier(@NotNull final Supplier internalSupplier) { + this.internalSupplier = internalSupplier; + } + + @Override + public OUTPUT_TYPE get() { + if (!hasCachedResult) { + synchronized (this) { + if (!hasCachedResult) { + cachedResult = internalSupplier.get(); + hasCachedResult = true; + } + } + } + + return cachedResult; + } +} diff --git a/engine/context/build.gradle b/engine/context/build.gradle index 0f683dcb5b9..da73cfac799 100644 --- a/engine/context/build.gradle +++ b/engine/context/build.gradle @@ -24,6 +24,7 @@ dependencies { implementation 'com.github.f4b6a3:uuid-creator:5.2.0' Classpaths.inheritCommonsText(project, 'implementation') + Classpaths.inheritImmutables(project) testImplementation TestTools.projectDependency(project, 'Base') testImplementation project(':engine-test-utils') diff --git a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java index d6909402ced..1acec6cac88 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java @@ -3,12 +3,11 @@ // package io.deephaven.engine.context; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.ExecutionContextRegistrationException; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.io.File; -import java.util.Map; public class PoisonedQueryCompiler extends QueryCompiler { @@ -31,8 +30,9 @@ public void setParentClassLoader(ClassLoader parentClassLoader) { } @Override - public Class compile(@NotNull String className, @NotNull String classBody, @NotNull String packageNameRoot, - @Nullable StringBuilder codeLog, @NotNull Map> parameterClasses) { - return fail(); + public void compile( + @NotNull final QueryCompilerRequest[] requests, + @NotNull final CompletionStageFuture.Resolver>[] resolvers) { + fail(); } } diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 8f67f1ab2c2..15cb670e788 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -5,16 +5,19 @@ import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.FileUtils; -import io.deephaven.base.Pair; import io.deephaven.configuration.Configuration; import io.deephaven.configuration.DataDir; import io.deephaven.datastructures.util.CollectionUtil; +import io.deephaven.engine.context.util.SynchronizedJavaFileManager; +import io.deephaven.engine.updategraph.OperationInitializer; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.util.ByteUtils; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.CompletionStageFutureImpl; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import javax.tools.*; import java.io.*; @@ -31,9 +34,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.function.Supplier; import java.util.jar.Attributes; import java.util.jar.Manifest; @@ -41,6 +42,7 @@ import java.util.stream.Stream; public class QueryCompiler { + private static final Logger log = LoggerFactory.getLogger(QueryCompiler.class); /** * We pick a number just shy of 65536, leaving a little elbow room for good luck. @@ -56,9 +58,9 @@ public class QueryCompiler { private static final long CODEGEN_TIMEOUT_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); // 10 seconds private static final String CODEGEN_LOOP_DELAY_PROP = "QueryCompiler.codegen.retry.delay"; private static final long CODEGEN_LOOP_DELAY_MS_DEFAULT = 100; - private static final long codegenTimeoutMs = + private static final long CODEGEN_TIMEOUT_MS = Configuration.getInstance().getLongWithDefault(CODEGEN_TIMEOUT_PROP, CODEGEN_TIMEOUT_MS_DEFAULT); - private static final long codegenLoopDelayMs = + private static final long CODEGEN_LOOP_DELAY_MS = Configuration.getInstance().getLongWithDefault(CODEGEN_LOOP_DELAY_PROP, CODEGEN_LOOP_DELAY_MS_DEFAULT); private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); @@ -76,7 +78,7 @@ static QueryCompiler createForUnitTests() { return new QueryCompiler(queryCompilerDir.toFile()); } - private final Map>> knownClasses = new HashMap<>(); + private final Map>> knownClasses = new HashMap<>(); private final String[] dynamicPatterns = new String[] {DYNAMIC_GROOVY_CLASS_PREFIX, FORMULA_PREFIX}; @@ -125,7 +127,7 @@ private QueryCompiler( /** * Enables or disables compilation logging. * - * @param logEnabled Whether or not logging should be enabled + * @param logEnabled Whether logging should be enabled * @return The value of {@code logEnabled} before calling this method. */ public static boolean setLogEnabled(boolean logEnabled) { @@ -204,66 +206,131 @@ public File getFakeClassDestination() { } public void setParentClassLoader(final ClassLoader parentClassLoader) { + // noinspection NonAtomicOperationOnVolatileField ucl = new WritableURLClassLoader(ucl.getURLs(), parentClassLoader); } - public final Class compile(String className, String classBody, String packageNameRoot) { - return compile(className, classBody, packageNameRoot, null, Collections.emptyMap()); - } - - public final Class compile(String className, String classBody, String packageNameRoot, - Map> parameterClasses) { - return compile(className, classBody, packageNameRoot, null, parameterClasses); + /** + * Compile a class. + * + * @param request The compilation request + */ + public Class compile(@NotNull final QueryCompilerRequest request) { + final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + compile(request, resolver); + try { + return resolver.getFuture().get(); + } catch (InterruptedException | ExecutionException e) { + throw new UncheckedDeephavenException("Could not compile class", e); + } } - public final Class compile(String className, String classBody, String packageNameRoot, StringBuilder codeLog) { - return compile(className, classBody, packageNameRoot, codeLog, Collections.emptyMap()); + /** + * Compile a class. + * + * @param request The compilation request + */ + public void compile( + @NotNull final QueryCompilerRequest request, + @NotNull final CompletionStageFuture.Resolver> resolver) { + // noinspection unchecked + compile(new QueryCompilerRequest[] {request}, new CompletionStageFuture.Resolver[] {resolver}); } /** - * Compile a class. + * Compiles all requests. * - * @param className Class name - * @param classBody Class body, before update with "$CLASS_NAME$" replacement and package name prefixing - * @param packageNameRoot Package name prefix - * @param codeLog Optional "log" for final class code - * @param parameterClasses Generic parameters, empty if none required - * @return The compiled class + * @param requests The compilation requests */ - public Class compile(@NotNull final String className, - @NotNull final String classBody, - @NotNull final String packageNameRoot, - @Nullable final StringBuilder codeLog, - @NotNull final Map> parameterClasses) { - CompletableFuture> future; - final boolean alreadyExists; + public void compile( + @NotNull final QueryCompilerRequest[] requests, + @NotNull final CompletionStageFuture.Resolver>[] resolvers) { + if (requests.length == 0) { + return; + } + if (requests.length != resolvers.length) { + throw new IllegalArgumentException("Requests and resolvers must be the same length"); + } + + // noinspection unchecked + final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; + + final List newRequests = new ArrayList<>(); + final List>> newResolvers = new ArrayList<>(); synchronized (this) { - future = knownClasses.get(classBody); - if (future != null) { - alreadyExists = true; - } else { - future = new CompletableFuture<>(); - knownClasses.put(classBody, future); - alreadyExists = false; + for (int ii = 0; ii < requests.length; ++ii) { + final QueryCompilerRequest request = requests[ii]; + final CompletionStageFuture.Resolver> resolver = resolvers[ii]; + + CompletionStageFuture> future = + knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); + if (future == null) { + newRequests.add(request); + newResolvers.add(resolver); + future = resolver.getFuture(); + } + allFutures[ii] = future; } } - // Someone else has already made the future. I'll just wait for the answer. - if (alreadyExists) { + /* + * @formatter:off + * 3. try to resolve CFs without compiling; retain next hash to try + * 4. compile all remaining with a single compilation task + * 5. goto step 3 + * 6. probably need Consumer> to fit DhFormulaColumn pattern? (other select columns don't need this) + * @formatter:on + */ + + if (!newResolvers.isEmpty()) { + // It's my job to fulfill the future of these futures. try { - return future.get(); - } catch (InterruptedException | ExecutionException error) { - throw new UncheckedDeephavenException(error); + compileHelper(newRequests, newResolvers); + } catch (RuntimeException e) { + // These failures are not applicable to a single request, so we can't just complete the future and + // leave the failure in the cache. + for (int ii = 0; ii < newRequests.size(); ++ii) { + if (newResolvers.get(ii).completeExceptionally(e)) { + knownClasses.remove(newRequests.get(ii).classBody()); + } + } + throw e; } } - // It's my job to fulfill the future. - try { - return compileHelper(className, classBody, packageNameRoot, codeLog, parameterClasses, future); - } catch (RuntimeException e) { - future.completeExceptionally(e); - throw e; + Error firstError = null; + RuntimeException firstException = null; + for (int ii = 0; ii < requests.length; ++ii) { + try { + resolvers[ii].complete(allFutures[ii].get()); + } catch (InterruptedException | ExecutionException e) { + final RuntimeException err; + if (e instanceof InterruptedException) { + err = new UncheckedDeephavenException("Interrupted while waiting for codegen", e); + } else { + Throwable cause = e.getCause(); + if (cause instanceof Error) { + firstError = (Error) cause; + resolvers[ii].completeExceptionally(cause); + continue; + } else if (cause instanceof RuntimeException) { + err = (RuntimeException) cause; + } else { + err = new UncheckedDeephavenException("Error during codegen", e); + } + } + if (firstException == null) { + firstException = err; + } + resolvers[ii].completeExceptionally(err); + } + } + if (firstError != null) { + throw firstError; + } + if (firstException != null) { + throw firstException; } } @@ -416,89 +483,166 @@ private String getClassPath() { return sb.toString(); } - private Class compileHelper(@NotNull final String className, - @NotNull final String classBody, - @NotNull final String packageNameRoot, - @Nullable final StringBuilder codeLog, - @NotNull final Map> parameterClasses, - @NotNull final CompletableFuture> resultFuture) { + private void compileHelper( + @NotNull final List requests, + @NotNull final List>> resolvers) { final MessageDigest digest; try { digest = MessageDigest.getInstance("SHA-256"); } catch (NoSuchAlgorithmException e) { throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); } - final String basicHashText = - ByteUtils.byteArrToHex(digest.digest(classBody.getBytes(StandardCharsets.UTF_8))); - - for (int pi = 0; pi < MAX_CLASS_COLLISIONS; ++pi) { - final String packageNameSuffix = "c_" + basicHashText - + (pi == 0 ? "" : ("p" + pi)) - + "v" + JAVA_CLASS_VERSION; - final String packageName = (packageNameRoot.isEmpty() - ? packageNameSuffix - : packageNameRoot + (packageNameRoot.endsWith(".") ? "" : ".") + packageNameSuffix); - final String fqClassName = packageName + "." + className; - - // Ask the classloader to load an existing class with this name. This might: - // 1. Fail to find a class (returning null) - // 2. Find a class whose body has the formula we are looking for - // 3. Find a class whose body has a different formula (hash collision) - Class result = tryLoadClassByFqName(fqClassName, parameterClasses); - if (result == null) { - // Couldn't find one, so try to create it. This might: - // A. succeed - // B. Lose a race to another process on the same file system which is compiling the identical formula - // C. Lose a race to another process on the same file system compiling a different formula that - // happens to have the same hash (same packageName). - // However, regardless of A-C, there will be *some* class being found (i.e. tryLoadClassByFqName won't - // return null). - maybeCreateClass(className, classBody, packageName, fqClassName); - - // We could be running on a screwy filesystem that is slow (e.g. NFS). - // If we wrote a file and can't load it ... then give the filesystem some time. - result = tryLoadClassByFqName(fqClassName, parameterClasses); + + final String[] basicHashText = new String[requests.size()]; + for (int ii = 0; ii < requests.size(); ++ii) { + basicHashText[ii] = ByteUtils.byteArrToHex(digest.digest( + requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); + } + + int numCompiled = 0; + final int[] next_pi = new int[requests.size()]; + final boolean[] compiled = new boolean[requests.size()]; + final String[] packageName = new String[requests.size()]; + final String[] fqClassName = new String[requests.size()]; + + while (numCompiled < requests.size()) { + for (int ii = 0; ii < requests.size(); ++ii) { + if (compiled[ii]) { + continue; + } + + while (true) { + final int pi = next_pi[ii]++; + final String packageNameSuffix = "c_" + basicHashText[ii] + + (pi == 0 ? "" : ("p" + pi)) + + "v" + JAVA_CLASS_VERSION; + + final QueryCompilerRequest request = requests.get(ii); + if (pi >= MAX_CLASS_COLLISIONS) { + Exception err = new IllegalStateException("Found too many collisions for package name root " + + request.packageNameRoot() + ", class name=" + request.className() + ", class body " + + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); + resolvers.get(ii).completeExceptionally(err); + compiled[ii] = true; + ++numCompiled; + break; + } + + packageName[ii] = request.getPackageName(packageNameSuffix); + fqClassName[ii] = packageName[ii] + "." + request.className(); + + // Ask the classloader to load an existing class with this name. This might: + // 1. Fail to find a class (returning null) + // 2. Find a class whose body has the formula we are looking for + // 3. Find a class whose body has a different formula (hash collision) + Class result = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); + if (result == null) { + break; // we'll try to compile it + } + + if (completeIfResultMatchesQueryCompilerRequest(packageName[ii], request, resolvers.get(ii), + result)) { + compiled[ii] = true; + ++numCompiled; + break; + } + } + } + + if (numCompiled == requests.size()) { + return; + } + + // Couldn't resolve at least one of the requests, so try a round of compilation. + final CompilationRequestAttempt[] compilationRequestAttempts = + new CompilationRequestAttempt[requests.size() - numCompiled]; + for (int ii = 0, jj = 0; ii < requests.size(); ++ii) { + if (!compiled[ii]) { + final QueryCompilerRequest request = requests.get(ii); + compilationRequestAttempts[jj++] = new CompilationRequestAttempt( + request, + packageName[ii], + fqClassName[ii], + resolvers.get(ii)); + } + } + + maybeCreateClass(compilationRequestAttempts); + + // We could be running on a screwy filesystem that is slow (e.g. NFS). If we wrote a file and can't load it + // ... then give the filesystem some time. All requests should use the same deadline. + final long deadline = System.currentTimeMillis() + CODEGEN_TIMEOUT_MS - CODEGEN_LOOP_DELAY_MS; + for (int ii = 0; ii < requests.size(); ++ii) { + if (compiled[ii]) { + continue; + } + + final QueryCompilerRequest request = requests.get(ii); + final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); + if (resolver.getFuture().isDone()) { + compiled[ii] = true; + ++numCompiled; + continue; + } + + // This request may have: + // A. succeeded + // B. Lost a race to another process on the same file system which is compiling the identical formula + // C. Lost a race to another process on the same file system compiling a different formula that collides + + Class clazz = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); try { - final long deadline = System.currentTimeMillis() + codegenTimeoutMs - codegenLoopDelayMs; - while (result == null && System.currentTimeMillis() < deadline) { - Thread.sleep(codegenLoopDelayMs); - result = tryLoadClassByFqName(fqClassName, parameterClasses); + while (clazz == null && System.currentTimeMillis() < deadline) { + // noinspection BusyWait + Thread.sleep(CODEGEN_LOOP_DELAY_MS); + clazz = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); } - } catch (InterruptedException ignored) { - // we got interrupted, just quit looping and ignore it. + } catch (final InterruptedException ie) { + throw new UncheckedDeephavenException("Interrupted while waiting for codegen", ie); } - if (result == null) { + // However, regardless of A-C, there will be *some* class being found + if (clazz == null) { throw new IllegalStateException("Should have been able to load *some* class here"); } - } - final String identifyingFieldValue = loadIdentifyingField(result); - - // If the class we found was indeed the class we were looking for, then complete the future and return it. - if (classBody.equals(identifyingFieldValue)) { - if (codeLog != null) { - // If the caller wants a textual copy of the code we either made, or just found in the cache. - codeLog.append(makeFinalCode(className, classBody, packageName)); - } - resultFuture.complete(result); - synchronized (this) { - // Note we are doing something kind of subtle here. We are removing an entry whose key was matched - // by value equality and replacing it with a value-equal but reference-different string that is a - // static member of the class we just loaded. This should be easier on the garbage collector because - // we are replacing a calculated value with a classloaded value and so in effect we are - // "canonicalizing" the string. This is important because these long strings stay in knownClasses - // forever. - knownClasses.remove(identifyingFieldValue); - knownClasses.put(identifyingFieldValue, resultFuture); - } - return result; + if (completeIfResultMatchesQueryCompilerRequest(packageName[ii], request, resolver, clazz)) { + compiled[ii] = true; + ++numCompiled; + } } - // Try the next hash name } - throw new IllegalStateException("Found too many collisions for package name root " + packageNameRoot - + ", class name=" + className - + ", class body hash=" + basicHashText + " - contact Deephaven support!"); + } + + private boolean completeIfResultMatchesQueryCompilerRequest( + final String packageName, + final QueryCompilerRequest request, + final CompletionStageFuture.Resolver> resolver, + final Class result) { + final String identifyingFieldValue = loadIdentifyingField(result); + if (!request.classBody().equals(identifyingFieldValue)) { + return false; + } + + // If the caller wants a textual copy of the code we either made, or just found in the cache. + request.codeLog() + .ifPresent(sb -> sb.append(makeFinalCode(request.className(), request.classBody(), packageName))); + + // If the class we found was indeed the class we were looking for, then complete the future and return it. + resolver.complete(result); + + synchronized (this) { + // Note we are doing something kind of subtle here. We are removing an entry whose key was matched + // by value equality and replacing it with a value-equal but reference-different string that is a + // static member of the class we just loaded. This should be easier on the garbage collector because + // we are replacing a calculated value with a classloaded value and so in effect we are + // "canonicalizing" the string. This is important because these long strings stay in knownClasses + // forever. + knownClasses.remove(identifyingFieldValue); + knownClasses.put(identifyingFieldValue, resolver.getFuture()); + } + + return true; } private Class tryLoadClassByFqName(String fqClassName, Map> parameterClasses) { @@ -589,11 +733,19 @@ private static int calcBytesConsumed(final char ch) { } private static class JavaSourceFromString extends SimpleJavaFileObject { + final String description; final String code; + final CompletionStageFuture.Resolver> resolver; - JavaSourceFromString(String name, String code) { + JavaSourceFromString( + final String description, + final String name, + final String code, + final CompletionStageFuture.Resolver> resolver) { super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); + this.description = description; this.code = code; + this.resolver = resolver; } public CharSequence getCharContent(boolean ignoreEncodingErrors) { @@ -601,56 +753,59 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) { } } - private static class JavaSourceFromFile extends SimpleJavaFileObject { - private static final int JAVA_LENGTH = Kind.SOURCE.extension.length(); - final String code; - - private JavaSourceFromFile(File basePath, File file) { - super(URI.create("string:///" + createName(basePath, file).replace('.', '/') + Kind.SOURCE.extension), - Kind.SOURCE); - try { - this.code = FileUtils.readTextFile(file); - } catch (IOException e) { - throw new UncheckedIOException(e); + private static class CompilationRequestAttempt { + final String description; + final String fqClassName; + final String finalCode; + final String packageName; + final String[] splitPackageName; + final QueryCompilerRequest request; + final CompletionStageFuture.Resolver> resolver; + + private CompilationRequestAttempt( + @NotNull final QueryCompilerRequest request, + @NotNull final String packageName, + @NotNull final String fqClassName, + @NotNull final CompletionStageFuture.Resolver> resolver) { + this.description = request.description(); + this.fqClassName = fqClassName; + this.resolver = resolver; + this.packageName = packageName; + this.request = request; + + finalCode = makeFinalCode(request.className(), request.classBody(), packageName); + + if (logEnabled) { + log.info().append("Generating code ").append(finalCode).endl(); } - } - private static String createName(File basePath, File file) { - final String base = basePath.getAbsolutePath(); - final String fileName = file.getAbsolutePath(); - if (!fileName.startsWith(base)) { - throw new IllegalArgumentException(file + " is not in " + basePath); - } - final String basename = fileName.substring(base.length()); - if (basename.endsWith(".java")) { - return basename.substring(0, basename.length() - JAVA_LENGTH); - } else { - return basename; + splitPackageName = packageName.split("\\."); + if (splitPackageName.length == 0) { + final Exception err = new UncheckedDeephavenException(String.format( + "packageName %s expected to have at least one .", packageName)); + resolver.completeExceptionally(err); } } - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return code; - } - } - - private void maybeCreateClass(String className, String code, String packageName, String fqClassName) { - final String finalCode = makeFinalCode(className, code, packageName); + public void ensureDirectories(@NotNull final String rootPath) { + if (splitPackageName.length == 0) { + // we've already failed + return; + } - if (logEnabled) { - log.info().append("Generating code ").append(finalCode).endl(); + final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); + final Path rootPathWithPackage = Paths.get(rootPath, truncatedSplitPackageName); + final File rpf = rootPathWithPackage.toFile(); + QueryCompiler.ensureDirectories(rpf, () -> "Couldn't create package directories: " + rootPathWithPackage); } - final File ctxClassDestination = getClassDestination(); - - final String[] splitPackageName = packageName.split("\\."); - if (splitPackageName.length == 0) { - throw new UncheckedDeephavenException(String.format( - "packageName %s expected to have at least one .", packageName)); + public JavaSourceFromString makeSource() { + return new JavaSourceFromString(description, fqClassName, finalCode, resolver); } - final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); + } + private void maybeCreateClass( + @NotNull final CompilationRequestAttempt[] requests) { // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. // Also create a temp directory e.g. /tmp/workspace/cache/classes/temporaryCompilationDirectory12345 @@ -663,125 +818,167 @@ private void maybeCreateClass(String className, String code, String packageName, final String rootPathAsString; final String tempDirAsString; try { - rootPathAsString = ctxClassDestination.getAbsolutePath(); - final Path rootPathWithPackage = Paths.get(rootPathAsString, truncatedSplitPackageName); - final File rpf = rootPathWithPackage.toFile(); - ensureDirectories(rpf, () -> "Couldn't create package directories: " + rootPathWithPackage); + rootPathAsString = getClassDestination().getAbsolutePath(); final Path tempPath = Files.createTempDirectory(Paths.get(rootPathAsString), "temporaryCompilationDirectory"); tempDirAsString = tempPath.toFile().getAbsolutePath(); - } catch (IOException ioe) { - throw new UncheckedIOException(ioe); - } - try { - maybeCreateClassHelper(fqClassName, finalCode, splitPackageName, rootPathAsString, tempDirAsString); - } finally { - try { - FileUtils.deleteRecursively(new File(tempDirAsString)); - } catch (Exception e) { - // ignore errors here + for (final CompilationRequestAttempt request : requests) { + request.ensureDirectories(rootPathAsString); } + } catch (IOException ioe) { + Exception err = new UncheckedIOException(ioe); + for (final CompilationRequestAttempt request : requests) { + request.resolver.completeExceptionally(err); + } + return; } - } - private void maybeCreateClassHelper(String fqClassName, String finalCode, String[] splitPackageName, - String rootPathAsString, String tempDirAsString) { - final StringWriter compilerOutput = new StringWriter(); final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); if (compiler == null) { throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); } - final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); - final List compilerOptions = Arrays.asList("-d", tempDirAsString, "-cp", classPathAsString); - - final JavaFileManager fileManager = compiler.getStandardFileManager(null, null, null); + final JavaFileManager fileManager = new SynchronizedJavaFileManager( + compiler.getStandardFileManager(null, null, null)); - boolean result = false; - boolean exceptionThrown = false; + boolean exceptionCaught = false; try { - result = compiler.getTask(compilerOutput, - fileManager, - null, - compilerOptions, - null, - Collections.singletonList(new JavaSourceFromString(fqClassName, finalCode))) - .call(); - } catch (final Throwable err) { - exceptionThrown = true; - throw err; + final OperationInitializer operationInitializer = ExecutionContext.getContext().getOperationInitializer(); + int parallelismFactor = operationInitializer.parallelismFactor(); + + int requestsPerTask = Math.max(32, (requests.length + parallelismFactor - 1) / parallelismFactor); + log.info().append("Compiling with parallelismFactor = ").append(parallelismFactor) + .append(" requestsPerTask = ").append(requestsPerTask).endl(); + if (parallelismFactor == 1 || requestsPerTask >= requests.length) { + maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + 0, requests.length, false); + } else { + int numTasks = (requests.length + requestsPerTask - 1) / requestsPerTask; + final Future[] tasks = new Future[numTasks]; + for (int jobId = 0; jobId < numTasks; ++jobId) { + final int startInclusive = jobId * requestsPerTask; + final int endExclusive = Math.min(requests.length, (jobId + 1) * requestsPerTask); + tasks[jobId] = operationInitializer.submit(() -> { + maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + startInclusive, endExclusive, false); + }); + } + for (int jobId = 0; jobId < numTasks; ++jobId) { + try { + tasks[jobId].get(); + } catch (Exception err) { + throw new UncheckedDeephavenException("Exception waiting for compilation task", err); + } + } + } + } catch (final Throwable t) { + exceptionCaught = true; + throw t; } finally { + try { + FileUtils.deleteRecursively(new File(tempDirAsString)); + } catch (Exception e) { + // ignore errors here + } + try { fileManager.close(); - } catch (final IOException ioe) { - if (!exceptionThrown) { + } catch (IOException ioe) { + if (!exceptionCaught) { // noinspection ThrowFromFinallyBlock throw new UncheckedIOException("Could not close JavaFileManager", ioe); } } } - if (!result) { - throw new UncheckedDeephavenException("Error compiling class " + fqClassName + ":\n" + compilerOutput); + } + + private void maybeCreateClassHelper( + @NotNull final JavaCompiler compiler, + @NotNull final JavaFileManager fileManager, + @NotNull final CompilationRequestAttempt[] requests, + @NotNull final String rootPathAsString, + @NotNull final String tempDirAsString, + final int startInclusive, + final int endExclusive, + final boolean isRetry) { + final StringWriter compilerOutput = new StringWriter(); + + final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); + final List compilerOptions = Arrays.asList( + "-d", tempDirAsString, + "-cp", classPathAsString, + // this option allows the compiler to attempt to process all source files even if some of them fail + "--should-stop=ifError=GENERATE"); + + final MutableInt numFailures = new MutableInt(0); + compiler.getTask(compilerOutput, + fileManager, + diagnostic -> { + if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { + return; + } + + final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); + final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " + + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); + if (source.resolver.completeExceptionally(err)) { + // only count the first failure for each source + numFailures.increment(); + } + }, + compilerOptions, + null, + Arrays.stream(requests, startInclusive, endExclusive) + .map(CompilationRequestAttempt::makeSource) + .collect(Collectors.toList())) + .call(); + + final List shouldRetry; + if (!isRetry && numFailures.intValue() > 0 && numFailures.intValue() != endExclusive - startInclusive) { + // if this is the first attempt, and we had some failures, but not all of them failed, then we should retry + shouldRetry = new ArrayList<>(); + } else { + shouldRetry = null; } + // The above has compiled into e.g. // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various // class files} // We want to atomically move it to e.g. // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} - Path srcDir = Paths.get(tempDirAsString, splitPackageName); - Path destDir = Paths.get(rootPathAsString, splitPackageName); - try { - Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException ioe) { - // The move might have failed for a variety of bad reasons. However, if the reason was because - // we lost the race to some other process, that's a harmless/desirable outcome, and we can ignore - // it. - if (!Files.exists(destDir)) { - throw new UncheckedIOException("Move failed for some reason other than destination already existing", - ioe); - } - } - } - - /** - * Try to compile the set of files, returning a pair of success and compiler output. - * - * @param basePath the base path for the java classes - * @param javaFiles the java source files - * @return a Pair of success, and the compiler output - */ - private Pair tryCompile(File basePath, Collection javaFiles) throws IOException { - final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); - } - - final File outputDirectory = Files.createTempDirectory("temporaryCompilationDirectory").toFile(); - - try { - final StringWriter compilerOutput = new StringWriter(); - final String javaClasspath = getJavaClassPath(); - - final Collection javaFileObjects = javaFiles.stream() - .map(f -> new JavaSourceFromFile(basePath, f)).collect(Collectors.toList()); + Arrays.stream(requests, startInclusive, endExclusive).forEach(request -> { + final Path srcDir = Paths.get(tempDirAsString, request.splitPackageName); + final Path destDir = Paths.get(rootPathAsString, request.splitPackageName); + try { + Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException ioe) { + if (shouldRetry != null && !Files.exists(srcDir) && !request.resolver.getFuture().isDone()) { + // This source actually succeeded in compiling, but was not written because some other source failed + // to compile. Let's recursively call ourselves to try again. + shouldRetry.add(request); + return; + } - final boolean result = compiler.getTask(compilerOutput, null, null, - Arrays.asList("-d", outputDirectory.getAbsolutePath(), "-cp", - getClassPath() + File.pathSeparator + javaClasspath), - null, javaFileObjects).call(); + if (!Files.exists(destDir) && !request.resolver.getFuture().isDone()) { + // The move might have failed for a variety of bad reasons. However, if the reason was because + // we lost the race to some other process, that's a harmless/desirable outcome, and we can ignore + // it. + request.resolver.completeExceptionally(new UncheckedIOException( + "Move failed for some reason other than destination already existing", ioe)); + } + } + }); - return new Pair<>(result, compilerOutput.toString()); - } finally { - FileUtils.deleteRecursively(outputDirectory); + if (shouldRetry != null && !shouldRetry.isEmpty()) { + maybeCreateClassHelper(compiler, fileManager, shouldRetry.toArray(CompilationRequestAttempt[]::new), + rootPathAsString, tempDirAsString, 0, shouldRetry.size(), true); } } /** - * Retrieve the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables. - * - * @return + * @return the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables */ private static String getJavaClassPath() { String javaClasspath; @@ -792,14 +989,17 @@ private static String getJavaClassPath() { if (teamCityWorkDir != null) { // We are running in TeamCity, get the classpath differently final File[] classDirs = new File(teamCityWorkDir + "/_out_/classes").listFiles(); - - for (File f : classDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + if (classDirs != null) { + for (File f : classDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } } - final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); - for (File f : testDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); + if (testDirs != null) { + for (File f : testDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } } final File[] jars = FileUtils.findAllFiles(new File(teamCityWorkDir + "/lib")); @@ -835,7 +1035,7 @@ private static String getJavaClassPath() { // use the default path separator final String filePaths = Stream.of(extendedClassPath.split("file:/")) .map(String::trim) - .filter(fileName -> fileName.length() > 0) + .filter(fileName -> !fileName.isEmpty()) .collect(Collectors.joining(File.pathSeparator)); // Remove the classpath jar in question, and expand it with the files from the manifest diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java new file mode 100644 index 00000000000..38be0b97fe4 --- /dev/null +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java @@ -0,0 +1,72 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.context; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Immutable; + +import java.util.Map; +import java.util.Optional; + +/** + * A request to compile a java class. + */ +@Immutable +@BuildableStyle +public abstract class QueryCompilerRequest { + public static Builder builder() { + return ImmutableQueryCompilerRequest.builder(); + } + + /** + * @return the description to add to the query performance recorder nugget for this request + */ + public abstract String description(); + + /** + * @return the class name to use for the generated class + */ + public abstract String className(); + + /** + * @return the class body, before update with "$CLASS_NAME$" replacement and package name prefixing + */ + public abstract String classBody(); + + /** + * @return the package name prefix + */ + public abstract String packageNameRoot(); + + /** Optional "log" for final class code. */ + public abstract Optional codeLog(); + + /** + * @return the generic parameters, empty if none required + */ + public abstract Map> parameterClasses(); + + String getPackageName(final String packageNameSuffix) { + final String root = packageNameRoot(); + return root.isEmpty() + ? packageNameSuffix + : root + (root.endsWith(".") ? "" : ".") + packageNameSuffix; + } + + public interface Builder { + Builder description(String description); + + Builder className(String className); + + Builder classBody(String classBody); + + Builder packageNameRoot(String packageNameRoot); + + Builder codeLog(StringBuilder codeLog); + + Builder putAllParameterClasses(Map> parameterClasses); + + QueryCompilerRequest build(); + } +} diff --git a/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java b/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java new file mode 100644 index 00000000000..80a4d0c1667 --- /dev/null +++ b/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java @@ -0,0 +1,132 @@ +package io.deephaven.engine.context.util; + +import javax.tools.FileObject; +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Iterator; +import java.util.ServiceLoader; +import java.util.Set; + +public class SynchronizedJavaFileManager implements JavaFileManager { + + private final JavaFileManager delegate; + + public SynchronizedJavaFileManager(JavaFileManager delegate) { + this.delegate = delegate; + } + + @Override + public synchronized ClassLoader getClassLoader(Location location) { + return delegate.getClassLoader(location); + } + + @Override + public synchronized Iterable list( + Location location, + String packageName, + Set kinds, + boolean recurse) throws IOException { + return delegate.list(location, packageName, kinds, recurse); + } + + @Override + public synchronized String inferBinaryName(Location location, JavaFileObject file) { + return delegate.inferBinaryName(location, file); + } + + @Override + public synchronized boolean isSameFile(FileObject a, FileObject b) { + return delegate.isSameFile(a, b); + } + + @Override + public synchronized boolean handleOption(String current, Iterator remaining) { + return delegate.handleOption(current, remaining); + } + + @Override + public synchronized boolean hasLocation(Location location) { + return delegate.hasLocation(location); + } + + @Override + public synchronized JavaFileObject getJavaFileForInput( + Location location, + String className, + JavaFileObject.Kind kind) throws IOException { + return delegate.getJavaFileForInput(location, className, kind); + } + + @Override + public synchronized JavaFileObject getJavaFileForOutput( + Location location, + String className, + JavaFileObject.Kind kind, + FileObject sibling) throws IOException { + return delegate.getJavaFileForOutput(location, className, kind, sibling); + } + + @Override + public synchronized FileObject getFileForInput( + Location location, + String packageName, + String relativeName) throws IOException { + return delegate.getFileForInput(location, packageName, relativeName); + } + + @Override + public synchronized FileObject getFileForOutput( + Location location, + String packageName, + String relativeName, + FileObject sibling) throws IOException { + return delegate.getFileForOutput(location, packageName, relativeName, sibling); + } + + @Override + public synchronized void flush() throws IOException { + delegate.flush(); + } + + @Override + public synchronized void close() throws IOException { + delegate.close(); + } + + @Override + public synchronized int isSupportedOption(String option) { + return delegate.isSupportedOption(option); + } + + @Override + public synchronized Location getLocationForModule(Location location, String moduleName) throws IOException { + return delegate.getLocationForModule(location, moduleName); + } + + @Override + public synchronized Location getLocationForModule(Location location, JavaFileObject fo) throws IOException { + return delegate.getLocationForModule(location, fo); + } + + @Override + public synchronized ServiceLoader getServiceLoader(Location location, Class service) throws IOException { + return delegate.getServiceLoader(location, service); + } + + @Override + public synchronized String inferModuleName(Location location) throws IOException { + return delegate.inferModuleName(location); + } + + @Override + public synchronized Iterable> listLocationsForModules(Location location) throws IOException { + return delegate.listLocationsForModules(location); + } + + @Override + public synchronized boolean contains(Location location, FileObject fo) throws IOException { + return delegate.contains(location, fo); + } +} diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index 461e2ae232d..aa00c48473d 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -3,9 +3,13 @@ // package io.deephaven.engine.context; +import io.deephaven.UncheckedDeephavenException; +import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.time.DateTimeUtils; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.CompletionStageFutureImpl; import io.deephaven.util.SafeCloseable; import org.junit.After; import org.junit.Before; @@ -18,8 +22,8 @@ import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; public class TestQueryCompiler { private final static int NUM_THREADS = 500; @@ -165,7 +169,7 @@ private void sleepIgnoringInterruptions(final long waitMillis) { } } - private void compile(boolean printDetails, final String className) throws Exception { + private void compile(boolean printDetails, final String className) { final long startMillis; if (printDetails) { startMillis = System.currentTimeMillis(); @@ -173,8 +177,13 @@ private void compile(boolean printDetails, final String className) throws Except } else { startMillis = 0; } - ExecutionContext.getContext().getQueryCompiler() - .compile(className, CLASS_CODE, "io.deephaven.temp"); + ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className(className) + .classBody(CLASS_CODE) + .packageNameRoot("io.deephaven.temp") + .build()); if (printDetails) { final long endMillis = System.currentTimeMillis(); System.out.println(printMillis(endMillis) + ": Thread 0 ending compile: (" + (endMillis - startMillis) @@ -201,8 +210,14 @@ public void testSimpleCompile() throws Exception { "}"); StringBuilder codeLog = new StringBuilder(); - final Class clazz1 = ExecutionContext.getContext().getQueryCompiler() - .compile("Test", program1Text, "com.deephaven.test", codeLog, Collections.emptyMap()); + final Class clazz1 = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className("Test") + .classBody(program1Text) + .packageNameRoot("com.deephaven.test") + .codeLog(codeLog) + .build()); final Method m1 = clazz1.getMethod("main", String[].class); Object[] args1 = new Object[] {new String[] {"hello", "there"}}; m1.invoke(null, args1); @@ -224,21 +239,84 @@ public void testCollidingCompile() throws Exception { Thread t = new Thread(() -> { StringBuilder codeLog = new StringBuilder(); try { - final Class clazz1 = ExecutionContext.getContext().getQueryCompiler() - .compile("Test", program1Text, "com.deephaven.test", codeLog, - Collections.emptyMap()); + final Class clazz1 = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className("Test") + .classBody(program1Text) + .packageNameRoot("com.deephaven.test") + .codeLog(codeLog) + .build()); final Method m1 = clazz1.getMethod("main", String[].class); Object[] args1 = new Object[] {new String[] {"hello", "there"}}; m1.invoke(null, args1); } catch (Exception e) { - throw new RuntimeException(e); + throw new UncheckedDeephavenException(e); } }); t.start(); threads.add(t); } - for (int i = 0; i < threads.size(); ++i) { - threads.get(i).join(); + for (final Thread thread : threads) { + thread.join(); + } + } + + @Test + public void testMultiCompileWithFailure() throws ExecutionException, InterruptedException { + final String goodProgram = String.join( + "\n", + "public class GoodTest {", + " public static void main (String [] args) {", + " }", + "}"); + final String badProgram = String.join( + "\n", + "public class BadTest {", + " public static void main (String [] args) {", + " }", + "}}"); + + QueryCompilerRequest[] requests = new QueryCompilerRequest[] { + QueryCompilerRequest.builder() + .description("Test Bad Compile") + .className("BadTest") + .classBody(badProgram) + .packageNameRoot("com.deephaven.test") + .build(), + QueryCompilerRequest.builder() + .description("Test Good Compile") + .className("GoodTest") + .classBody(goodProgram) + .packageNameRoot("com.deephaven.test") + .build(), + }; + + // noinspection unchecked + CompletionStageFuture.Resolver>[] resolvers = + (CompletionStageFuture.Resolver>[]) new CompletionStageFuture.Resolver[] { + CompletionStageFutureImpl.make(), + CompletionStageFutureImpl.make(), + }; + + Exception firstErr; + try { + ExecutionContext.getContext().getQueryCompiler().compile(requests, resolvers); + // noinspection DataFlowIssue + throw Assert.statementNeverExecuted(); + } catch (Exception err) { + firstErr = err; + } + + Assert.eqTrue(resolvers[0].getFuture().isDone(), "resolvers[0].getFuture().isDone()"); + Assert.eqTrue(resolvers[1].getFuture().isDone(), "resolvers[0].getFuture().isDone()"); + Assert.neqNull(resolvers[1].getFuture().get(), "resolvers[1].getFuture().get()"); + try { + resolvers[0].getFuture().get(); + // noinspection DataFlowIssue + throw Assert.statementNeverExecuted(); + } catch (ExecutionException err) { + Assert.eq(firstErr, "firstErr", err.getCause(), "err"); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java index 1645a88ab3a..c87b539058a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java @@ -20,6 +20,7 @@ import java.util.*; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -46,16 +47,20 @@ public DeferredViewTable(@NotNull final TableDefinition definition, this.deferredViewColumns = deferredViewColumns == null ? SelectColumn.ZERO_LENGTH_SELECT_COLUMN_ARRAY : deferredViewColumns; final TableDefinition parentDefinition = tableReference.getDefinition(); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); SelectAndViewAnalyzer.initializeSelectColumns( - parentDefinition.getColumnNameMap(), this.deferredViewColumns); + parentDefinition.getColumnNameMap(), variableSupplier, this.deferredViewColumns, compilationProcessor); this.deferredFilters = deferredFilters == null ? WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY : deferredFilters; for (final WhereFilter sf : this.deferredFilters) { - sf.init(parentDefinition); + sf.init(parentDefinition, variableSupplier, compilationProcessor); if (sf instanceof LivenessReferent && sf.isRefreshing()) { manage((LivenessReferent) sf); setRefreshing(true); } } + compilationProcessor.compile(); // we really only expect one of these things to be set! final boolean haveDrop = this.deferredDropColumns.length > 0; @@ -78,9 +83,13 @@ public DeferredViewTable(@NotNull final TableDefinition definition, @Override public Table where(Filter filter) { final WhereFilter[] whereFilters = WhereFilter.fromInternal(filter); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (WhereFilter f : whereFilters) { - f.init(definition); + f.init(definition, variableSupplier, compilationProcessor); } + compilationProcessor.compile(); return getResultTableWithWhere(whereFilters); } @@ -189,8 +198,11 @@ private PreAndPostFilters applyFilterRenamings(WhereFilter[] filters) { } } + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (final WhereFilter filter : filters) { - filter.init(definition); + filter.init(definition, variableSupplier, compilationProcessor); final boolean isPostView = Stream.of(filter.getColumns(), filter.getColumnArrays()) .flatMap(Collection::stream) @@ -220,6 +232,7 @@ private PreAndPostFilters applyFilterRenamings(WhereFilter[] filters) { postViewFilters.add(filter); } } + compilationProcessor.compile(); return new PreAndPostFilters(preViewFilters.toArray(WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY), postViewFilters.toArray(WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY)); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java index 544c3ccb25a..f219651845a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java @@ -8,6 +8,7 @@ import io.deephaven.api.filter.Filter; import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.updategraph.UpdateSourceRegistrar; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.util.TableTools; @@ -24,6 +25,7 @@ import java.util.*; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -151,16 +153,20 @@ protected TableAndRemainingFilters getWithWhere(WhereFilter... whereFilters) { @Override public Table selectDistinctInternal(Collection columns) { final List selectColumns = Arrays.asList(SelectColumn.from(columns)); + try { + SelectAndViewAnalyzer.initializeSelectColumns(table.getDefinition().getColumnNameMap(), + selectColumns.toArray(SelectColumn[]::new)); + } catch (Exception e) { + return null; + } + + final Set newColumns = new HashSet<>(); for (final SelectColumn selectColumn : selectColumns) { - try { - selectColumn.initDef(getDefinition().getColumnNameMap()); - } catch (Exception e) { - return null; - } - if (!((PartitionAwareSourceTable) table).isValidAgainstColumnPartitionTable(selectColumn.getColumns(), - selectColumn.getColumnArrays())) { + if (!((PartitionAwareSourceTable) table).isValidAgainstColumnPartitionTable( + selectColumn.getColumns(), selectColumn.getColumnArrays(), newColumns)) { return null; } + newColumns.add(selectColumn.getName()); } return table.selectDistinct(selectColumns); } @@ -280,8 +286,11 @@ private Table whereImpl(final WhereFilter[] whereFilters) { Set groupingColumnNames = groupingColumns.stream().map(ColumnDefinition::getName).collect(Collectors.toSet()); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition); + whereFilter.init(definition, variableSupplier, compilationProcessor); List columns = whereFilter.getColumns(); if (whereFilter instanceof ReindexingFilter) { otherFilters.add(whereFilter); @@ -294,6 +303,7 @@ private Table whereImpl(final WhereFilter[] whereFilters) { otherFilters.add(whereFilter); } } + compilationProcessor.compile(); // if there was nothing that actually required the partition, defer the result. if (partitionFilters.isEmpty()) { @@ -323,13 +333,18 @@ private Table whereImpl(final WhereFilter[] whereFilters) { @Override public final Table selectDistinct(Collection columns) { final List selectColumns = Arrays.asList(SelectColumn.from(columns)); + SelectAndViewAnalyzer.initializeSelectColumns( + definition.getColumnNameMap(), selectColumns.toArray(SelectColumn[]::new)); + + final Set newColumns = new HashSet<>(); for (SelectColumn selectColumn : selectColumns) { - selectColumn.initDef(definition.getColumnNameMap()); - if (!isValidAgainstColumnPartitionTable(selectColumn.getColumns(), selectColumn.getColumnArrays())) { + if (!isValidAgainstColumnPartitionTable( + selectColumn.getColumns(), selectColumn.getColumnArrays(), newColumns)) { // Be sure to invoke the super-class version of this method, rather than the array-based one that // delegates to this method. return super.selectDistinct(selectColumns); } + newColumns.add(selectColumn.getName()); } initializeAvailableLocations(); final List existingLocationKeys = @@ -352,15 +367,20 @@ public final Table selectDistinct(Collection columns) { // Needs lazy region allocation. } - private boolean isValidAgainstColumnPartitionTable(@NotNull final Collection columnNames, + private boolean isValidAgainstColumnPartitionTable( + @NotNull final Collection columnNames, @NotNull final Collection columnArrayNames) { - if (columnArrayNames.size() > 0) { - return false; - } - return columnNames.stream().allMatch(partitioningColumnDefinitions::containsKey); + return isValidAgainstColumnPartitionTable(columnNames, columnArrayNames, Collections.emptySet()); } - private boolean isValidAgainstColumnPartitionTable(Collection columns) { - return columns.stream().map(ColumnName::name).allMatch(partitioningColumnDefinitions::containsKey); + private boolean isValidAgainstColumnPartitionTable( + @NotNull final Collection columnNames, + @NotNull final Collection columnArrayNames, + @NotNull final Collection newColumns) { + if (!columnArrayNames.isEmpty()) { + return false; + } + return columnNames.stream().allMatch( + columnName -> partitioningColumnDefinitions.containsKey(columnName) || newColumns.contains(columnName)); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java new file mode 100644 index 00000000000..20518ba64fe --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.table.impl; + +import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; +import io.deephaven.util.SafeCloseable; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.CompletionStageFutureImpl; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +public interface QueryCompilerRequestProcessor { + /** + * Submit a request for compilation. The QueryCompilerRequestProcessor is not required to immediately compile this + * request. + * + * @param request the request to compile + */ + CompletionStageFuture> submit(@NotNull QueryCompilerRequest request); + + /** + * A QueryCompilerRequestProcessor that immediately compiles requests. + */ + class ImmediateProcessor implements QueryCompilerRequestProcessor { + public static final ImmediateProcessor INSTANCE = new ImmediateProcessor(); + + @Override + public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { + final String desc = "Compile " + request.description(); + final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(desc)) { + ExecutionContext.getContext().getQueryCompiler().compile(request, resolver); + } + return resolver.getFuture(); + } + } + + /** + * A QueryCompilerRequestProcessor that batches requests and compiles them all at once. + *

+ * The compile method must be called to actually compile the requests. + */ + class BatchProcessor implements QueryCompilerRequestProcessor { + private final List requests = new ArrayList<>(); + private final List>> resolvers = new ArrayList<>(); + + @Override + public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { + final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + requests.add(request); + resolvers.add(resolver); + return resolver.getFuture(); + } + + /** + * Compile all the requests that have been submitted. + */ + public void compile() { + if (requests.isEmpty()) { + return; + } + + final String desc; + if (requests.size() == 1) { + desc = "Compile: " + requests.get(0).description(); + } else { + final StringBuilder descriptionBuilder = new StringBuilder(); + descriptionBuilder.append("Batch Compile of ").append(requests.size()).append(" requests:\n"); + for (final QueryCompilerRequest request : requests) { + descriptionBuilder.append('\t').append(request.description()).append('\n'); + } + desc = descriptionBuilder.toString(); + } + + try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(desc)) { + final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); + if (requests.size() == 1) { + compiler.compile(requests.get(0), resolvers.get(0)); + } else { + compiler.compile( + requests.toArray(QueryCompilerRequest[]::new), + resolvers.toArray(CompletionStageFuture.Resolver[]::new)); + } + } + } + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java index ad6115905c0..7acd8e9187a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java @@ -79,7 +79,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -1187,8 +1186,12 @@ private QueryTable whereInternal(final WhereFilter... filters) { List selectFilters = new LinkedList<>(); List>>> shiftColPairs = new LinkedList<>(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + final Supplier> variableSupplier = + SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); for (final WhereFilter filter : filters) { - filter.init(getDefinition()); + filter.init(getDefinition(), variableSupplier, compilationProcessor); if (filter instanceof AbstractConditionFilter && ((AbstractConditionFilter) filter).hasConstantArrayAccess()) { shiftColPairs.add(((AbstractConditionFilter) filter).getFormulaShiftColPair()); @@ -1196,6 +1199,7 @@ private QueryTable whereInternal(final WhereFilter... filters) { selectFilters.add(filter); } } + compilationProcessor.compile(); if (!shiftColPairs.isEmpty()) { return (QueryTable) ShiftedColumnsFactory.where(this, shiftColPairs, selectFilters); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java index 206121ab899..684cb4e8a2d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java @@ -9,6 +9,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.select.*; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import org.jetbrains.annotations.NotNull; import java.util.*; @@ -49,8 +50,12 @@ private Table viewInternal(Collection selectables, boolean final Map> resultColumnsExternal = new LinkedHashMap<>(); final Map> allColumns = new HashMap<>(definition.getColumnNameMap()); boolean simpleRetain = true; + + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (final SelectColumn selectColumn : columns) { - List usedColumnNames = selectColumn.initDef(allColumns); + List usedColumnNames = selectColumn.initDef( + allColumns, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), compilationProcessor); usedColumnNames.addAll(selectColumn.getColumnArrays()); resultColumnsInternal.addAll(usedColumnNames.stream() .filter(usedColumnName -> !resultColumnsExternal.containsKey(usedColumnName)) @@ -65,6 +70,7 @@ private Table viewInternal(Collection selectables, boolean resultColumnsExternal.put(selectColumn.getName(), columnDef); allColumns.put(selectColumn.getName(), columnDef); } + compilationProcessor.compile(); TableDefinition newDefExternal = TableDefinition.of( resultColumnsExternal.values().toArray(ColumnDefinition.ZERO_LENGTH_COLUMN_DEFINITION_ARRAY)); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java index 49766528990..97ee9353f62 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java @@ -10,6 +10,7 @@ import io.deephaven.engine.rowset.*; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.liveness.LivenessArtifact; import io.deephaven.engine.table.impl.select.WhereFilter; @@ -21,6 +22,7 @@ import org.jetbrains.annotations.Nullable; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -91,9 +93,18 @@ public Result initialize(boolean usePrev, long beforeClock) { final List dependencies = new ArrayList<>(); final Map> newColumns = new LinkedHashMap<>(parent.getColumnSourceMap()); - matchColumns.forEach(holder -> { - final WhereFilter filter = holder.getFilter(); - filter.init(parent.getDefinition()); + final Supplier> variableSupplier = + SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + final WhereFilter[] filters = matchColumns.stream().map(ColumnHolder::getFilter) + .peek(holder -> holder.init(parent.getDefinition(), variableSupplier, compilationProcessor)) + .toArray(WhereFilter[]::new); + compilationProcessor.compile(); + + for (int ii = 0; ii < filters.length; ++ii) { + final ColumnHolder holder = matchColumns.get(ii); + final WhereFilter filter = filters[ii]; final WritableRowSet result = filter.filter(fullRowSet, fullRowSet, parent, usePrev); holder.column = new IndexWrapperColumnSource( holder.getColumnName(), parent, result.toTracking(), filter); @@ -114,7 +125,7 @@ public Result initialize(boolean usePrev, long beforeClock) { if (filter.isRefreshing()) { anyRefreshing.setTrue(); } - }); + } this.resultTable = new QueryTable(parent.getRowSet(), newColumns); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java index 80464ec1751..505955435d6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java @@ -38,11 +38,8 @@ import io.deephaven.engine.table.ChunkSource; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.impl.BaseTable; -import io.deephaven.engine.table.impl.QueryTable; -import io.deephaven.engine.table.impl.TupleSourceFactory; import io.deephaven.engine.table.impl.by.rollup.NullColumns; import io.deephaven.engine.table.impl.by.rollup.RollupAggregation; import io.deephaven.engine.table.impl.by.rollup.RollupAggregationOutputs; @@ -96,6 +93,7 @@ import io.deephaven.engine.table.impl.by.ssmcountdistinct.unique.ShortRollupUniqueOperator; import io.deephaven.engine.table.impl.by.ssmminmax.SsmChunkedMinMaxOperator; import io.deephaven.engine.table.impl.by.ssmpercentile.SsmChunkedPercentileOperator; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.ssms.SegmentedSortedMultiSet; import io.deephaven.engine.table.impl.util.freezeby.FreezeByCountOperator; @@ -282,7 +280,13 @@ public AggregationContext makeAggregationContext( @NotNull final String... groupByColumnNames) { switch (type) { case NORMAL: - return new NormalConverter(table, requireStateChangeRecorder, groupByColumnNames).build(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + final AggregationContext aggContext = new NormalConverter( + table, requireStateChangeRecorder, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), + compilationProcessor, groupByColumnNames).build(); + compilationProcessor.compile(); + return aggContext; case ROLLUP_BASE: return new RollupBaseConverter(table, requireStateChangeRecorder, groupByColumnNames).build(); case ROLLUP_REAGGREGATED: @@ -664,12 +668,18 @@ final void addWeightedAvgOrSumOperator( * {@link AggregationContext} for standard aggregations. Accumulates state by visiting each aggregation. */ private final class NormalConverter extends Converter { + private final Supplier> queryScopeVariables; + private final QueryCompilerRequestProcessor compilationProcessor; private NormalConverter( @NotNull final Table table, final boolean requireStateChangeRecorder, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor, @NotNull final String... groupByColumnNames) { super(table, requireStateChangeRecorder, groupByColumnNames); + this.queryScopeVariables = queryScopeVariables; + this.compilationProcessor = compilationProcessor; } // ------------------------------------------------------------------------------------------------------------- @@ -744,7 +754,8 @@ public void visit(@NotNull final AggSpecFormula formula) { final GroupByChunkedOperator groupByChunkedOperator = new GroupByChunkedOperator(table, false, null, resultPairs.stream().map(pair -> MatchPair.of((Pair) pair.input())).toArray(MatchPair[]::new)); final FormulaChunkedOperator formulaChunkedOperator = new FormulaChunkedOperator(groupByChunkedOperator, - true, formula.formula(), formula.paramToken(), MatchPair.fromPairs(resultPairs)); + true, formula.formula(), formula.paramToken(), queryScopeVariables, compilationProcessor, + MatchPair.fromPairs(resultPairs)); addNoInputOperator(formulaChunkedOperator); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java index 3bf31cb5e8b..d076711b5b9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.*; import io.deephaven.engine.table.ChunkSource.GetContext; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaUtil; import io.deephaven.engine.liveness.LivenessReferent; import io.deephaven.engine.table.ModifiedColumnSet; @@ -29,6 +30,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import static io.deephaven.engine.table.impl.sources.ArrayBackedColumnSource.BLOCK_SIZE; @@ -69,10 +71,13 @@ class FormulaChunkedOperator implements IterativeChunkedAggregationOperator { * @param columnParamName The token to substitute column names for * @param resultColumnPairs The names for formula input and result columns */ - FormulaChunkedOperator(@NotNull final GroupByChunkedOperator groupBy, + FormulaChunkedOperator( + @NotNull final GroupByChunkedOperator groupBy, final boolean delegateToBy, @NotNull final String formula, @NotNull final String columnParamName, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor, @NotNull final MatchPair... resultColumnPairs) { this.groupBy = groupBy; this.delegateToBy = delegateToBy; @@ -95,9 +100,10 @@ class FormulaChunkedOperator implements IterativeChunkedAggregationOperator { final ColumnDefinition inputColumnDefinition = ColumnDefinition .fromGenericType(inputColumnName, inputColumnSource.getType(), inputColumnSource.getComponentType()); - formulaColumn.initDef(Collections.singletonMap(inputColumnName, inputColumnDefinition)); - // noinspection unchecked - resultColumns[ci] = ArrayBackedColumnSource.getMemoryColumnSource(0, formulaColumn.getReturnedType()); + formulaColumn.initDef(Collections.singletonMap(inputColumnName, inputColumnDefinition), + queryScopeVariables, compilationProcessor); + resultColumns[ci] = ArrayBackedColumnSource.getMemoryColumnSource( + 0, formulaColumn.getReturnedType(), formulaColumn.getReturnedComponentType()); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java index 6ff7f0460fe..bc007401aec 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java @@ -11,6 +11,7 @@ import io.deephaven.chunk.util.hashing.CharChunkHasher; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetBuilderRandom; @@ -19,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.impl.MultiJoinModifiedSlotTracker; import io.deephaven.engine.table.impl.NaturalJoinModifiedSlotTracker; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.asofjoin.RightIncrementalAsOfJoinStateManagerTypedBase; import io.deephaven.engine.table.impl.asofjoin.StaticAsOfJoinStateManagerTypedBase; import io.deephaven.engine.table.impl.asofjoin.TypedAsOfJoinFactory; @@ -566,12 +568,18 @@ public static T make(HasherConfig hasherConfig, ColumnSource[] tableKe final String javaString = Arrays.stream(javaStrings).filter(s -> !s.startsWith("package ")).collect(Collectors.joining("\n")); - final Class clazz = ExecutionContext.getContext().getQueryCompiler().compile(className, javaString, - "io.deephaven.engine.table.impl.by.typed." + hasherConfig.packageMiddle + ".gen"); + final Class clazz = ExecutionContext.getContext().getQueryCompiler().compile(QueryCompilerRequest.builder() + .description("TypedHasherFactory: " + className) + .className(className) + .classBody(javaString) + .packageNameRoot("io.deephaven.engine.table.impl.by.typed." + hasherConfig.packageMiddle + ".gen") + .build()); + if (!hasherConfig.baseClass.isAssignableFrom(clazz)) { throw new IllegalStateException("Generated class is not a " + hasherConfig.baseClass.getCanonicalName()); } + // noinspection unchecked final Class castedClass = (Class) clazz; T retVal; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java index 9b943837bbe..61a0b855cfc 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java @@ -12,10 +12,7 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.hierarchical.RollupTable; import io.deephaven.engine.table.hierarchical.TreeTable; -import io.deephaven.engine.table.impl.AbsoluteSortColumnConventions; -import io.deephaven.engine.table.impl.NoSuchColumnException; -import io.deephaven.engine.table.impl.QueryTable; -import io.deephaven.engine.table.impl.TableAdapter; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; @@ -284,16 +281,24 @@ private Stream absoluteSelectColumns() { // custom columns in the future. For now, we've plumbed absolute column value sorting via naming // conventions. Note that we simply avoid telling the client about these columns when sending schemas, so we // have no need to drop them post-sort. - return sortColumns.stream() + + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + + final SelectColumn[] columns = sortColumns.stream() .map(sc -> sc.column().name()) .filter(AbsoluteSortColumnConventions::isAbsoluteColumnName) .map(cn -> { final String baseColumnName = AbsoluteSortColumnConventions.absoluteColumnNameToBaseName(cn); final Selectable selectable = AbsoluteSortColumnConventions.makeSelectable(cn, baseColumnName); final SelectColumn selectColumn = SelectColumn.of(selectable); - selectColumn.initDef(Map.of(baseColumnName, getDefinition().getColumn(baseColumnName))); + selectColumn.initDef(Map.of(baseColumnName, getDefinition().getColumn(baseColumnName)), + SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), compilationProcessor); return selectColumn; - }); + }).toArray(SelectColumn[]::new); + + compilationProcessor.compile(); + return Stream.of(columns); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java index 03c7d193274..ea0814d525d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java @@ -19,12 +19,14 @@ import io.deephaven.engine.table.hierarchical.RollupTable; import io.deephaven.engine.table.impl.BaseTable.CopyAttributeOperation; import io.deephaven.engine.table.impl.NotificationStepSource; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.SortOperation; import io.deephaven.engine.table.impl.by.AggregationProcessor; import io.deephaven.engine.table.impl.by.AggregationRowLookup; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.WhereFilter; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; import io.deephaven.engine.table.impl.util.RowRedirection; import io.deephaven.util.type.TypeUtils; @@ -35,6 +37,7 @@ import java.util.*; import java.util.function.Function; import java.util.function.LongUnaryOperator; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -275,8 +278,11 @@ public static WhereFilter[] initializeAndValidateFilters( @NotNull final Collection filters, @NotNull final Function exceptionFactory) { final WhereFilter[] whereFilters = WhereFilter.from(filters); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (final WhereFilter whereFilter : whereFilters) { - whereFilter.init(source.getDefinition()); + whereFilter.init(source.getDefinition(), variableSupplier, compilationProcessor); final List invalidColumnsUsed = whereFilter.getColumns().stream().map(ColumnName::of) .filter(cn -> !groupByColumns.contains(cn)).map(ColumnName::name).collect(Collectors.toList()); if (!invalidColumnsUsed.isEmpty()) { @@ -290,6 +296,8 @@ public static WhereFilter[] initializeAndValidateFilters( + " may not use column arrays, but uses column arrays from " + whereFilter.getColumnArrays()); } } + compilationProcessor.compile(); + return whereFilters; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java index 9fdfc15138a..d28a469286b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java @@ -9,14 +9,17 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.hierarchical.TreeTable; import io.deephaven.engine.table.hierarchical.TreeTable.NodeOperationsRecorder; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.WhereFilter; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -137,7 +140,16 @@ public Table where(Filter filter) { } private Stream whereFilters() { - return Stream.of(WhereFilter.fromInternal(filter)).peek(wf -> wf.init(getDefinition())); + final Supplier> variableSupplier = + SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + final WhereFilter[] filters = WhereFilter.fromInternal(filter); + for (final WhereFilter filter : filters) { + filter.init(getDefinition(), variableSupplier, compilationProcessor); + } + compilationProcessor.compile(); + return Stream.of(filters); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java index e40cd841662..779765c5c8e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java @@ -19,6 +19,7 @@ import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.chunkboxer.ChunkBoxer; import io.deephaven.engine.table.impl.select.WhereFilter; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.ArrayBackedColumnSource; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.util.SafeCloseable; @@ -31,6 +32,7 @@ import java.util.*; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Stream; /** @@ -151,7 +153,12 @@ private TreeTableFilter(@NotNull final TreeTableImpl tree, @NotNull final WhereF parentIdColumnName = tree.getParentIdentifierColumn(); sourceRowLookup = tree.getSourceRowLookup(); this.filters = filters; - Arrays.stream(filters).forEach((final WhereFilter filter) -> filter.init(source.getDefinition())); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + Arrays.stream(filters).forEach((final WhereFilter filter) -> filter.init(source.getDefinition(), + variableSupplier, compilationProcessor)); + compilationProcessor.compile(); idSource = source.getColumnSource(tree.getIdentifierColumn().name()); parentIdSource = source.getColumnSource(tree.getParentIdentifierColumn().name()); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java index dfb0130594c..ce5d62cb06f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.impl.by.AggregationProcessor; import io.deephaven.engine.table.impl.by.AggregationRowLookup; import io.deephaven.engine.table.impl.select.WhereFilter; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.NotNull; @@ -23,6 +24,7 @@ import java.util.*; import java.util.function.LongUnaryOperator; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -154,13 +156,18 @@ public TreeTable withFilter(@NotNull Filter filter) { if (whereFilters.length == 0) { return noopResult(); } + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); final Map> nodeSuitabilityToFilters = Stream.of(whereFilters) - .peek(wf -> wf.init(source.getDefinition())) + .peek(wf -> wf.init(source.getDefinition(), variableSupplier, compilationProcessor)) .collect(Collectors.partitioningBy(wf -> { // Node-level filters have only node-filter columns and use no column arrays return wf.getColumns().stream().map(ColumnName::of).allMatch(nodeFilterColumns::contains) && wf.getColumnArrays().isEmpty(); })); + compilationProcessor.compile(); + final List nodeFilters = nodeSuitabilityToFilters.get(true); final List sourceFilters = nodeSuitabilityToFilters.get(false); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java index 507f27a88fd..6d854abef0b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java @@ -28,6 +28,12 @@ public final Class getReturnedType() { return Table.class; } + @Override + public Class getReturnedComponentType() { + // Table does not have a component type + return null; + } + @Override public final List getColumnArrays() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java index 2dfe17bd3eb..6111dbf6cea 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java @@ -11,6 +11,7 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -21,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.function.BinaryOperator; +import java.util.function.Supplier; /** * {@link SelectColumn} implementation to wrap transformer functions for {@link PartitionedTable#partitionedTransform @@ -57,7 +59,10 @@ public List initInputs( } @Override - public List initDef(@NotNull final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { validateInputColumnDefinition(inputOutputColumnName, columnDefinitionMap); validateInputColumnDefinition(secondInputColumnName, columnDefinitionMap); return getColumns(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java index 453be979a3d..a12dd7ace2f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java @@ -10,6 +10,7 @@ import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.LongSingleValueSource; @@ -20,6 +21,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * {@link SelectColumn} implementation to assign a constant {@code long} value. @@ -44,7 +46,10 @@ public List initInputs( } @Override - public List initDef(@NotNull final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { return getColumns(); } @@ -74,6 +79,12 @@ public final Class getReturnedType() { return long.class; } + @Override + public Class getReturnedComponentType() { + // long does not have a component type + return null; + } + @Override public final List getColumnArrays() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java index 95751fba30e..591447a0cd6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java @@ -19,13 +19,11 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.*; -import io.deephaven.engine.table.impl.BaseTable; -import io.deephaven.engine.table.impl.MatchPair; -import io.deephaven.engine.table.impl.MemoizedOperationKey; -import io.deephaven.engine.table.impl.QueryTable; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.engine.table.impl.select.MatchFilter; import io.deephaven.engine.table.impl.select.WhereFilter; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; import io.deephaven.engine.table.impl.sources.UnionSourceManager; import io.deephaven.engine.table.iterators.ChunkedObjectColumnIterator; @@ -42,6 +40,7 @@ import java.lang.ref.WeakReference; import java.util.*; import java.util.function.BinaryOperator; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -235,10 +234,14 @@ private Map computeSharedAttributes(@NotNull final Iterator filters) { final WhereFilter[] whereFilters = WhereFilter.from(filters); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); final boolean invalidFilter = Arrays.stream(whereFilters).flatMap((final WhereFilter filter) -> { - filter.init(table.getDefinition()); + filter.init(table.getDefinition(), variableSupplier, compilationProcessor); return Stream.concat(filter.getColumns().stream(), filter.getColumnArrays().stream()); }).anyMatch((final String columnName) -> columnName.equals(constituentColumnName)); + compilationProcessor.compile(); if (invalidFilter) { throw new IllegalArgumentException("Unsupported filter against constituent column " + constituentColumnName + " found in filters: " + filters); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java index 24f9111bf77..dbf38fa2359 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java @@ -32,6 +32,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BinaryOperator; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -464,9 +465,13 @@ public PartitionedTable.Proxy sort(Collection columnsToSortBy) { public PartitionedTable.Proxy where(Filter filter) { final WhereFilter[] whereFilters = WhereFilter.fromInternal(filter); final TableDefinition definition = target.constituentDefinition(); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition); + whereFilter.init(definition, variableSupplier, compilationProcessor); } + compilationProcessor.compile(); return basicTransform(ct -> ct.where(Filter.and(WhereFilter.copyFrom(whereFilters)))); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java index cf796fe5852..fa3c7c24c32 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java @@ -12,6 +12,7 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -21,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.function.Supplier; /** * {@link SelectColumn} implementation to wrap transformer functions for {@link PartitionedTable#transform @@ -52,7 +54,10 @@ public List initInputs( } @Override - public List initDef(@NotNull final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { validateInputColumnDefinition(inputOutputColumnName, columnDefinitionMap); return getColumns(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java index 686e4ab7130..f73f3e21c68 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java @@ -17,13 +17,16 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.table.impl.select.WhereFilter; import io.deephaven.engine.table.impl.select.WhereFilterImpl; import org.jetbrains.annotations.NotNull; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_DOUBLE; import static io.deephaven.util.QueryConstants.NULL_FLOAT; @@ -50,7 +53,10 @@ public List getColumnArrays() { } @Override - public void init(@NotNull final TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeSupplier, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { final ColumnDefinition columnDefinition = tableDefinition.getColumn(columnName.name()); if (columnDefinition == null) { throw new IllegalArgumentException(String.format("Missing expected input column %s", diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java index 95098cdb5b4..0cd05704618 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java @@ -3,35 +3,30 @@ // package io.deephaven.engine.table.impl.select; -import io.deephaven.api.util.NameValidator; import io.deephaven.base.Pair; -import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.WritableRowSet; -import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; +import io.deephaven.engine.table.impl.select.codegen.FormulaAnalyzer; import io.deephaven.engine.table.impl.select.python.ArgumentsChunked; import io.deephaven.engine.table.impl.select.python.DeephavenCompatibleFunction; import io.deephaven.engine.util.PyCallableWrapperJpyImpl; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.vector.ObjectVector; import org.jetbrains.annotations.NotNull; import org.jpy.PyObject; import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; import java.net.MalformedURLException; import java.util.*; -import java.util.function.BiConsumer; +import java.util.function.Supplier; import java.util.stream.Collectors; import static io.deephaven.engine.table.impl.select.DhFormulaColumn.COLUMN_SUFFIX; @@ -80,89 +75,22 @@ public List getColumnArrays() { } @Override - public synchronized void init(TableDefinition tableDefinition) { + public synchronized void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (initialized) { return; } - final Map> possibleVariables = new HashMap<>(); - possibleVariables.put("i", int.class); - possibleVariables.put("ii", long.class); - possibleVariables.put("k", long.class); - - final Map[]> possibleVariableParameterizedTypes = new HashMap<>(); - try { - final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); - final Map queryScopeVariables = queryScope.toMap( - (name, value) -> NameValidator.isValidQueryParameterName(name)); - for (Map.Entry param : queryScopeVariables.entrySet()) { - possibleVariables.put(param.getKey(), QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); - Type declaredType = QueryScopeParamTypeUtil.getDeclaredType(param.getValue()); - if (declaredType instanceof ParameterizedType) { - ParameterizedType pt = (ParameterizedType) declaredType; - Class[] paramTypes = Arrays.stream(pt.getActualTypeArguments()) - .map(QueryScopeParamTypeUtil::classFromType) - .toArray(Class[]::new); - possibleVariableParameterizedTypes.put(param.getKey(), paramTypes); - } - } - - final Set columnVariables = new HashSet<>(); - columnVariables.add("i"); - columnVariables.add("ii"); - columnVariables.add("k"); - - final BiConsumer> createColumnMappings = (columnName, column) -> { - final Class vectorType = DhFormulaColumn.getVectorType(column.getDataType()); - - columnVariables.add(columnName); - if (possibleVariables.put(columnName, column.getDataType()) != null) { - possibleVariableParameterizedTypes.remove(columnName); - } - columnVariables.add(columnName + COLUMN_SUFFIX); - if (possibleVariables.put(columnName + COLUMN_SUFFIX, vectorType) != null) { - possibleVariableParameterizedTypes.remove(columnName + COLUMN_SUFFIX); - } - - final Class compType = column.getComponentType(); - if (compType != null && !compType.isPrimitive()) { - possibleVariableParameterizedTypes.put(columnName, new Class[] {compType}); - } - if (vectorType == ObjectVector.class) { - possibleVariableParameterizedTypes.put(columnName + COLUMN_SUFFIX, - new Class[] {column.getDataType()}); - } - }; - - // By default all columns are available to the formula - for (final ColumnDefinition column : tableDefinition.getColumns()) { - createColumnMappings.accept(column.getName(), column); - } - // Overwrite any existing column mapping using the provided renames. - for (final Map.Entry entry : outerToInnerNames.entrySet()) { - final String columnName = entry.getKey(); - final ColumnDefinition column = tableDefinition.getColumn(entry.getValue()); - createColumnMappings.accept(columnName, column); - } - - log.debug("Expression (before) : " + formula); - final TimeLiteralReplacedExpression timeConversionResult = TimeLiteralReplacedExpression.convertExpression(formula); - log.debug("Expression (after time conversion) : " + timeConversionResult.getConvertedFormula()); - - possibleVariables.putAll(timeConversionResult.getNewVariables()); + final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( + timeConversionResult, tableDefinition.getColumnNameMap(), outerToInnerNames, + queryScopeVariables.get(), unboxArguments); - final QueryLanguageParser.Result result = new QueryLanguageParser( - timeConversionResult.getConvertedFormula(), - ExecutionContext.getContext().getQueryLibrary().getPackageImports(), - ExecutionContext.getContext().getQueryLibrary().getClassImports(), - ExecutionContext.getContext().getQueryLibrary().getStaticImports(), - possibleVariables, possibleVariableParameterizedTypes, queryScopeVariables, columnVariables, - unboxArguments) - .getResult(); formulaShiftColPair = result.getFormulaShiftColPair(); if (formulaShiftColPair != null) { log.debug("Formula (after shift conversion) : " + formulaShiftColPair.getFirst()); @@ -225,7 +153,7 @@ public synchronized void init(TableDefinition tableDefinition) { final Class resultType = result.getType(); checkReturnType(result, resultType); - generateFilterCode(tableDefinition, timeConversionResult, result); + generateFilterCode(tableDefinition, timeConversionResult, result, compilationProcessor); initialized = true; } } catch (Exception e) { @@ -270,15 +198,19 @@ private void checkAndInitializeVectorization(QueryLanguageParser.Result result, checkReturnType(result, pyCallableWrapper.getReturnType()); for (String variable : result.getVariablesUsed()) { - if (variable.equals("i")) { - usesI = true; - usedColumns.add("i"); - } else if (variable.equals("ii")) { - usesII = true; - usedColumns.add("ii"); - } else if (variable.equals("k")) { - usesK = true; - usedColumns.add("k"); + switch (variable) { + case "i": + usesI = true; + usedColumns.add("i"); + break; + case "ii": + usesII = true; + usedColumns.add("ii"); + break; + case "k": + usesK = true; + usedColumns.add("k"); + break; } } ArgumentsChunked argumentsChunked = pyCallableWrapper.buildArgumentsChunked(usedColumns); @@ -302,9 +234,12 @@ private void checkReturnType(QueryLanguageParser.Result result, Class resultT } } - protected abstract void generateFilterCode(TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) throws MalformedURLException, ClassNotFoundException; + protected abstract void generateFilterCode( + @NotNull TableDefinition tableDefinition, + @NotNull TimeLiteralReplacedExpression timeConversionResult, + @NotNull QueryLanguageParser.Result result, + @NotNull QueryCompilerRequestProcessor compilationProcessor) + throws MalformedURLException, ClassNotFoundException; @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java index 52111727915..a0937aaab04 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java @@ -3,12 +3,14 @@ // package io.deephaven.engine.table.impl.select; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.verify.Require; import io.deephaven.configuration.Configuration; import io.deephaven.engine.table.*; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.table.impl.vector.*; import io.deephaven.engine.table.impl.select.formula.*; @@ -18,9 +20,14 @@ import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.api.util.NameValidator; +import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Consumer; /** @@ -34,11 +41,12 @@ public abstract class AbstractFormulaColumn implements FormulaColumn { protected String formulaString; + protected final String originalFormulaString; protected List usedColumns; @NotNull protected final String columnName; - protected FormulaFactory formulaFactory; + protected Future formulaFactory; private Formula formula; protected QueryScopeParam[] params; protected Map> columnSources; @@ -61,6 +69,7 @@ public abstract class AbstractFormulaColumn implements FormulaColumn { */ protected AbstractFormulaColumn(String columnName, String formulaString) { this.formulaString = Require.neqNull(formulaString, "formulaString"); + this.originalFormulaString = formulaString; this.columnName = NameValidator.validateColumnName(columnName); } @@ -69,6 +78,11 @@ public Class getReturnedType() { return returnedType; } + @Override + public Class getReturnedComponentType() { + return returnedType.getComponentType(); + } + @Override public List initInputs( @NotNull final TrackingRowSet rowSet, @@ -79,6 +93,9 @@ public List initInputs( if (usedColumns != null) { return usedColumns; } + + // we'll have to assume that initDef has already been invoked if we could have grouped compilation requests + // otherwise this call will compile immediately if necessary return initDef(extractDefinitions(columnsOfInterest)); } @@ -220,7 +237,15 @@ private ColumnSource getViewColumnSource(boolean lazy) { private Formula getFormula(boolean initLazyMap, Map> columnsToData, QueryScopeParam... params) { - formula = formulaFactory.createFormula(rowSet, initLazyMap, columnsToData, params); + try { + // the future must already be completed or else it is an error + formula = formulaFactory.get(0, TimeUnit.SECONDS).createFormula( + StringEscapeUtils.escapeJava(columnName), rowSet, initLazyMap, columnsToData, params); + } catch (TimeoutException e) { + throw new IllegalStateException("Formula factory not already compiled!"); + } catch (InterruptedException | ExecutionException e) { + throw new UncheckedDeephavenException("Error creating formula for " + columnName, e); + } return formula; } @@ -254,26 +279,29 @@ private static Vector makeAppropriateVectorWrapper(ColumnSource cs, RowSet return new ObjectVectorColumnWrapper<>((ColumnSource) cs, rowSet); } - protected FormulaFactory createKernelFormulaFactory(final FormulaKernelFactory formulaKernelFactory) { + protected Future createKernelFormulaFactory( + @NotNull final CompletionStageFuture formulaKernelFactoryFuture) { final FormulaSourceDescriptor sd = getSourceDescriptor(); - return (rowSet, lazy, columnsToData, params) -> { - // Maybe warn that we ignore "lazy". By the way, "lazy" is the wrong term anyway. "lazy" doesn't mean - // "cached", which is how we are using it. - final Map> netColumnSources = new HashMap<>(); - for (final String columnName : sd.sources) { - final ColumnSource columnSourceToUse = columnsToData.get(columnName); - netColumnSources.put(columnName, columnSourceToUse); - } - - final Vector[] vectors = new Vector[sd.arrays.length]; - for (int ii = 0; ii < sd.arrays.length; ++ii) { - final ColumnSource cs = columnsToData.get(sd.arrays[ii]); - vectors[ii] = makeAppropriateVectorWrapper(cs, rowSet); - } - final FormulaKernel fk = formulaKernelFactory.createInstance(vectors, params); - return new FormulaKernelAdapter(rowSet, sd, netColumnSources, fk); - }; + return formulaKernelFactoryFuture + .thenApply(formulaKernelFactory -> (columnName, rowSet, lazy, columnsToData, params) -> { + // Maybe warn that we ignore "lazy". By the way, "lazy" is the wrong term anyway. "lazy" doesn't + // mean + // "cached", which is how we are using it. + final Map> netColumnSources = new HashMap<>(); + for (final String sourceColumnName : sd.sources) { + final ColumnSource columnSourceToUse = columnsToData.get(sourceColumnName); + netColumnSources.put(sourceColumnName, columnSourceToUse); + } + + final Vector[] vectors = new Vector[sd.arrays.length]; + for (int ii = 0; ii < sd.arrays.length; ++ii) { + final ColumnSource cs = columnsToData.get(sd.arrays[ii]); + vectors[ii] = makeAppropriateVectorWrapper(cs, rowSet); + } + final FormulaKernel fk = formulaKernelFactory.createInstance(vectors, params); + return new FormulaKernelAdapter(rowSet, sd, netColumnSources, fk); + }); } protected abstract FormulaSourceDescriptor getSourceDescriptor(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java index a967463f1bd..513dafc7d36 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java @@ -32,13 +32,13 @@ public abstract class AbstractRangeFilter extends WhereFilterImpl { /** * The chunkFilter can be applied to the columns native type. - * + *

* In practice, this is for non-reinterpretable DateTimes. */ ChunkFilter chunkFilter; /** - * If the column can be be reinterpreted to a long, then we should prefer to use the longFilter instead. - * + * If the column can be reinterpreted to a long, then we should prefer to use the longFilter instead. + *

* In practice, this is used for reinterpretable DateTimes. */ ChunkFilter longFilter; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java index 78d8c588417..2e38b00e0d2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java @@ -8,6 +8,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import io.deephaven.util.QueryConstants; @@ -16,10 +17,12 @@ import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * Base class for filters that will release more rows of a table on each UGP cycle. - * + *

* The use case is for benchmarks that want to replay a table in order to better understand incremental processing * capacity. */ @@ -67,7 +70,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { initialized = true; if (!started) { return; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java index c80de6413d8..7ad699fe1f1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java @@ -12,6 +12,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ByteRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -20,6 +21,9 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class ByteRangeFilter extends AbstractRangeFilter { public static ByteRangeFilter lt(String columnName, byte x) { return new ByteRangeFilter(columnName, QueryConstants.NULL_BYTE, x, true, false); @@ -67,7 +71,10 @@ static WhereFilter makeByteRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java index 7c22ea023ce..06df9a3bc66 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java @@ -8,6 +8,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.CharRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -16,6 +17,9 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class CharRangeFilter extends AbstractRangeFilter { public static CharRangeFilter lt(String columnName, char x) { return new CharRangeFilter(columnName, QueryConstants.NULL_CHAR, x, true, false); @@ -63,7 +67,10 @@ static WhereFilter makeCharRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java index 71a38223370..f1751d83be7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java @@ -11,6 +11,7 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageFunctionUtils; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.updategraph.DynamicNode; @@ -24,6 +25,8 @@ import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * Boilerplate super-class for various clock-oriented filters. @@ -46,7 +49,10 @@ public ClockFilter(@NotNull final String columnName, @NotNull final Clock clock, } @Override - public final void init(@NotNull final TableDefinition tableDefinition) {} + public final void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @Override public final List getColumns() { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java index c5ab78e9baa..1f76242f348 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.util.compare.ObjectComparisons; import io.deephaven.engine.table.ColumnSource; @@ -17,6 +18,9 @@ import io.deephaven.util.annotations.TestUseOnly; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class ComparableRangeFilter extends AbstractRangeFilter { private final Comparable upper; private final Comparable lower; @@ -41,7 +45,10 @@ public static ComparableRangeFilter makeForTest(String columnName, Comparable } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java index 9777c518f5b..cde86f837a6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java @@ -4,12 +4,15 @@ package io.deephaven.engine.table.impl.select; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.liveness.LivenessArtifact; import io.deephaven.engine.table.impl.DependencyStreamProvider; import io.deephaven.util.annotations.TestUseOnly; +import org.jetbrains.annotations.NotNull; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Stream; public abstract class ComposedFilter extends WhereFilterLivenessArtifactImpl implements DependencyStreamProvider { @@ -56,9 +59,12 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { for (WhereFilter filter : componentFilters) { - filter.init(tableDefinition); + filter.init(tableDefinition, queryScopeVariables, compilationProcessor); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java index 8728d9173bc..6bad2b7a70d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java @@ -7,6 +7,7 @@ import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Context; import io.deephaven.engine.table.SharedContext; @@ -14,11 +15,11 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; import io.deephaven.engine.table.impl.util.codegen.CodeGenerator; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.ColumnSource; import io.deephaven.chunk.*; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; @@ -34,6 +35,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import static io.deephaven.engine.table.impl.select.DhFormulaColumn.COLUMN_SUFFIX; @@ -44,7 +49,7 @@ public class ConditionFilter extends AbstractConditionFilter { public static final int CHUNK_SIZE = 4096; - private Class filterKernelClass = null; + private Future> filterKernelClass = null; private List>> usedInputs; // that is columns and special variables private String classBody; private Filter filter = null; @@ -378,46 +383,53 @@ private static String toTitleCase(String input) { @Override protected void generateFilterCode( - TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) { + @NotNull final TableDefinition tableDefinition, + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final QueryLanguageParser.Result result, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { final StringBuilder classBody = getClassBody(tableDefinition, timeConversionResult, result); - if (classBody == null) + if (classBody == null) { return; - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(formula)) { - final List> paramClasses = new ArrayList<>(); - final Consumer> addParamClass = (cls) -> { - if (cls != null) { - paramClasses.add(cls); - } - }; - for (String usedColumn : usedColumns) { - usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); - final ColumnDefinition column = tableDefinition.getColumn(usedColumn); - addParamClass.accept(column.getDataType()); - addParamClass.accept(column.getComponentType()); - } - for (String usedColumn : usedColumnArrays) { - usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); - final ColumnDefinition column = tableDefinition.getColumn(usedColumn); - addParamClass.accept(column.getDataType()); - addParamClass.accept(column.getComponentType()); - } - for (final QueryScopeParam param : params) { - addParamClass.accept(QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); - } - - filterKernelClass = ExecutionContext.getContext().getQueryCompiler() - .compile("GeneratedFilterKernel", this.classBody = classBody.toString(), - QueryCompiler.FORMULA_PREFIX, QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)); } + + final List> paramClasses = new ArrayList<>(); + final Consumer> addParamClass = (cls) -> { + if (cls != null) { + paramClasses.add(cls); + } + }; + for (String usedColumn : usedColumns) { + usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); + final ColumnDefinition column = tableDefinition.getColumn(usedColumn); + addParamClass.accept(column.getDataType()); + addParamClass.accept(column.getComponentType()); + } + for (String usedColumn : usedColumnArrays) { + usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); + final ColumnDefinition column = tableDefinition.getColumn(usedColumn); + addParamClass.accept(column.getDataType()); + addParamClass.accept(column.getComponentType()); + } + for (final QueryScopeParam param : params) { + addParamClass.accept(QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); + } + + this.classBody = classBody.toString(); + + filterKernelClass = compilationProcessor.submit(QueryCompilerRequest.builder() + .description("Filter Expression: " + formula) + .className("GeneratedFilterKernel") + .classBody(this.classBody) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) + .build()); } @Nullable private StringBuilder getClassBody( - TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) { + @NotNull final TableDefinition tableDefinition, + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final QueryLanguageParser.Result result) { if (filterKernelClass != null) { return null; } @@ -577,15 +589,23 @@ private StringBuilder getClassBody( protected Filter getFilter(Table table, RowSet fullSet) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { if (filter == null) { - final FilterKernel filterKernel = (FilterKernel) filterKernelClass - .getConstructor(Table.class, RowSet.class, QueryScopeParam[].class) - .newInstance(table, fullSet, (Object) params); - final String[] columnNames = usedInputs.stream() - .map(p -> outerToInnerNames.getOrDefault(p.first, p.first)) - .toArray(String[]::new); - filter = new ChunkFilter(filterKernel, columnNames, CHUNK_SIZE); - // note this filter is not valid for use in other contexts, as it captures references from the source table - filterValidForCopy = false; + try { + final FilterKernel filterKernel = (FilterKernel) filterKernelClass + .get(0, TimeUnit.SECONDS) + .getConstructor(Table.class, RowSet.class, QueryScopeParam[].class) + .newInstance(table, fullSet, (Object) params); + final String[] columnNames = usedInputs.stream() + .map(p -> outerToInnerNames.getOrDefault(p.first, p.first)) + .toArray(String[]::new); + filter = new ChunkFilter(filterKernel, columnNames, CHUNK_SIZE); + // note this filter is not valid for use in other contexts, as it captures references from the source + // table + filterValidForCopy = false; + } catch (TimeoutException e) { + throw new IllegalStateException("Formula factory not already compiled!"); + } catch (InterruptedException | ExecutionException e) { + throw new FormulaCompilationException("Formula compilation error for: " + formula, e); + } } return filter; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index 6d788018888..ec5200ab9dc 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -3,18 +3,20 @@ // package io.deephaven.engine.table.impl.select; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.Pair; import io.deephaven.chunk.ChunkType; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.Table; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.select.codegen.FormulaAnalyzer; import io.deephaven.engine.table.impl.select.codegen.JavaKernelBuilder; import io.deephaven.engine.table.impl.select.codegen.RichType; @@ -31,11 +33,10 @@ import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.util.SafeCloseable; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.type.TypeUtils; import io.deephaven.vector.ObjectVector; import io.deephaven.vector.Vector; -import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; import org.jpy.PyObject; @@ -43,13 +44,11 @@ import java.math.BigInteger; import java.time.Instant; import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import static io.deephaven.engine.util.IterableUtils.makeCommaSeparatedList; @@ -182,7 +181,10 @@ public static Class getVectorType(Class declaredType) { } @Override - public List initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (formulaFactory != null) { validateColumnDefinition(columnDefinitionMap); return formulaColumnPython != null ? formulaColumnPython.usedColumns : usedColumns; @@ -191,8 +193,8 @@ public List initDef(Map> columnDefinitionMap try { final TimeLiteralReplacedExpression timeConversionResult = TimeLiteralReplacedExpression.convertExpression(formulaString); - final QueryLanguageParser.Result result = FormulaAnalyzer.getCompiledFormula(columnDefinitionMap, - timeConversionResult); + final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( + timeConversionResult, columnDefinitionMap, Collections.emptyMap(), queryScopeVariables.get(), true); analyzedFormula = FormulaAnalyzer.analyze(formulaString, columnDefinitionMap, timeConversionResult, result); hasConstantValue = result.isConstantValueExpression(); @@ -209,18 +211,23 @@ public List initDef(Map> columnDefinitionMap formulaString = result.getConvertedExpression(); // check if this is a column to be created with a Python vectorizable function - checkAndInitializeVectorization(columnDefinitionMap); + checkAndInitializeVectorization(columnDefinitionMap, queryScopeVariables, compilationRequestProcessor); } catch (Exception e) { - throw new FormulaCompilationException("Formula compilation error for: " + formulaString, e); + throw new FormulaCompilationException("Formula compilation error for: " + originalFormulaString, e); } - formulaFactory = useKernelFormulasProperty - ? createKernelFormulaFactory(getFormulaKernelFactory()) - : createFormulaFactory(); + if (useKernelFormulasProperty) { + formulaFactory = createKernelFormulaFactory(getFormulaKernelFactory(compilationRequestProcessor)); + } else { + compileFormula(compilationRequestProcessor); + } return formulaColumnPython != null ? formulaColumnPython.usedColumns : usedColumns; } - private void checkAndInitializeVectorization(Map> columnDefinitionMap) { + private void checkAndInitializeVectorization( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection SuspiciousToArrayCall final PyCallableWrapperJpyImpl[] cws = Arrays.stream(params) .filter(p -> p.getValue() instanceof PyCallableWrapperJpyImpl) @@ -241,7 +248,7 @@ private void checkAndInitializeVectorization(Map> co pyCallableWrapper.getReturnType(), this.analyzedFormula.sourceDescriptor.sources, argumentsChunked, true)); - formulaColumnPython.initDef(columnDefinitionMap); + formulaColumnPython.initDef(columnDefinitionMap, queryScopeVariables, compilationRequestProcessor); } } @@ -253,6 +260,7 @@ String generateClassBody() { CodeGenerator.create(ExecutionContext.getContext().getQueryLibrary().getImportStrings().toArray()), "", "public class $CLASSNAME$ extends [[FORMULA_CLASS_NAME]]", CodeGenerator.block( generateFormulaFactoryLambda(), "", + "private final String __columnName;", CodeGenerator.repeated("instanceVar", "private final [[TYPE]] [[NAME]];"), "private final Map [[LAZY_RESULT_CACHE_NAME]];", analyzedFormula.timeInstanceVariables, "", @@ -304,12 +312,14 @@ private CodeGenerator generateFormulaFactoryLambda() { private CodeGenerator generateConstructor() { final CodeGenerator g = CodeGenerator.create( - "public $CLASSNAME$(final TrackingRowSet __rowSet,", CodeGenerator.indent( + "public $CLASSNAME$(final String __columnName,", CodeGenerator.indent( + "final TrackingRowSet __rowSet,", "final boolean __lazy,", "final java.util.Map __columnsToData,", "final [[PARAM_CLASSNAME]]... __params)"), CodeGenerator.block( "super(__rowSet);", + "this.__columnName = __columnName;", CodeGenerator.repeated("initColumn", "[[COLUMN_NAME]] = __columnsToData.get(\"[[COLUMN_NAME]]\");"), CodeGenerator.repeated("initNormalColumnArray", @@ -355,7 +365,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { "try", CodeGenerator.block( "return [[FORMULA_STRING]];"), CodeGenerator.samelineBlock("catch (java.lang.Exception __e)", - "throw new [[EXCEPTION_TYPE]](\"In formula: [[COLUMN_NAME]] = \" + [[JOINED_FORMULA_STRING]], __e);"))); + "throw new [[EXCEPTION_TYPE]](\"In formula: \" + __columnName + \" = \" + [[JOINED_FORMULA_STRING]], __e);"))); g.replace("RETURN_TYPE", ta.typeString); final List args = visitFormulaParameters(n -> n.typeString + " " + n.name, n -> n.typeString + " " + n.name, @@ -363,8 +373,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(formulaString)); - g.replace("COLUMN_NAME", StringEscapeUtils.escapeJava(columnName)); - final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(formulaString); + final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", EVALUATION_EXCEPTION_CLASSNAME); return g.freeze(); @@ -705,11 +714,13 @@ protected FormulaSourceDescriptor getSourceDescriptor() { return analyzedFormula.sourceDescriptor; } - protected FormulaKernelFactory getFormulaKernelFactory() { - return invokeKernelBuilder().formulaKernelFactory; + protected CompletionStageFuture getFormulaKernelFactory( + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + return invokeKernelBuilder(compilationRequestProcessor).thenApply(result -> result.formulaKernelFactory); } - private JavaKernelBuilder.Result invokeKernelBuilder() { + private CompletionStageFuture invokeKernelBuilder( + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { final FormulaAnalyzer.Result af = analyzedFormula; final FormulaSourceDescriptor sd = af.sourceDescriptor; final Map columnDict = makeNameToRichTypeDict(sd.sources, columnDefinitions); @@ -722,8 +733,15 @@ private JavaKernelBuilder.Result invokeKernelBuilder() { for (final String p : sd.params) { paramDict.put(p, allParamDict.get(p)); } - return JavaKernelBuilder.create(af.cookedFormulaString, sd.returnType, af.timeInstanceVariables, columnDict, - arrayDict, paramDict); + return JavaKernelBuilder.create( + originalFormulaString, + af.cookedFormulaString, + sd.returnType, + af.timeInstanceVariables, + columnDict, + arrayDict, + paramDict, + compilationRequestProcessor); } /** @@ -731,7 +749,11 @@ private JavaKernelBuilder.Result invokeKernelBuilder() { */ @NotNull String generateKernelClassBody() { - return invokeKernelBuilder().classBody; + try { + return invokeKernelBuilder(QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE).get().classBody; + } catch (InterruptedException | ExecutionException e) { + throw new UncheckedDeephavenException("Failed to compile formula: ", e); + } } @Override @@ -758,48 +780,46 @@ public Pair>> getFormulaShiftColPair() { return formulaShiftColPair; } - private FormulaFactory createFormulaFactory() { - final String classBody = generateClassBody(); + private void compileFormula(@NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { final String what = "Compile regular formula: " + formulaString; - final Class clazz = compileFormula(what, classBody, "Formula"); - try { - return (FormulaFactory) clazz.getField(FORMULA_FACTORY_NAME).get(null); - } catch (ReflectiveOperationException e) { - throw new FormulaCompilationException("Formula compilation error for: " + what, e); - } - } + final String className = "Formula"; + final String classBody = generateClassBody(); - @SuppressWarnings("SameParameterValue") - private Class compileFormula(final String what, final String classBody, final String className) { - // System.out.printf("compileFormula: what is %s. Code is...%n%s%n", what, classBody); - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(what)) { - // Compilation needs to take place with elevated privileges, but the created object should not have them. + final List> paramClasses = new ArrayList<>(); + final Consumer> addParamClass = (cls) -> { + if (cls != null) { + paramClasses.add(cls); + } + }; + visitFormulaParameters(null, + csp -> { + addParamClass.accept(csp.type); + addParamClass.accept(csp.columnDefinition.getComponentType()); + return null; + }, + cap -> { + addParamClass.accept(cap.dataType); + addParamClass.accept(cap.columnDefinition.getComponentType()); + return null; + }, + p -> { + addParamClass.accept(p.type); + return null; + }); - final List> paramClasses = new ArrayList<>(); - final Consumer> addParamClass = (cls) -> { - if (cls != null) { - paramClasses.add(cls); - } - }; - visitFormulaParameters(null, - csp -> { - addParamClass.accept(csp.type); - addParamClass.accept(csp.columnDefinition.getComponentType()); - return null; - }, - cap -> { - addParamClass.accept(cap.dataType); - addParamClass.accept(cap.columnDefinition.getComponentType()); - return null; - }, - p -> { - addParamClass.accept(p.type); - return null; - }); - final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); - return compiler.compile(className, classBody, QueryCompiler.FORMULA_PREFIX, - QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)); - } + formulaFactory = compilationRequestProcessor.submit(QueryCompilerRequest.builder() + .description("Formula Expression: " + formulaString) + .className(className) + .classBody(classBody) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) + .build()).thenApply(clazz -> { + try { + return (FormulaFactory) clazz.getField(FORMULA_FACTORY_NAME).get(null); + } catch (ReflectiveOperationException e) { + throw new FormulaCompilationException("Formula compilation error for: " + what, e); + } + }); } private static class IndexParameter { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java index 3c4cc277672..c69b7100947 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java @@ -10,6 +10,7 @@ import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.compare.DoubleComparisons; import io.deephaven.engine.table.impl.chunkfilter.DoubleRangeComparator; import io.deephaven.engine.table.ColumnSource; @@ -18,6 +19,9 @@ import io.deephaven.util.QueryConstants; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class DoubleRangeFilter extends AbstractRangeFilter { public static DoubleRangeFilter lt(String columnName, double x) { @@ -79,7 +83,10 @@ static WhereFilter makeDoubleRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java index 3d45c6c6edf..2a1ed5bc5b6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java @@ -9,6 +9,7 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.time.DateTimeUtils; import io.deephaven.engine.updategraph.DynamicNode; import io.deephaven.engine.table.ColumnSource; @@ -17,6 +18,8 @@ import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * Utilities for downsampling non-ticking time series data within a query. The input table must be sorted by the @@ -85,7 +88,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java index bbed868385f..7ea956c163b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java @@ -26,6 +26,7 @@ import org.jetbrains.annotations.NotNull; import java.util.*; +import java.util.function.Supplier; /** * A where filter that extracts a set of inclusion or exclusion keys from a set table. @@ -221,7 +222,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java index d5f89f0fbe7..def411b61c8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.compare.FloatComparisons; import io.deephaven.engine.table.impl.chunkfilter.FloatRangeComparator; import io.deephaven.engine.table.ColumnSource; @@ -14,6 +15,9 @@ import io.deephaven.util.QueryConstants; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class FloatRangeFilter extends AbstractRangeFilter { public static FloatRangeFilter lt(String columnName, float x) { @@ -75,7 +79,10 @@ static WhereFilter makeFloatRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java index 511de8ee358..0ddbb47682b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java @@ -8,6 +8,7 @@ import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.NoSuchColumnException; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.engine.table.impl.sources.SparseArrayColumnSource; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -25,6 +26,7 @@ import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.function.Supplier; public class FunctionalColumn implements SelectColumn { @@ -106,7 +108,10 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { @@ -125,6 +130,11 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + return componentType; + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java index 1089a546ad1..3328ccab9b6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; import io.deephaven.engine.table.ColumnSource; @@ -18,6 +19,8 @@ import org.jetbrains.annotations.NotNull; import java.time.Instant; +import java.util.Map; +import java.util.function.Supplier; public class InstantRangeFilter extends LongRangeFilter { @@ -37,7 +40,10 @@ public InstantRangeFilter( } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java index e730768584c..3e787851bb0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java @@ -12,6 +12,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.IntRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -20,6 +21,9 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class IntRangeFilter extends AbstractRangeFilter { public static IntRangeFilter lt(String columnName, int x) { return new IntRangeFilter(columnName, QueryConstants.NULL_INT, x, true, false); @@ -67,7 +71,10 @@ static WhereFilter makeIntRangeFilter(String columnName, Condition condition, St } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java index a412b35b613..9d4fad06cbe 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java @@ -12,6 +12,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.LongRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -20,6 +21,9 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class LongRangeFilter extends AbstractRangeFilter { public static LongRangeFilter lt(String columnName, long x) { return new LongRangeFilter(columnName, QueryConstants.NULL_LONG, x, true, false); @@ -67,7 +71,10 @@ static WhereFilter makeLongRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 0cd410d13d8..3ee5b9ee254 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -10,6 +10,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.preview.DisplayWrapper; import io.deephaven.engine.context.QueryScope; import io.deephaven.time.DateTimeUtils; @@ -23,6 +24,7 @@ import java.math.BigInteger; import java.time.Instant; import java.util.*; +import java.util.function.Supplier; public class MatchFilter extends WhereFilterImpl { @@ -116,7 +118,10 @@ public List getColumnArrays() { } @Override - public synchronized void init(TableDefinition tableDefinition) { + public synchronized void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariablesSupplier, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (initialized) { return; } @@ -130,12 +135,12 @@ public synchronized void init(TableDefinition tableDefinition) { return; } final List valueList = new ArrayList<>(); - final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); + final Map queryScopeVariables = queryScopeVariablesSupplier.get(); final ColumnTypeConvertor convertor = ColumnTypeConvertorFactory.getConvertor(column.getDataType(), column.getName()); for (String strValue : strValues) { - if (queryScope != null && queryScope.hasParamName(strValue)) { - Object paramValue = queryScope.readParamValue(strValue); + if (queryScopeVariables.containsKey(strValue)) { + Object paramValue = queryScopeVariables.get(strValue); if (paramValue != null && paramValue.getClass().isArray()) { ArrayTypeUtils.ArrayAccessor accessor = ArrayTypeUtils.getArrayAccessor(paramValue); for (int ai = 0; ai < accessor.length(); ++ai) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java index 9f1cb344cc9..8e5c1342a40 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java @@ -9,6 +9,7 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.NoSuchColumnException; import io.deephaven.engine.table.impl.PrevColumnSource; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.engine.table.impl.sources.SparseArrayColumnSource; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -19,11 +20,11 @@ import io.deephaven.engine.table.impl.chunkfillers.ChunkFiller; import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; -import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.NotNull; import java.util.*; import java.util.function.BiFunction; +import java.util.function.Supplier; import java.util.stream.Collectors; // TODO: Comment the heck out of this... @@ -94,7 +95,10 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { NoSuchColumnException.throwIf(columnDefinitionMap.keySet(), sourceNames); return getColumns(); } @@ -104,6 +108,11 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + return componentType; + } + @Override public List getColumns() { return Collections.unmodifiableList(sourceNames); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java index 3cdb2939328..7ed11fe9d1f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java @@ -5,6 +5,7 @@ import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.*; import io.deephaven.engine.rowset.TrackingRowSet; import org.jetbrains.annotations.NotNull; @@ -12,6 +13,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * A SelectColumn implementation that can be used to replace columns with {@link NullValueColumnSource}s @@ -32,7 +34,10 @@ public List initInputs(final TrackingRowSet rowSet, } @Override - public List initDef(final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { return Collections.emptyList(); } @@ -41,6 +46,11 @@ public Class getReturnedType() { return nvcs.getType(); } + @Override + public Class getReturnedComponentType() { + return nvcs.getComponentType(); + } + @Override public List getColumns() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java index 374b0a4f0df..b6c37e32dbb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java @@ -7,6 +7,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.time.DateTimeUtils; import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.rowset.RowSet; @@ -18,6 +19,8 @@ import java.math.BigInteger; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * A filter for comparable types (including Instant) for {@link Condition} values:
@@ -129,7 +132,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (filter != null) { return; } @@ -179,7 +185,7 @@ public void init(TableDefinition tableDefinition) { } } - filter.init(tableDefinition); + filter.init(tableDefinition, queryScopeVariables, compilationProcessor); } public static char parseCharFilter(String value) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java index 0d258a9072a..968e0287c9d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.rowset.TrackingRowSet; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.ConvertibleTimeSource; import io.deephaven.engine.table.impl.sources.LocalDateWrapperSource; import io.deephaven.engine.table.impl.sources.LocalTimeWrapperSource; @@ -34,17 +35,18 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.function.Supplier; /** * Allows {@link ColumnSource} reinterpretation via view-type ({@link Table#view} and {@link Table#updateView}) * {@link Table} operations. - * + *

* TODO: If we come up with other valid, useful reinterpretations, it would be trivial to create a general purpose * syntax for use in view()/updateView() column expressions. - * + *

* The syntax I have in mind is: "<ColumnNameB>=<ColumnNameA>.as(<ClassName>)" * "<ColumnName>.as(<ClassName>)" - * + *

* Making this work would consist of any one of: 1. Adding a V1 version and updating SelectColumnFactory and * SelectColumnAdaptor 2. Adding the appropriate if-regex-matches to realColumn selection in V2 SwitchColumn 3. Creating * a V2-native SelectColumnFactory @@ -151,7 +153,10 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { @@ -169,6 +174,12 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + // we don't support reinterpretting column types with components + return null; + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java index fdd46b07550..66975516e66 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java @@ -9,12 +9,15 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import org.jetbrains.annotations.NotNull; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * This will filter a table starting off with the first N rows, and then adding new rows to the table on each run. @@ -45,7 +48,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java index 480e4d8e10f..9728f556c1d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java @@ -12,6 +12,7 @@ import io.deephaven.api.expression.Method; import io.deephaven.api.filter.Filter; import io.deephaven.api.literal.Literal; +import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.ColumnDefinition; @@ -19,6 +20,9 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; +import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; import java.util.Arrays; @@ -26,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -71,7 +76,8 @@ static Collection copyFrom(Collection selectColumns) List initInputs(TrackingRowSet rowSet, Map> columnsOfInterest); /** - * Initialize any internal column definitions from the provided initial. + * Initialize any internal column definitions from the provided initial. Any formulae will be compiled immediately + * using the {@link QueryCompiler} in the current {@link ExecutionContext}. * * @param columnDefinitionMap the starting set of column definitions; valid for this call only * @@ -80,7 +86,29 @@ static Collection copyFrom(Collection selectColumns) * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. */ - List initDef(Map> columnDefinitionMap); + @FinalDefault + default List initDef(@NotNull Map> columnDefinitionMap) { + return initDef(columnDefinitionMap, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), + QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE); + } + + /** + * Initialize any internal column definitions from the provided initial. A compilation request consumer is provided + * to allow for deferred compilation of expressions that belong to the same query. + * + * @param columnDefinitionMap the starting set of column definitions; valid for this call only + * @param queryScopeVariables a caching supplier of the set of query scope variables; valid for this call only + * @param compilationRequestProcessor a consumer to submit compilation requests; valid for this call only + * + * @return a list of columns on which the result of this is dependent + * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or + * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. + * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. + */ + List initDef( + @NotNull Map> columnDefinitionMap, + @NotNull Supplier> queryScopeVariables, + @NotNull QueryCompilerRequestProcessor compilationRequestProcessor); /** * Get the data type stored in the resultant column. @@ -89,6 +117,13 @@ static Collection copyFrom(Collection selectColumns) */ Class getReturnedType(); + /** + * Get the data component type stored in the resultant column. + * + * @return the component type + */ + Class getReturnedComponentType(); + /** * Get a list of the names of columns used in this SelectColumn. Behavior is undefined if none of the init* methods * have been called yet. diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java index 737f8cf0f3e..cedccd871c1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java @@ -12,6 +12,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ShortRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -20,6 +21,9 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class ShortRangeFilter extends AbstractRangeFilter { public static ShortRangeFilter lt(String columnName, short x) { return new ShortRangeFilter(columnName, QueryConstants.NULL_SHORT, x, true, false); @@ -67,7 +71,10 @@ static WhereFilter makeShortRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java index 01b04d4561a..6801ff9de4d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.util.compare.ObjectComparisons; import io.deephaven.engine.table.ColumnSource; @@ -17,6 +18,9 @@ import io.deephaven.util.annotations.TestUseOnly; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class SingleSidedComparableRangeFilter extends AbstractRangeFilter { private final Comparable pivot; private final boolean isGreaterThan; @@ -34,7 +38,10 @@ public static SingleSidedComparableRangeFilter makeForTest(String columnName, Co } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java index a0f53b462ea..0a0449c7fca 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java @@ -7,6 +7,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.NoSuchColumnException; @@ -19,6 +20,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class SourceColumn implements SelectColumn { @@ -46,7 +48,7 @@ public SourceColumn(String sourceName, String destName) { this(NameValidator.validateColumnName(sourceName), NameValidator.validateColumnName(destName), true); } - private SourceColumn(String sourceName, String destName, boolean unused) { + private SourceColumn(@NotNull final String sourceName, @NotNull final String destName, boolean unused) { this.sourceName = sourceName; this.destName = destName; } @@ -61,7 +63,10 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { sourceDefinition = columnDefinitionMap.get(sourceName); if (sourceDefinition == null) { throw new NoSuchColumnException(columnDefinitionMap.keySet(), sourceName); @@ -78,6 +83,15 @@ public Class getReturnedType() { return sourceColumn.getType(); } + @Override + public Class getReturnedComponentType() { + // Try to be a little flexible, depending on whether initInputs or initDef was called. + if (sourceDefinition != null) { + return sourceDefinition.getComponentType(); + } + return sourceColumn.getComponentType(); + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java index 4e44a7bd3a2..48caddc72a8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java @@ -8,6 +8,7 @@ import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.python.FormulaColumnPython; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.rowset.TrackingRowSet; @@ -15,6 +16,7 @@ import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class SwitchColumn implements SelectColumn { @@ -45,7 +47,10 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (realColumn == null) { if (columnDefinitionMap.get(expression) != null) { realColumn = new SourceColumn(expression, columnName); @@ -53,7 +58,8 @@ public List initDef(Map> columnDefinitionMap realColumn = FormulaColumn.createFormulaColumn(columnName, expression, parser); } } - List usedColumns = realColumn.initDef(columnDefinitionMap); + final List usedColumns = realColumn.initDef( + columnDefinitionMap, queryScopeVariables, compilationRequestProcessor); if (realColumn instanceof DhFormulaColumn) { FormulaColumnPython formulaColumnPython = ((DhFormulaColumn) realColumn).getFormulaColumnPython(); realColumn = formulaColumnPython != null ? formulaColumnPython : realColumn; @@ -66,6 +72,11 @@ public Class getReturnedType() { return getRealColumn().getReturnedType(); } + @Override + public Class getReturnedComponentType() { + return getRealColumn().getReturnedComponentType(); + } + @Override public List getColumns() { return getRealColumn().getColumns(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java index 4ca30178f64..2b6bf85eb8f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java @@ -12,6 +12,7 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import io.deephaven.time.DateTimeUtils; @@ -21,6 +22,8 @@ import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * This will filter a table for the most recent N nanoseconds (must be on an {@link Instant} column). @@ -59,7 +62,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java index ab40e561ee8..c493d194d50 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java @@ -11,8 +11,10 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.util.annotations.FinalDefault; import io.deephaven.util.annotations.InternalUseOnly; import org.jetbrains.annotations.NotNull; @@ -20,6 +22,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * Interface for individual filters within a where clause. @@ -103,7 +107,25 @@ interface RecomputeListener { * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or * {@link QueryCompiler} usage needs to be resolved within init. Implementations must be idempotent. */ - void init(TableDefinition tableDefinition); + @FinalDefault + default void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), + QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE); + } + + /** + * Initialize this select filter given the table definition + * + * @param tableDefinition the definition of the table that will be filtered + * @param queryScopeVariables a caching supplier of the set of query scope variables; valid for this call only + * @param compilationProcessor the processor to use for compilation + * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or + * {@link QueryCompiler} usage needs to be resolved within init. Implementations must be idempotent. + */ + void init( + @NotNull TableDefinition tableDefinition, + @NotNull Supplier> queryScopeVariables, + @NotNull QueryCompilerRequestProcessor compilationProcessor); /** * Validate that this {@code WhereFilter} is safe to use in the context of the provided sourceTable. diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java index b4838d0827e..e051f16e299 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java @@ -11,12 +11,15 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.DependencyStreamProvider; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.util.annotations.VisibleForTesting; import org.jetbrains.annotations.NotNull; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import java.util.stream.Stream; class WhereFilterInvertedImpl @@ -57,8 +60,11 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { - filter.init(tableDefinition); + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + filter.init(tableDefinition, queryScopeVariables, compilationProcessor); } @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java index f3cf1df29b6..f66094e9083 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java @@ -16,12 +16,15 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter.ObjectChunkFilter; import org.jetbrains.annotations.NotNull; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; final class WhereFilterPatternImpl extends WhereFilterImpl { @@ -43,7 +46,10 @@ private WhereFilterPatternImpl(FilterPattern filterPattern) { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { final String columnName = columnName(); final ColumnDefinition column = tableDefinition.getColumn(columnName); if (column == null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java index 4495af78636..9b1317ee747 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java @@ -8,10 +8,13 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import org.jetbrains.annotations.NotNull; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * A Select filter that always returns an empty RowSet. @@ -33,7 +36,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java index ecb4207fc4a..0d9c0892c4d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java @@ -59,16 +59,6 @@ final Map> getColumnSourcesRecurse(GetMode mode) { return result; } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - for (Map.Entry> entry : sources.entrySet()) { - final String name = entry.getKey(); - final ColumnSource cs = entry.getValue(); - final ColumnDefinition cd = ColumnDefinition.fromGenericType(name, cs.getType(), cs.getComponentType()); - columnDefinitions.put(name, cd); - } - } - @Override public void applyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, JobScheduler jobScheduler, @Nullable LivenessNode liveResultOwner, SelectLayerCompletionHandler onCompletion) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java index 4ad846ceba3..67fa89b424a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java @@ -36,14 +36,6 @@ public abstract class DependencyLayerBase extends SelectAndViewAnalyzer { this.myModifiedColumnSet = mcsBuilder; } - - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - final ColumnDefinition cd = - ColumnDefinition.fromGenericType(name, columnSource.getType(), columnSource.getComponentType()); - columnDefinitions.put(name, cd); - } - @Override void populateModifiedColumnSetRecurse(ModifiedColumnSet mcsBuilder, Set remainingDepsToSatisfy) { // Later-defined columns override earlier-defined columns. So we satisfy column dependencies "on the way diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java index f9f9b44ffb9..59963f1df13 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java @@ -164,11 +164,6 @@ public SelectAndViewAnalyzer getInner() { return inner; } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - inner.updateColumnDefinitionsFromTopLayer(columnDefinitions); - } - @Override public void startTrackingPrev() { rowRedirection.startTrackingPrevValues(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java index 42eeb4b933c..d6ee54c2b0d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java @@ -3,15 +3,19 @@ // package io.deephaven.engine.table.impl.select.analyzers; +import io.deephaven.api.util.NameValidator; import io.deephaven.base.Pair; import io.deephaven.base.log.LogOutputAppendable; import io.deephaven.datastructures.util.CollectionUtil; +import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.liveness.LivenessNode; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.select.SelectColumn; @@ -28,12 +32,14 @@ import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.util.SafeCloseable; import io.deephaven.util.SafeCloseablePair; +import io.deephaven.util.datastructures.CachingSupplier; import io.deephaven.vector.Vector; import org.jetbrains.annotations.Nullable; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Stream; public abstract class SelectAndViewAnalyzer implements LogOutputAppendable { @@ -44,14 +50,31 @@ public enum Mode { VIEW_LAZY, VIEW_EAGER, SELECT_STATIC, SELECT_REFRESHING, SELECT_REDIRECTED_REFRESHING, SELECT_REDIRECTED_STATIC } + public static Supplier> newQueryScopeVariableSupplier() { + final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); + return new CachingSupplier<>( + () -> queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name))); + } + public static void initializeSelectColumns( final Map> parentColumnMap, final SelectColumn[] selectColumns) { + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + initializeSelectColumns(parentColumnMap, newQueryScopeVariableSupplier(), selectColumns, compilationProcessor); + compilationProcessor.compile(); + } + + public static void initializeSelectColumns( + final Map> parentColumnMap, + final Supplier> queryScopeVariables, + final SelectColumn[] selectColumns, + final QueryCompilerRequestProcessor compilationProcessor) { final Map> targetColumnMap = new HashMap<>(parentColumnMap); for (SelectColumn column : selectColumns) { - column.initDef(targetColumnMap); - final ColumnDefinition columnDefinition = - ColumnDefinition.fromGenericType(column.getName(), column.getReturnedType()); + column.initDef(targetColumnMap, queryScopeVariables, compilationProcessor); + final ColumnDefinition columnDefinition = ColumnDefinition.fromGenericType( + column.getName(), column.getReturnedType(), column.getReturnedComponentType()); targetColumnMap.put(column.getName(), columnDefinition); } } @@ -89,30 +112,62 @@ public static SelectAndViewAnalyzerWrapper create( rowRedirection = null; } - final TrackingRowSet originalRowSet = rowSet; - boolean flatResult = rowSet.isFlat(); - // if we preserve a column, we set this to false - boolean flattenedResult = !flatResult - && allowInternalFlatten - && (columnSources.isEmpty() || !publishTheseSources) - && mode == Mode.SELECT_STATIC; - int numberOfInternallyFlattenedColumns = 0; - List processedCols = new LinkedList<>(); List remainingCols = null; FormulaColumn shiftColumn = null; boolean shiftColumnHasPositiveOffset = false; final HashSet resultColumns = new HashSet<>(); - final HashMap> resultAlias = new HashMap<>(); + + // First pass to initialize all columns and to compile formulas in one batch. + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + for (Map.Entry> entry : columnSources.entrySet()) { + final String name = entry.getKey(); + final ColumnSource cs = entry.getValue(); + final ColumnDefinition cd = ColumnDefinition.fromGenericType(name, cs.getType(), cs.getComponentType()); + columnDefinitions.put(name, cd); + } + for (final SelectColumn sc : selectColumns) { if (remainingCols != null) { remainingCols.add(sc); continue; } - analyzer.updateColumnDefinitionsFromTopLayer(columnDefinitions); - sc.initDef(columnDefinitions); + sc.initDef(columnDefinitions, variableSupplier, compilationProcessor); + final ColumnDefinition cd = ColumnDefinition.fromGenericType( + sc.getName(), sc.getReturnedType(), sc.getReturnedComponentType()); + columnDefinitions.put(sc.getName(), cd); + + if (useShiftedColumns && hasConstantArrayAccess(sc)) { + remainingCols = new LinkedList<>(); + shiftColumn = sc instanceof FormulaColumn + ? (FormulaColumn) sc + : (FormulaColumn) ((SwitchColumn) sc).getRealColumn(); + shiftColumnHasPositiveOffset = hasPositiveOffsetConstantArrayAccess(sc); + continue; + } + + processedCols.add(sc); + } + + compilationProcessor.compile(); + + // Second pass builds the analyzer and destination columns + final TrackingRowSet originalRowSet = rowSet; + boolean flatResult = rowSet.isFlat(); + // if we preserve a column, we set this to false + boolean flattenedResult = !flatResult + && allowInternalFlatten + && (columnSources.isEmpty() || !publishTheseSources) + && mode == Mode.SELECT_STATIC; + int numberOfInternallyFlattenedColumns = 0; + + final HashMap> resultAlias = new HashMap<>(); + for (final SelectColumn sc : processedCols) { + sc.initInputs(rowSet, analyzer.getAllColumnSources()); // When flattening the result, intermediate columns generate results in position space. When we discover @@ -138,12 +193,8 @@ public static SelectAndViewAnalyzerWrapper create( final ModifiedColumnSet mcsBuilder = new ModifiedColumnSet(parentMcs); if (useShiftedColumns && hasConstantArrayAccess(sc)) { - remainingCols = new LinkedList<>(); - shiftColumn = sc instanceof FormulaColumn - ? (FormulaColumn) sc - : (FormulaColumn) ((SwitchColumn) sc).getRealColumn(); - shiftColumnHasPositiveOffset = hasPositiveOffsetConstantArrayAccess(sc); - continue; + // we use the first shifted column to split between processed columns and remaining columns + throw new IllegalStateException("Found ShiftedColumn in processed column list"); } // shifted columns appear to not be safe for refresh, so we do not validate them until they are rewritten @@ -152,8 +203,6 @@ public static SelectAndViewAnalyzerWrapper create( sc.validateSafeForRefresh(sourceTable); } - processedCols.add(sc); - if (hasConstantValue(sc)) { final WritableColumnSource constViewSource = SingleValueColumnSource.getSingleValueColumnSource(sc.getReturnedType()); @@ -177,8 +226,7 @@ public static SelectAndViewAnalyzerWrapper create( if (!sourceIsNew) { if (numberOfInternallyFlattenedColumns > 0) { // we must preserve this column, but have already created an analyzer for the internally - // flattened - // column, therefore must start over without permitting internal flattening + // flattened column, therefore must start over without permitting internal flattening return create(sourceTable, mode, columnSources, originalRowSet, parentMcs, publishTheseSources, useShiftedColumns, false, selectColumns); } else { @@ -265,6 +313,7 @@ public static SelectAndViewAnalyzerWrapper create( throw new UnsupportedOperationException("Unsupported case " + mode); } } + return new SelectAndViewAnalyzerWrapper(analyzer, shiftColumn, shiftColumnHasPositiveOffset, remainingCols, processedCols); } @@ -522,8 +571,6 @@ public final Map calcEffects(boolean forcePublishAllResources) public abstract SelectAndViewAnalyzer getInner(); - public abstract void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions); - public abstract void startTrackingPrev(); /** @@ -562,8 +609,8 @@ public boolean alreadyFlattenedSources() { abstract public boolean allowCrossColumnParallelization(); /** - * A class that handles the completion of one select column. The handlers are chained together so that when a column - * completes all of the downstream dependencies may execute. + * A class that handles the completion of one select column. The handlers are chained together; all downstream + * dependencies may execute when a column completes. */ public static abstract class SelectLayerCompletionHandler { /** @@ -591,7 +638,7 @@ public static abstract class SelectLayerCompletionHandler { * Create the final completion handler, which has no next handler. * * @param requiredColumns the columns required for this handler to fire - * @param completedColumns the set of completed columns, shared with all of the other handlers + * @param completedColumns the set of completed columns, shared with all the other handlers */ public SelectLayerCompletionHandler(BitSet requiredColumns, BitSet completedColumns) { this.requiredColumns = requiredColumns; @@ -601,9 +648,9 @@ public SelectLayerCompletionHandler(BitSet requiredColumns, BitSet completedColu /** * Called when a single column is completed. - * + *

* If we are ready, then we call {@link #onAllRequiredColumnsCompleted()}. - * + *

* We may not be ready, but other columns downstream of us may be ready, so they are also notified (the * nextHandler). * @@ -639,7 +686,7 @@ protected void onError(Exception error) { } /** - * Called when all of the required columns are completed. + * Called when all required columns are completed. */ protected abstract void onAllRequiredColumnsCompleted(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java index 9a0cf12a180..25827b2ca19 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java @@ -117,11 +117,6 @@ int getLayerIndexFor(String column) { return inner.getLayerIndexFor(column); } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - inner.updateColumnDefinitionsFromTopLayer(columnDefinitions); - } - @Override public void startTrackingPrev() { throw new UnsupportedOperationException("StaticFlattenLayer is used in only non-refreshing scenarios"); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java index 7dbd37baa25..c38bcff1ebc 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java @@ -3,9 +3,9 @@ // package io.deephaven.engine.table.impl.select.codegen; -import io.deephaven.api.util.NameValidator; import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; import io.deephaven.engine.table.impl.select.QueryScopeParamTypeUtil; @@ -17,10 +17,14 @@ import io.deephaven.engine.rowset.TrackingWritableRowSet; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; +import org.jetbrains.annotations.NotNull; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.*; +import java.util.function.BiConsumer; + +import static io.deephaven.engine.table.impl.select.AbstractFormulaColumn.COLUMN_SUFFIX; public class FormulaAnalyzer { private static final Logger log = LoggerFactory.getLogger(FormulaAnalyzer.class); @@ -67,8 +71,41 @@ public static Result analyze(final String rawFormulaString, queryLanguageResult.isConstantValueExpression()); } - public static QueryLanguageParser.Result getCompiledFormula(Map> availableColumns, - TimeLiteralReplacedExpression timeConversionResult) throws Exception { + /** + * Get the compiled formula for a given formula string. + * + * @param timeConversionResult The formula preprocessed to extract time literals + * @param availableColumns The columns available for use in the formula + * @param columnRenames Outer to inner column name mapping + * @param queryScopeVariables The query scope variables + * @return The parsed formula {@link QueryLanguageParser.Result result} + * @throws Exception If the formula cannot be parsed + */ + public static QueryLanguageParser.Result parseFormula( + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final Map> availableColumns, + @NotNull final Map columnRenames, + @NotNull final Map queryScopeVariables) throws Exception { + return parseFormula(timeConversionResult, availableColumns, columnRenames, queryScopeVariables, true); + } + + /** + * Get the compiled formula for a given formula string. + * + * @param timeConversionResult The formula preprocessed to extract time literals + * @param availableColumns The columns available for use in the formula + * @param columnRenames Outer to inner column name mapping + * @param queryScopeVariables The query scope variables + * @param unboxArguments If true it will unbox the query scope arguments + * @return The parsed formula {@link QueryLanguageParser.Result result} + * @throws Exception If the formula cannot be parsed + */ + public static QueryLanguageParser.Result parseFormula( + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final Map> availableColumns, + @NotNull final Map columnRenames, + @NotNull final Map queryScopeVariables, + final boolean unboxArguments) throws Exception { final Map> possibleVariables = new HashMap<>(); possibleVariables.put("i", int.class); possibleVariables.put("ii", long.class); @@ -81,34 +118,66 @@ public static QueryLanguageParser.Result getCompiledFormula(Map[]> possibleVariableParameterizedTypes = new HashMap<>(); + // Column names get the highest priority. + final BiConsumer> processColumn = (columnName, column) -> { + if (!columnVariables.add(columnName)) { + // this column was renamed + return; + } + + possibleVariables.put(columnName, column.getDataType()); + + final Class compType = column.getComponentType(); + if (compType != null && !compType.isPrimitive()) { + possibleVariableParameterizedTypes.put(columnName, new Class[] {compType}); + } + }; + + // Renames trump the original columns; so they go first. + for (Map.Entry columnRename : columnRenames.entrySet()) { + final String columnName = columnRename.getKey(); + final ColumnDefinition column = availableColumns.get(columnRename.getValue()); + processColumn.accept(columnName, column); + } + + // Now process the original columns. for (ColumnDefinition columnDefinition : availableColumns.values()) { - // add column-vectors - final String columnSuffix = DhFormulaColumn.COLUMN_SUFFIX; - final Class vectorType = DhFormulaColumn.getVectorType(columnDefinition.getDataType()); + processColumn.accept(columnDefinition.getName(), columnDefinition); + } - possibleVariables.put(columnDefinition.getName() + columnSuffix, vectorType); - columnVariables.add(columnDefinition.getName() + columnSuffix); + // Column arrays come between columns and parameters. + final BiConsumer> processColumnArray = (columnName, column) -> { + final String columnArrayName = columnName + COLUMN_SUFFIX; - if (vectorType == ObjectVector.class) { - possibleVariableParameterizedTypes.put(columnDefinition.getName() + columnSuffix, - new Class[] {columnDefinition.getDataType()}); + if (!columnVariables.add(columnArrayName)) { + // Either this is a rename or overloads an existing column name. + return; } - // add columns - columnVariables.add(columnDefinition.getName()); - possibleVariables.put(columnDefinition.getName(), columnDefinition.getDataType()); - final Class compType = columnDefinition.getComponentType(); - if (compType != null && !compType.isPrimitive()) { - possibleVariableParameterizedTypes.put(columnDefinition.getName(), new Class[] {compType}); + final Class vectorType = DhFormulaColumn.getVectorType(column.getDataType()); + possibleVariables.put(columnArrayName, vectorType); + + if (vectorType == ObjectVector.class) { + possibleVariableParameterizedTypes.put(columnArrayName, new Class[] {column.getDataType()}); } + }; + + // Renames still trump the original columns; so they go first. + for (Map.Entry columnRename : columnRenames.entrySet()) { + final String columnName = columnRename.getKey(); + final ColumnDefinition column = availableColumns.get(columnRename.getValue()); + processColumnArray.accept(columnName, column); } - final ExecutionContext context = ExecutionContext.getContext(); - final Map queryScopeVariables = context.getQueryScope().toMap( - (name, value) -> NameValidator.isValidQueryParameterName(name)); + // Now process the original columns. + for (ColumnDefinition columnDefinition : availableColumns.values()) { + processColumnArray.accept(columnDefinition.getName(), columnDefinition); + } + + // Parameters come last. for (Map.Entry param : queryScopeVariables.entrySet()) { if (possibleVariables.containsKey(param.getKey())) { - // skip any existing matches + // Columns and column arrays take precedence over parameters. continue; } @@ -124,22 +193,18 @@ public static QueryLanguageParser.Result getCompiledFormula(Map> classImports = - new HashSet<>(context.getQueryLibrary().getClassImports()); + final QueryLibrary queryLibrary = ExecutionContext.getContext().getQueryLibrary(); + final Set> classImports = new HashSet<>(queryLibrary.getClassImports()); classImports.add(TrackingWritableRowSet.class); classImports.add(WritableColumnSource.class); - return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), - context.getQueryLibrary().getPackageImports(), - classImports, context.getQueryLibrary().getStaticImports(), possibleVariables, - possibleVariableParameterizedTypes, queryScopeVariables, columnVariables) - .getResult(); + return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), queryLibrary.getPackageImports(), + classImports, queryLibrary.getStaticImports(), possibleVariables, possibleVariableParameterizedTypes, + queryScopeVariables, columnVariables, unboxArguments).getResult(); } public static class Result { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java index e46f6edb9f4..29e61b54fdb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java @@ -5,10 +5,11 @@ import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.util.SafeCloseable; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.context.QueryScopeParam; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.DhFormulaColumn; import io.deephaven.engine.table.impl.select.FormulaCompilationException; @@ -30,21 +31,38 @@ public class JavaKernelBuilder { private static final String FORMULA_KERNEL_FACTORY_NAME = "__FORMULA_KERNEL_FACTORY"; - public static Result create(String cookedFormulaString, Class returnedType, String timeInstanceVariables, - Map columns, Map> arrays, Map> params) { - final JavaKernelBuilder jkf = new JavaKernelBuilder(cookedFormulaString, returnedType, timeInstanceVariables, - columns, arrays, params); + public static CompletionStageFuture create( + @NotNull final String originalFormulaString, + @NotNull final String cookedFormulaString, + @NotNull final Class returnedType, + @NotNull final String timeInstanceVariables, + @NotNull final Map columns, + @NotNull final Map> arrays, + @NotNull final Map> params, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + final JavaKernelBuilder jkf = new JavaKernelBuilder( + originalFormulaString, cookedFormulaString, returnedType, timeInstanceVariables, columns, arrays, + params); final String classBody = jkf.generateKernelClassBody(); - final Class clazz = compileFormula(cookedFormulaString, classBody, "Formula"); - final FormulaKernelFactory fkf; - try { - fkf = (FormulaKernelFactory) clazz.getField(FORMULA_KERNEL_FACTORY_NAME).get(null); - } catch (ReflectiveOperationException e) { - throw new FormulaCompilationException("Formula compilation error for: " + cookedFormulaString, e); - } - return new Result(classBody, clazz, fkf); + + return compilationRequestProcessor.submit(QueryCompilerRequest.builder() + .description("FormulaKernel: " + originalFormulaString) + .className("Formula") + .classBody(classBody) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .build()).thenApply(clazz -> { + final FormulaKernelFactory fkf; + try { + fkf = (FormulaKernelFactory) clazz.getField(FORMULA_KERNEL_FACTORY_NAME).get(null); + } catch (ReflectiveOperationException e) { + throw new FormulaCompilationException( + "Formula compilation error for: " + cookedFormulaString, e); + } + return new Result(classBody, clazz, fkf); + }); } + private final String originalFormulaString; private final String cookedFormulaString; private final Class returnedType; private final String timeInstanceVariables; @@ -62,8 +80,15 @@ public static Result create(String cookedFormulaString, Class returnedType, S */ private final Map> params; - private JavaKernelBuilder(String cookedFormulaString, Class returnedType, String timeInstanceVariables, - Map columns, Map> arrays, Map> params) { + private JavaKernelBuilder( + @NotNull final String originalFormulaString, + @NotNull final String cookedFormulaString, + @NotNull final Class returnedType, + @NotNull final String timeInstanceVariables, + @NotNull final Map columns, + @NotNull final Map> arrays, + @NotNull final Map> params) { + this.originalFormulaString = originalFormulaString; this.cookedFormulaString = cookedFormulaString; this.returnedType = returnedType; this.timeInstanceVariables = timeInstanceVariables; @@ -214,7 +239,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(cookedFormulaString)); - final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(cookedFormulaString); + final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", FormulaEvaluationException.class.getCanonicalName()); return g.freeze(); @@ -257,23 +282,16 @@ private List visitFormulaParameters( return results; } - @SuppressWarnings("SameParameterValue") - private static Class compileFormula(final String what, final String classBody, final String className) { - // System.out.printf("compileFormula: formulaString is %s. Code is...%n%s%n", what, classBody); - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(what)) { - // Compilation needs to take place with elevated privileges, but the created object should not have them. - final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); - return compiler.compile(className, classBody, QueryCompiler.FORMULA_PREFIX); - } - } - public static class Result { public final String classBody; public final Class clazz; public final FormulaKernelFactory formulaKernelFactory; - public Result(String classBody, Class clazz, FormulaKernelFactory formulaKernelFactory) { + public Result( + @NotNull final String classBody, + @NotNull final Class clazz, + @NotNull final FormulaKernelFactory formulaKernelFactory) { this.classBody = classBody; this.clazz = clazz; this.formulaKernelFactory = formulaKernelFactory; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java index 3fa8eb25bf5..8c32326dc5d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java @@ -12,6 +12,7 @@ public interface FormulaFactory { Formula createFormula( + String columnName, TrackingRowSet rowSet, boolean initLazyMap, Map columnsToData, QueryScopeParam... params); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java index 2b181753891..e60d7db35db 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java @@ -5,17 +5,21 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.context.QueryScopeParam; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.table.impl.select.AbstractFormulaColumn; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.formula.FormulaKernel; import io.deephaven.engine.table.impl.select.formula.FormulaKernelFactory; import io.deephaven.engine.table.impl.select.formula.FormulaSourceDescriptor; +import org.jetbrains.annotations.NotNull; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import static io.deephaven.datastructures.util.CollectionUtil.ZERO_LENGTH_STRING_ARRAY; @@ -25,8 +29,7 @@ public class FormulaColumnPython extends AbstractFormulaColumn implements FormulaKernelFactory { @SuppressWarnings("unused") // called from python - public static FormulaColumnPython create(String columnName, - DeephavenCompatibleFunction dcf) { + public static FormulaColumnPython create(String columnName, DeephavenCompatibleFunction dcf) { return new FormulaColumnPython(columnName, dcf); } @@ -39,13 +42,16 @@ private FormulaColumnPython(String columnName, } @Override - public final List initDef(Map> columnDefinitionMap) { + public final List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (formulaFactory != null) { validateColumnDefinition(columnDefinitionMap); } else { returnedType = dcf.getReturnedType(); applyUsedVariables(columnDefinitionMap, new LinkedHashSet<>(dcf.getColumnNames()), Map.of()); - formulaFactory = createKernelFormulaFactory(this); + formulaFactory = createKernelFormulaFactory(CompletionStageFuture.completedFuture(this)); } return usedColumns; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java index 45e594997ce..8a4ecb8bf35 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java @@ -12,7 +12,9 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.updateby.delta.*; import io.deephaven.engine.table.impl.updateby.em.*; import io.deephaven.engine.table.impl.updateby.emstd.*; @@ -38,6 +40,7 @@ import java.math.MathContext; import java.time.Instant; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -85,7 +88,10 @@ final Collection getOutputColumns(@NotNull final Collection getOperators(@NotNull final Collection specs) { - final OperationVisitor v = new OperationVisitor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + + final OperationVisitor v = new OperationVisitor(compilationProcessor); specs.forEach(s -> s.walk(v)); // Do we have a combined rolling group operator to create? @@ -93,6 +99,8 @@ final Collection getOperators(@NotNull final Collection, UpdateByOperation.Visitor { + private final Supplier> queryScopeVariables; + private final QueryCompilerRequestProcessor compilationProcessor; private final List ops = new ArrayList<>(); private MatchPair[] pairs; @@ -289,6 +299,12 @@ private class OperationVisitor implements UpdateBySpec.Visitor, UpdateByOp RollingGroupSpec rollingGroupSpec; MatchPair[] rollingGroupPairs; + OperationVisitor( + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + this.queryScopeVariables = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + this.compilationProcessor = compilationProcessor; + } + /** * Check if the supplied type is one of the supported time types. * @@ -1364,47 +1380,47 @@ private UpdateByOperator makeRollingFormulaOperator(@NotNull final MatchPair pai return new BooleanRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == byte.class || csType == Byte.class) { return new ByteRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == char.class || csType == Character.class) { return new CharRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == short.class || csType == Short.class) { return new ShortRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == int.class || csType == Integer.class) { return new IntRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == long.class || csType == Long.class) { return new LongRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == float.class || csType == Float.class) { return new FloatRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == double.class || csType == Double.class) { return new DoubleRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } return new ObjectRollingFormulaOperator<>(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java index 32c662fe872..9f04380105a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java @@ -10,6 +10,7 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.select.FormulaUtil; import io.deephaven.engine.table.impl.sources.ArrayBackedColumnSource; @@ -27,6 +28,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; abstract class BaseRollingFormulaOperator extends UpdateByOperator { protected final String PARAM_COLUMN_NAME = "__PARAM_COLUMN__"; @@ -95,7 +97,9 @@ public BaseRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef) { + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, true); this.formulaColumnMap = formulaColumnMap; this.tableDef = tableDef; @@ -113,7 +117,8 @@ public BaseRollingFormulaOperator( final ColumnDefinition inputColumnDefinition = ColumnDefinition .fromGenericType(PARAM_COLUMN_NAME, inputVectorType, inputColumnType); - tmp.initDef(Collections.singletonMap(PARAM_COLUMN_NAME, inputColumnDefinition)); + tmp.initDef(Collections.singletonMap(PARAM_COLUMN_NAME, inputColumnDefinition), queryScopeVariables, + compilationProcessor); return tmp; }); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java index b463a44a0f1..4dbe63b8ae2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; import io.deephaven.engine.table.impl.updateby.UpdateByOperator; @@ -28,6 +29,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -160,9 +162,11 @@ public BooleanRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef) { + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } protected BooleanRollingFormulaOperator( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java index af6029a58aa..bcb5da64243 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -32,6 +33,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -163,12 +165,14 @@ public ByteRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef - // region extra-constructor-args - // endregion extra-constructor-args + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java index 205ee8f2bfa..73dd6d03370 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -28,6 +29,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -159,12 +161,14 @@ public CharRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef - // region extra-constructor-args - // endregion extra-constructor-args + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java index 5bc233a533d..161b9842cff 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -32,6 +33,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -163,12 +165,14 @@ public DoubleRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef - // region extra-constructor-args - // endregion extra-constructor-args + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java index 459fa465bff..f994066ec0d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -32,6 +33,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -163,12 +165,14 @@ public FloatRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef - // region extra-constructor-args - // endregion extra-constructor-args + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java index 194d1dc7063..2e160f87ef4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -32,6 +33,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -163,12 +165,14 @@ public IntRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef - // region extra-constructor-args - // endregion extra-constructor-args + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java index 1171ffccaf6..15f0f13aac8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -32,6 +33,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -163,12 +165,14 @@ public LongRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef - // region extra-constructor-args - // endregion extra-constructor-args + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java index 849a7a1de28..932f1eba0b8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; import io.deephaven.engine.table.impl.updateby.UpdateByOperator; @@ -27,6 +28,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -159,9 +161,11 @@ public ObjectRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef) { + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } protected ObjectRollingFormulaOperator( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java index c0a6978a129..7fcb68ed3d1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java @@ -20,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -32,6 +33,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -163,12 +165,14 @@ public ShortRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef - // region extra-constructor-args - // endregion extra-constructor-args + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef); + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java index 2a9971b1747..da14d6b0499 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java @@ -5,6 +5,8 @@ import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import java.util.*; import java.util.function.Supplier; @@ -20,7 +22,7 @@ public static Supplier compileSimpleFunction(final Class res public static Supplier compileSimpleStatement(final Class resultType, final String code, final String... imports) { - final List importClasses = new ArrayList<>(); + final List> importClasses = new ArrayList<>(); for (final String importString : imports) { try { importClasses.add(Class.forName(importString)); @@ -33,14 +35,14 @@ public static Supplier compileSimpleStatement(final Class re } public static Supplier compileSimpleFunction(final Class resultType, final String code, - final Collection imports, final Collection staticImports) { + final Collection> imports, final Collection> staticImports) { final StringBuilder classBody = new StringBuilder(); classBody.append("import ").append(resultType.getName()).append(";\n"); - for (final Class im : imports) { + for (final Class im : imports) { classBody.append("import ").append(im.getName()).append(";\n"); } - for (final Class sim : staticImports) { + for (final Class sim : staticImports) { classBody.append("import static ").append(sim.getName()).append(".*;\n"); } @@ -52,8 +54,13 @@ public static Supplier compileSimpleFunction(final Class res classBody.append(" }\n"); classBody.append("}\n"); - final Class partitionClass = ExecutionContext.getContext().getQueryCompiler() - .compile("Function", classBody.toString(), QueryCompiler.FORMULA_PREFIX); + final Class partitionClass = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Simple Function: " + code) + .className("Function") + .classBody(classBody.toString()) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .build()); try { // noinspection unchecked @@ -63,7 +70,7 @@ public static Supplier compileSimpleFunction(final Class res } } - public static Class getClassThroughCompilation(final String object) { + public static Class getClassThroughCompilation(final String object) { final StringBuilder classBody = new StringBuilder(); classBody.append("public class $CLASSNAME$ implements ").append(Supplier.class.getCanonicalName()) .append("{ \n"); @@ -71,12 +78,17 @@ public static Class getClassThroughCompilation(final String object) { classBody.append(" public Class get() { return ").append(object).append(".class; }\n"); classBody.append("}\n"); - final Class partitionClass = ExecutionContext.getContext().getQueryCompiler() - .compile("Function", classBody.toString(), QueryCompiler.FORMULA_PREFIX); + final Class partitionClass = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Formula: return " + object + ".class") + .className("Function") + .classBody(classBody.toString()) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .build()); try { // noinspection unchecked - return ((Supplier) partitionClass.newInstance()).get(); + return ((Supplier>) partitionClass.newInstance()).get(); } catch (InstantiationException | IllegalAccessException e) { throw new RuntimeException("Could not instantiate function.", e); } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java index df1467197ac..cc927144403 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java @@ -21,6 +21,8 @@ import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; import static org.junit.Assert.assertEquals; @@ -57,7 +59,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeSupplier, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java index 2fd1498e064..4f7597807fe 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java @@ -12,6 +12,7 @@ import io.deephaven.engine.context.*; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.impl.lang.QueryLanguageParser.QueryLanguageParseException; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.testutil.ControlledUpdateGraph; import io.deephaven.engine.util.PyCallableWrapper; import io.deephaven.util.QueryConstants; @@ -3178,7 +3179,7 @@ private void check(String expression, String resultExpression, Class resultTy final Map possibleParams; final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); if (!(queryScope instanceof PoisonedQueryScope)) { - possibleParams = queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name)); + possibleParams = SelectAndViewAnalyzer.newQueryScopeVariableSupplier().get(); } else { possibleParams = null; } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java index 505bad205b0..d6a5c6abcc1 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java @@ -106,7 +106,7 @@ private long applyFormulaPerItem(long II, long ii, int I, int i) { try { return plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1)))); } catch (java.lang.Exception __e) { - throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: " + "plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1))))", __e); + throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: " + "I * II + q * ii + II_[i - 1]", __e); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java index 897f8697869..f24fd283dbc 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java @@ -73,6 +73,7 @@ public class FormulaSample extends io.deephaven.engine.table.impl.select.Formula { public static final io.deephaven.engine.table.impl.select.formula.FormulaFactory __FORMULA_FACTORY = FormulaSample::new; + private final String __columnName; private final io.deephaven.engine.table.ColumnSource II; private final io.deephaven.engine.table.ColumnSource I; private final io.deephaven.vector.LongVector II_; @@ -80,11 +81,13 @@ public class FormulaSample extends io.deephaven.engine.table.impl.select.Formula private final Map __lazyResultCache; - public FormulaSample(final TrackingRowSet __rowSet, + public FormulaSample(final String __columnName, + final TrackingRowSet __rowSet, final boolean __lazy, final java.util.Map __columnsToData, final io.deephaven.engine.context.QueryScopeParam... __params) { super(__rowSet); + this.__columnName = __columnName; II = __columnsToData.get("II"); I = __columnsToData.get("I"); II_ = new io.deephaven.engine.table.impl.vector.LongVectorColumnWrapper(__columnsToData.get("II"), __rowSet); @@ -190,7 +193,7 @@ private long applyFormulaPerItem(int i, long ii, long II, int I) { try { return plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1)))); } catch (java.lang.Exception __e) { - throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: Value = " + "plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1))))", __e); + throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: " + __columnName + " = " + "I * II + q * ii + II_[i - 1]", __e); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java b/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java index a2e1a5c71b7..0522779ed2c 100644 --- a/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java +++ b/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java @@ -18,7 +18,7 @@ public void testNotString() { DynamicCompileUtils.compileSimpleFunction(String.class, "return 7"); TestCase.fail("Should never have reached this statement."); } catch (RuntimeException e) { - TestCase.assertTrue(e.getMessage().contains("int cannot be converted to String")); + TestCase.assertTrue(e.getMessage().contains("int cannot be converted to java.lang.String")); } } } diff --git a/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java b/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java index 521314bf70c..646f4862707 100644 --- a/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java +++ b/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java @@ -8,8 +8,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static io.deephaven.util.QueryConstants.NULL_LONG; - // TODO: Move? /** diff --git a/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java b/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java index dd9d19d6626..0d8c176051f 100644 --- a/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java +++ b/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java @@ -337,15 +337,13 @@ public static TreeTable queryPerformanceAsTreeTable(@NotNull final Table qpl) { public static TreeTable queryOperationPerformanceAsTreeTable( @NotNull final Table qpl, @NotNull final Table qopl) { - // TODO (https://github.com/deephaven/deephaven-core/issues/4814): use NULL_INT for ParentOperationNumber and - // Depth once we can prevent any compilation or at least reduce multiple usages to a single formula Table mergeWithAggKeys = TableTools.merge( qpl.updateView( "EvalKey = Long.toString(EvaluationNumber)", "ParentEvalKey = ParentEvaluationNumber == null ? null : (Long.toString(ParentEvaluationNumber))", "OperationNumber = NULL_INT", - "ParentOperationNumber = OperationNumber", - "Depth = OperationNumber", + "ParentOperationNumber = NULL_INT", + "Depth = NULL_INT", "CallerLine = (String) null", "IsCompilation = NULL_BOOLEAN", "InputSizeLong = NULL_LONG"), From 7bd96f5049bf2ef922197962aaa134d035fc3911 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 5 Mar 2024 10:58:34 -0700 Subject: [PATCH 02/19] Corey's comments --- .../src/main/java/io/deephaven/base/Lazy.java | 22 -- .../io/deephaven/util/MultiException.java | 2 +- .../util/datastructures/CachingSupplier.java | 12 +- .../engine/context/QueryCompiler.java | 201 +++++++++--------- .../impl/QueryCompilerRequestProcessor.java | 28 ++- .../impl/select/AbstractFormulaColumn.java | 3 +- .../analyzers/SelectAndViewAnalyzer.java | 4 +- open-api/lang-parser/lang-parser.gradle | 1 + .../deephaven/lang/parse/ParsedDocument.java | 6 +- .../lang/completion/CompletionLookups.java | 10 +- 10 files changed, 149 insertions(+), 140 deletions(-) delete mode 100644 Base/src/main/java/io/deephaven/base/Lazy.java diff --git a/Base/src/main/java/io/deephaven/base/Lazy.java b/Base/src/main/java/io/deephaven/base/Lazy.java deleted file mode 100644 index 47744e787af..00000000000 --- a/Base/src/main/java/io/deephaven/base/Lazy.java +++ /dev/null @@ -1,22 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.base; - -import java.util.function.Supplier; - -public class Lazy implements Supplier { - private Supplier supplier; - - public Lazy(Supplier x) { - supplier = () -> { - T val = x.get(); - supplier = () -> val; - return val; - }; - } - - public synchronized T get() { - return supplier.get(); - } -}; diff --git a/Util/src/main/java/io/deephaven/util/MultiException.java b/Util/src/main/java/io/deephaven/util/MultiException.java index f2dcc13a90c..d8f275e5aaa 100644 --- a/Util/src/main/java/io/deephaven/util/MultiException.java +++ b/Util/src/main/java/io/deephaven/util/MultiException.java @@ -8,7 +8,7 @@ import java.util.List; /** - * An exception to use when a series of operations must all be executed, but may all throw exceptions themselves. This + * An exception to use when a series of operations mus all be executed, but may all throw exceptions themselves. This * allows for retention of all exception data. */ public class MultiException extends Exception { diff --git a/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java index 43ff488da7e..62a60707d4c 100644 --- a/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java +++ b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java @@ -17,7 +17,8 @@ public final class CachingSupplier implements Supplier private final Supplier internalSupplier; private volatile boolean hasCachedResult; - private volatile OUTPUT_TYPE cachedResult; + private OUTPUT_TYPE cachedResult; + private RuntimeException errorResult; /** * Construct a {@link Supplier} wrapper. @@ -33,12 +34,19 @@ public OUTPUT_TYPE get() { if (!hasCachedResult) { synchronized (this) { if (!hasCachedResult) { - cachedResult = internalSupplier.get(); + try { + cachedResult = internalSupplier.get(); + } catch (RuntimeException err) { + errorResult = err; + } hasCachedResult = true; } } } + if (errorResult != null) { + throw errorResult; + } return cachedResult; } } diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 15cb670e788..4791a34ba03 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -274,64 +274,33 @@ public void compile( } } - /* - * @formatter:off - * 3. try to resolve CFs without compiling; retain next hash to try - * 4. compile all remaining with a single compilation task - * 5. goto step 3 - * 6. probably need Consumer> to fit DhFormulaColumn pattern? (other select columns don't need this) - * @formatter:on - */ - if (!newResolvers.isEmpty()) { - // It's my job to fulfill the future of these futures. + // It's my job to fulfill these futures. try { compileHelper(newRequests, newResolvers); } catch (RuntimeException e) { // These failures are not applicable to a single request, so we can't just complete the future and // leave the failure in the cache. - for (int ii = 0; ii < newRequests.size(); ++ii) { - if (newResolvers.get(ii).completeExceptionally(e)) { - knownClasses.remove(newRequests.get(ii).classBody()); + synchronized (this) { + for (int ii = 0; ii < newRequests.size(); ++ii) { + if (newResolvers.get(ii).completeExceptionally(e)) { + knownClasses.remove(newRequests.get(ii).classBody()); + } } } throw e; } } - Error firstError = null; - RuntimeException firstException = null; for (int ii = 0; ii < requests.length; ++ii) { try { resolvers[ii].complete(allFutures[ii].get()); - } catch (InterruptedException | ExecutionException e) { - final RuntimeException err; - if (e instanceof InterruptedException) { - err = new UncheckedDeephavenException("Interrupted while waiting for codegen", e); - } else { - Throwable cause = e.getCause(); - if (cause instanceof Error) { - firstError = (Error) cause; - resolvers[ii].completeExceptionally(cause); - continue; - } else if (cause instanceof RuntimeException) { - err = (RuntimeException) cause; - } else { - err = new UncheckedDeephavenException("Error during codegen", e); - } - } - if (firstException == null) { - firstException = err; - } + } catch (ExecutionException err) { + resolvers[ii].completeExceptionally(err.getCause()); + } catch (Throwable err) { resolvers[ii].completeExceptionally(err); } } - if (firstError != null) { - throw firstError; - } - if (firstException != null) { - throw firstException; - } } private static void ensureDirectories(final File file, final Supplier runtimeErrMsg) { @@ -483,6 +452,13 @@ private String getClassPath() { return sb.toString(); } + private static class CompilationState { + int next_pi; + boolean compiled; + String packageName; + String fqClassName; + } + private void compileHelper( @NotNull final List requests, @NotNull final List>> resolvers) { @@ -500,19 +476,28 @@ private void compileHelper( } int numCompiled = 0; - final int[] next_pi = new int[requests.size()]; - final boolean[] compiled = new boolean[requests.size()]; - final String[] packageName = new String[requests.size()]; - final String[] fqClassName = new String[requests.size()]; + final CompilationState[] states = new CompilationState[requests.size()]; + for (int ii = 0; ii < requests.size(); ++ii) { + states[ii] = new CompilationState(); + } + + /* + * @formatter:off + * 1. try to resolve CFs without compiling; retain next hash to try + * 2. compile all remaining with a single compilation task + * 3. goto step 1 if any are unresolved + * @formatter:on + */ while (numCompiled < requests.size()) { for (int ii = 0; ii < requests.size(); ++ii) { - if (compiled[ii]) { + final CompilationState state = states[ii]; + if (state.compiled) { continue; } while (true) { - final int pi = next_pi[ii]++; + final int pi = state.next_pi++; final String packageNameSuffix = "c_" + basicHashText[ii] + (pi == 0 ? "" : ("p" + pi)) + "v" + JAVA_CLASS_VERSION; @@ -523,26 +508,26 @@ private void compileHelper( + request.packageNameRoot() + ", class name=" + request.className() + ", class body " + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); resolvers.get(ii).completeExceptionally(err); - compiled[ii] = true; + state.compiled = true; ++numCompiled; break; } - packageName[ii] = request.getPackageName(packageNameSuffix); - fqClassName[ii] = packageName[ii] + "." + request.className(); + state.packageName = request.getPackageName(packageNameSuffix); + state.fqClassName = state.packageName + "." + request.className(); // Ask the classloader to load an existing class with this name. This might: // 1. Fail to find a class (returning null) // 2. Find a class whose body has the formula we are looking for // 3. Find a class whose body has a different formula (hash collision) - Class result = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); + Class result = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); if (result == null) { break; // we'll try to compile it } - if (completeIfResultMatchesQueryCompilerRequest(packageName[ii], request, resolvers.get(ii), + if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolvers.get(ii), result)) { - compiled[ii] = true; + state.compiled = true; ++numCompiled; break; } @@ -554,16 +539,16 @@ private void compileHelper( } // Couldn't resolve at least one of the requests, so try a round of compilation. - final CompilationRequestAttempt[] compilationRequestAttempts = - new CompilationRequestAttempt[requests.size() - numCompiled]; - for (int ii = 0, jj = 0; ii < requests.size(); ++ii) { - if (!compiled[ii]) { + final List compilationRequestAttempts = new ArrayList<>(); + for (int ii = 0; ii < requests.size(); ++ii) { + final CompilationState state = states[ii]; + if (!state.compiled) { final QueryCompilerRequest request = requests.get(ii); - compilationRequestAttempts[jj++] = new CompilationRequestAttempt( + compilationRequestAttempts.add(new CompilationRequestAttempt( request, - packageName[ii], - fqClassName[ii], - resolvers.get(ii)); + state.packageName, + state.fqClassName, + resolvers.get(ii))); } } @@ -573,14 +558,15 @@ private void compileHelper( // ... then give the filesystem some time. All requests should use the same deadline. final long deadline = System.currentTimeMillis() + CODEGEN_TIMEOUT_MS - CODEGEN_LOOP_DELAY_MS; for (int ii = 0; ii < requests.size(); ++ii) { - if (compiled[ii]) { + final CompilationState state = states[ii]; + if (state.compiled) { continue; } final QueryCompilerRequest request = requests.get(ii); final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); if (resolver.getFuture().isDone()) { - compiled[ii] = true; + state.compiled = true; ++numCompiled; continue; } @@ -590,12 +576,12 @@ private void compileHelper( // B. Lost a race to another process on the same file system which is compiling the identical formula // C. Lost a race to another process on the same file system compiling a different formula that collides - Class clazz = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); + Class clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); try { while (clazz == null && System.currentTimeMillis() < deadline) { // noinspection BusyWait Thread.sleep(CODEGEN_LOOP_DELAY_MS); - clazz = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); + clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); } } catch (final InterruptedException ie) { throw new UncheckedDeephavenException("Interrupted while waiting for codegen", ie); @@ -606,8 +592,8 @@ private void compileHelper( throw new IllegalStateException("Should have been able to load *some* class here"); } - if (completeIfResultMatchesQueryCompilerRequest(packageName[ii], request, resolver, clazz)) { - compiled[ii] = true; + if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolver, clazz)) { + state.compiled = true; ++numCompiled; } } @@ -805,7 +791,7 @@ public JavaSourceFromString makeSource() { } private void maybeCreateClass( - @NotNull final CompilationRequestAttempt[] requests) { + @NotNull final List requests) { // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. // Also create a temp directory e.g. /tmp/workspace/cache/classes/temporaryCompilationDirectory12345 @@ -848,21 +834,21 @@ private void maybeCreateClass( final OperationInitializer operationInitializer = ExecutionContext.getContext().getOperationInitializer(); int parallelismFactor = operationInitializer.parallelismFactor(); - int requestsPerTask = Math.max(32, (requests.length + parallelismFactor - 1) / parallelismFactor); + int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); log.info().append("Compiling with parallelismFactor = ").append(parallelismFactor) .append(" requestsPerTask = ").append(requestsPerTask).endl(); - if (parallelismFactor == 1 || requestsPerTask >= requests.length) { + if (parallelismFactor == 1 || requestsPerTask >= requests.size()) { maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, - 0, requests.length, false); + 0, requests.size()); } else { - int numTasks = (requests.length + requestsPerTask - 1) / requestsPerTask; + int numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; final Future[] tasks = new Future[numTasks]; for (int jobId = 0; jobId < numTasks; ++jobId) { final int startInclusive = jobId * requestsPerTask; - final int endExclusive = Math.min(requests.length, (jobId + 1) * requestsPerTask); + final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); tasks[jobId] = operationInitializer.submit(() -> { maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, - startInclusive, endExclusive, false); + startInclusive, endExclusive); }); } for (int jobId = 0; jobId < numTasks; ++jobId) { @@ -897,12 +883,36 @@ private void maybeCreateClass( private void maybeCreateClassHelper( @NotNull final JavaCompiler compiler, @NotNull final JavaFileManager fileManager, - @NotNull final CompilationRequestAttempt[] requests, + @NotNull final List requests, + @NotNull final String rootPathAsString, + @NotNull final String tempDirAsString, + final int startInclusive, + final int endExclusive) { + final List toRetry = new ArrayList<>(); + final boolean wantRetry = maybeCreateClassHelper2(compiler, + fileManager, requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive, toRetry); + if (!wantRetry) { + return; + } + + final List ignored = new ArrayList<>(); + if (maybeCreateClassHelper2(compiler, + fileManager, toRetry, rootPathAsString, tempDirAsString, 0, toRetry.size(), ignored)) { + // We only retried compilation units that did not fail on the first pass, so we should not have any failures + // on the second pass. + throw new IllegalStateException("Unexpected failure during second pass of compilation"); + } + } + + private boolean maybeCreateClassHelper2( + @NotNull final JavaCompiler compiler, + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, @NotNull final String rootPathAsString, @NotNull final String tempDirAsString, final int startInclusive, final int endExclusive, - final boolean isRetry) { + List toRetry) { final StringWriter compilerOutput = new StringWriter(); final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); @@ -930,51 +940,48 @@ private void maybeCreateClassHelper( }, compilerOptions, null, - Arrays.stream(requests, startInclusive, endExclusive) + requests.subList(startInclusive, endExclusive).stream() .map(CompilationRequestAttempt::makeSource) .collect(Collectors.toList())) .call(); - final List shouldRetry; - if (!isRetry && numFailures.intValue() > 0 && numFailures.intValue() != endExclusive - startInclusive) { - // if this is the first attempt, and we had some failures, but not all of them failed, then we should retry - shouldRetry = new ArrayList<>(); - } else { - shouldRetry = null; - } + final boolean wantRetry = numFailures.intValue() > 0 && numFailures.intValue() != endExclusive - startInclusive; // The above has compiled into e.g. // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various // class files} // We want to atomically move it to e.g. // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} - Arrays.stream(requests, startInclusive, endExclusive).forEach(request -> { + requests.subList(startInclusive, endExclusive).forEach(request -> { final Path srcDir = Paths.get(tempDirAsString, request.splitPackageName); final Path destDir = Paths.get(rootPathAsString, request.splitPackageName); try { Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); } catch (IOException ioe) { - if (shouldRetry != null && !Files.exists(srcDir) && !request.resolver.getFuture().isDone()) { - // This source actually succeeded in compiling, but was not written because some other source failed - // to compile. Let's recursively call ourselves to try again. - shouldRetry.add(request); - return; + // The name "isDone" might be misleading here. We haven't called "complete" on the successful + // futures yet, so the only way they would be "done" at this point is if they completed + // exceptionally. + final boolean hasException = request.resolver.getFuture().isDone(); + + if (wantRetry && !Files.exists(srcDir)) { + // The move failed and the source directory does not exist. + if (!hasException) { + // This source actually succeeded in compiling, but was not written because some other source + // failed to compile. Let's schedule this work to try again. + toRetry.add(request); + } } - if (!Files.exists(destDir) && !request.resolver.getFuture().isDone()) { - // The move might have failed for a variety of bad reasons. However, if the reason was because - // we lost the race to some other process, that's a harmless/desirable outcome, and we can ignore - // it. + if (!Files.exists(destDir) && !hasException) { + // Propagate an error here only if the destination does not exist; ignoring issues related to + // collisions with another process. request.resolver.completeExceptionally(new UncheckedIOException( "Move failed for some reason other than destination already existing", ioe)); } } }); - if (shouldRetry != null && !shouldRetry.isEmpty()) { - maybeCreateClassHelper(compiler, fileManager, shouldRetry.toArray(CompilationRequestAttempt[]::new), - rootPathAsString, tempDirAsString, 0, shouldRetry.size(), true); - } + return wantRetry; } /** diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java index 20518ba64fe..99db140e0f4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java @@ -3,10 +3,12 @@ */ package io.deephaven.engine.table.impl; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; +import io.deephaven.util.MultiException; import io.deephaven.util.SafeCloseable; import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.CompletionStageFutureImpl; @@ -14,6 +16,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutionException; public interface QueryCompilerRequestProcessor { /** @@ -80,12 +83,25 @@ public void compile() { try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(desc)) { final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); - if (requests.size() == 1) { - compiler.compile(requests.get(0), resolvers.get(0)); - } else { - compiler.compile( - requests.toArray(QueryCompilerRequest[]::new), - resolvers.toArray(CompletionStageFuture.Resolver[]::new)); + // noinspection unchecked + compiler.compile( + requests.toArray(QueryCompilerRequest[]::new), + resolvers.toArray(CompletionStageFuture.Resolver[]::new)); + + final List exceptions = new ArrayList<>(); + for (CompletionStageFuture.Resolver> resolver : resolvers) { + try { + Object ignored2 = resolver.getFuture().get(); + } catch (ExecutionException err) { + exceptions.add(err.getCause()); + } catch (InterruptedException err) { + exceptions.add(err); + } + } + if (!exceptions.isEmpty()) { + throw new UncheckedDeephavenException("Compilation failed", + MultiException.maybeWrapInMultiException("Compilation exceptions for multiple requests", + exceptions)); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java index a0937aaab04..318e99ef80b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java @@ -286,8 +286,7 @@ protected Future createKernelFormulaFactory( return formulaKernelFactoryFuture .thenApply(formulaKernelFactory -> (columnName, rowSet, lazy, columnsToData, params) -> { // Maybe warn that we ignore "lazy". By the way, "lazy" is the wrong term anyway. "lazy" doesn't - // mean - // "cached", which is how we are using it. + // mean "cached", which is how we are using it. final Map> netColumnSources = new HashMap<>(); for (final String sourceColumnName : sd.sources) { final ColumnSource columnSourceToUse = columnsToData.get(sourceColumnName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java index d6ee54c2b0d..dc997685e2b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java @@ -52,8 +52,8 @@ public enum Mode { public static Supplier> newQueryScopeVariableSupplier() { final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); - return new CachingSupplier<>( - () -> queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name))); + return new CachingSupplier<>(() -> Collections.unmodifiableMap( + queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name)))); } public static void initializeSelectColumns( diff --git a/open-api/lang-parser/lang-parser.gradle b/open-api/lang-parser/lang-parser.gradle index 8baf8dc5632..d785dc7c187 100644 --- a/open-api/lang-parser/lang-parser.gradle +++ b/open-api/lang-parser/lang-parser.gradle @@ -10,6 +10,7 @@ plugins { apply from: "$rootDir/gradle/web-common.gradle" dependencies { + implementation project(':Util') api project(':open-api-shared-fu') implementation project(':log-factory') api project(':proto:proto-backplane-grpc') diff --git a/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/ParsedDocument.java b/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/ParsedDocument.java index 93f3269a790..a75cfb9ddfd 100644 --- a/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/ParsedDocument.java +++ b/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/ParsedDocument.java @@ -3,7 +3,6 @@ // package io.deephaven.lang.parse; -import io.deephaven.base.Lazy; import io.deephaven.base.verify.Assert; import io.deephaven.io.logger.Logger; import io.deephaven.lang.generated.*; @@ -11,6 +10,7 @@ import io.deephaven.proto.backplane.script.grpc.DocumentRange; import io.deephaven.proto.backplane.script.grpc.Position; import io.deephaven.proto.backplane.script.grpc.TextEdit; +import io.deephaven.util.datastructures.CachingSupplier; import io.deephaven.web.shared.fu.MappedIterable; import java.util.*; @@ -100,7 +100,7 @@ public boolean containsIndex(int i) { private static final Pattern NEW_LINE_PATTERN = Pattern.compile("\\r?\\n"); private final ChunkerDocument doc; - private final Lazy> statements; + private final CachingSupplier> statements; private final String src; private String errorSource; private ParseException error; @@ -112,7 +112,7 @@ public ParsedDocument(ChunkerDocument doc, String document) { this.src = document; computedPositions = new ConcurrentHashMap<>(4); assignments = new ConcurrentHashMap<>(12); - statements = new Lazy<>(() -> { + statements = new CachingSupplier<>(() -> { final LinkedHashSet stmts = new LinkedHashSet<>(); doc.childrenAccept(new ChunkerDefaultVisitor() { @Override diff --git a/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/CompletionLookups.java b/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/CompletionLookups.java index 554c043458e..e2d928cb7e7 100644 --- a/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/CompletionLookups.java +++ b/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/CompletionLookups.java @@ -3,11 +3,11 @@ // package io.deephaven.lang.completion; -import io.deephaven.base.Lazy; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.util.ScriptSession; +import io.deephaven.util.datastructures.CachingSupplier; import java.util.Collection; import java.util.Map; @@ -26,15 +26,15 @@ public class CompletionLookups { private static final WeakHashMap lookups = new WeakHashMap<>(); - private final Lazy>> statics; + private final CachingSupplier>> statics; private final Map referencedTables; - private final Lazy customCompletions; + private final CachingSupplier customCompletions; public CompletionLookups(Set customCompletionFactory) { final QueryLibrary ql = ExecutionContext.getContext().getQueryLibrary(); - statics = new Lazy<>(ql::getStaticImports); + statics = new CachingSupplier<>(ql::getStaticImports); referencedTables = new ConcurrentHashMap<>(); - customCompletions = new Lazy<>(() -> new DelegatingCustomCompletion(customCompletionFactory)); + customCompletions = new CachingSupplier<>(() -> new DelegatingCustomCompletion(customCompletionFactory)); // This can be slow, so lets start it on a background thread right away. final ForkJoinPool pool = ForkJoinPool.commonPool(); From 202edf385c9db230249c892f124e476a36328927 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 5 Mar 2024 11:48:40 -0700 Subject: [PATCH 03/19] Test fix --- .../io/deephaven/engine/context/QueryCompiler.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 4791a34ba03..05502ff4812 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -220,8 +220,14 @@ public Class compile(@NotNull final QueryCompilerRequest request) { compile(request, resolver); try { return resolver.getFuture().get(); - } catch (InterruptedException | ExecutionException e) { - throw new UncheckedDeephavenException("Could not compile class", e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + throw new UncheckedDeephavenException("Error while compiling class", cause); + } catch (InterruptedException e) { + throw new UncheckedDeephavenException("Interrupted while compile class", e); } } From 6d9ef348ca3ba23d94a217e08c0233e4e90fb436 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 5 Mar 2024 13:16:29 -0700 Subject: [PATCH 04/19] Test fix 2 --- .../io/deephaven/engine/context/QueryCompiler.java | 1 + .../deephaven/engine/context/TestQueryCompiler.java | 11 +---------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 05502ff4812..0302ea55d9a 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -975,6 +975,7 @@ private boolean maybeCreateClassHelper2( // This source actually succeeded in compiling, but was not written because some other source // failed to compile. Let's schedule this work to try again. toRetry.add(request); + return; } } diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index aa00c48473d..aa19e47dc45 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -299,24 +299,15 @@ public void testMultiCompileWithFailure() throws ExecutionException, Interrupted CompletionStageFutureImpl.make(), }; - Exception firstErr; try { ExecutionContext.getContext().getQueryCompiler().compile(requests, resolvers); // noinspection DataFlowIssue throw Assert.statementNeverExecuted(); - } catch (Exception err) { - firstErr = err; + } catch (Exception ignored) { } Assert.eqTrue(resolvers[0].getFuture().isDone(), "resolvers[0].getFuture().isDone()"); Assert.eqTrue(resolvers[1].getFuture().isDone(), "resolvers[0].getFuture().isDone()"); Assert.neqNull(resolvers[1].getFuture().get(), "resolvers[1].getFuture().get()"); - try { - resolvers[0].getFuture().get(); - // noinspection DataFlowIssue - throw Assert.statementNeverExecuted(); - } catch (ExecutionException err) { - Assert.eq(firstErr, "firstErr", err.getCause(), "err"); - } } } From ef308a7b2f0e89506f0bc19252922553f133eca1 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 7 Mar 2024 10:17:25 -0700 Subject: [PATCH 05/19] Rebase --- .../main/java/io/deephaven/util/CompletionStageFuture.java | 6 +++--- .../java/io/deephaven/util/CompletionStageFutureImpl.java | 6 +++--- .../io/deephaven/util/datastructures/CachingSupplier.java | 6 +++--- .../io/deephaven/engine/context/QueryCompilerRequest.java | 6 +++--- .../engine/context/util/SynchronizedJavaFileManager.java | 4 +++- .../engine/table/impl/QueryCompilerRequestProcessor.java | 6 +++--- .../updateby/rollingformula/CharRollingFormulaOperator.java | 4 ++-- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java index c410765e187..87eeaf57151 100644 --- a/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java @@ -1,6 +1,6 @@ -/** - * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending - */ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// package io.deephaven.util; import org.jetbrains.annotations.NotNull; diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java index d0ebef551fe..59884161404 100644 --- a/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java @@ -1,6 +1,6 @@ -/** - * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending - */ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// package io.deephaven.util; import org.jetbrains.annotations.NotNull; diff --git a/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java index 62a60707d4c..0a0880f7896 100644 --- a/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java +++ b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java @@ -1,6 +1,6 @@ -/** - * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending - */ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// package io.deephaven.util.datastructures; import org.jetbrains.annotations.NotNull; diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java index 38be0b97fe4..6f7608fa5ef 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java @@ -1,6 +1,6 @@ -/** - * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending - */ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// package io.deephaven.engine.context; import io.deephaven.annotations.BuildableStyle; diff --git a/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java b/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java index 80a4d0c1667..d03a5bfdb81 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java @@ -1,10 +1,12 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// package io.deephaven.engine.context.util; import javax.tools.FileObject; import javax.tools.JavaFileManager; import javax.tools.JavaFileObject; import java.io.IOException; -import java.io.UncheckedIOException; import java.util.Iterator; import java.util.ServiceLoader; import java.util.Set; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java index 99db140e0f4..08f3cac184d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java @@ -1,6 +1,6 @@ -/** - * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending - */ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// package io.deephaven.engine.table.impl; import io.deephaven.UncheckedDeephavenException; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java index 73dd6d03370..86a490caf42 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java @@ -164,8 +164,8 @@ public CharRollingFormulaOperator( @NotNull final TableDefinition tableDef, @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor - // region extra-constructor-args - // endregion extra-constructor-args + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); From 82c85a8bf4fa61aed579e924edb7a2c126707dcf Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 21 Mar 2024 17:05:20 -0600 Subject: [PATCH 06/19] Ryan's feedback --- .../deephaven/util/CompletionStageFuture.java | 34 +++++--- .../util/CompletionStageFutureImpl.java | 12 +-- .../io/deephaven/util/MultiException.java | 2 +- ...Function.java => SoftCachingFunction.java} | 6 +- ...Supplier.java => SoftCachingSupplier.java} | 4 +- .../engine/context/QueryCompiler.java | 52 ++++++------ .../engine/context/TestQueryCompiler.java | 5 +- .../engine/table/impl/DeferredViewTable.java | 21 ++--- .../table/impl/PartitionAwareSourceTable.java | 8 +- .../impl/QueryCompilerRequestProcessor.java | 79 +++++++++++++++++-- .../engine/table/impl/QueryTable.java | 6 +- .../engine/table/impl/RedefinableTable.java | 7 +- .../table/impl/WouldMatchOperation.java | 13 +-- .../table/impl/by/AggregationProcessor.java | 11 +-- .../table/impl/by/FormulaChunkedOperator.java | 4 +- .../BaseNodeOperationsRecorder.java | 4 +- .../impl/hierarchical/RollupTableImpl.java | 8 +- .../TreeNodeOperationsRecorder.java | 9 +-- .../impl/hierarchical/TreeTableFilter.java | 10 +-- .../impl/hierarchical/TreeTableImpl.java | 10 +-- .../BaseTableTransformationColumn.java | 1 - .../BiTableTransformationColumn.java | 7 +- .../impl/partitioned/LongConstantColumn.java | 7 +- .../partitioned/PartitionedTableImpl.java | 8 +- .../PartitionedTableProxyImpl.java | 7 +- .../TableTransformationColumn.java | 7 +- .../rangejoin/ValidFloatingPointFilter.java | 8 +- .../impl/select/AbstractConditionFilter.java | 9 ++- .../impl/select/AbstractFormulaColumn.java | 17 ++-- .../select/BaseIncrementalReleaseFilter.java | 8 +- .../table/impl/select/ByteRangeFilter.java | 9 +-- .../table/impl/select/CharRangeFilter.java | 9 +-- .../engine/table/impl/select/ClockFilter.java | 8 +- .../impl/select/ComparableRangeFilter.java | 9 +-- .../table/impl/select/ComposedFilter.java | 11 ++- .../table/impl/select/ConditionFilter.java | 18 ++--- .../table/impl/select/DhFormulaColumn.java | 25 +++--- .../table/impl/select/DoubleRangeFilter.java | 9 +-- .../impl/select/DownsampledWhereFilter.java | 8 +- .../table/impl/select/DynamicWhereFilter.java | 6 +- .../table/impl/select/FloatRangeFilter.java | 9 +-- .../table/impl/select/FunctionalColumn.java | 7 +- .../table/impl/select/InstantRangeFilter.java | 8 +- .../table/impl/select/IntRangeFilter.java | 9 +-- .../table/impl/select/LongRangeFilter.java | 9 +-- .../engine/table/impl/select/MatchFilter.java | 11 +-- .../select/MultiSourceFunctionalColumn.java | 7 +- .../table/impl/select/NullSelectColumn.java | 7 +- .../impl/select/RangeConditionFilter.java | 14 ++-- .../impl/select/ReinterpretedColumn.java | 7 +- .../impl/select/RollingReleaseFilter.java | 8 +- .../table/impl/select/SelectColumn.java | 20 ++--- .../table/impl/select/ShortRangeFilter.java | 9 +-- .../SingleSidedComparableRangeFilter.java | 9 +-- .../table/impl/select/SourceColumn.java | 7 +- .../table/impl/select/SwitchColumn.java | 10 ++- .../table/impl/select/TimeSeriesFilter.java | 8 +- .../engine/table/impl/select/WhereFilter.java | 20 ++--- .../impl/select/WhereFilterInvertedImpl.java | 11 +-- .../impl/select/WhereFilterPatternImpl.java | 8 +- .../table/impl/select/WhereNoneFilter.java | 8 +- .../analyzers/SelectAndViewAnalyzer.java | 25 ++---- .../select/python/FormulaColumnPython.java | 15 ++-- .../updateby/UpdateByOperatorFactory.java | 25 +++--- .../BaseRollingFormulaOperator.java | 5 +- .../BooleanRollingFormulaOperator.java | 4 +- .../ByteRollingFormulaOperator.java | 8 +- .../CharRollingFormulaOperator.java | 4 +- .../DoubleRollingFormulaOperator.java | 8 +- .../FloatRollingFormulaOperator.java | 8 +- .../IntRollingFormulaOperator.java | 8 +- .../LongRollingFormulaOperator.java | 8 +- .../ObjectRollingFormulaOperator.java | 4 +- .../ShortRollingFormulaOperator.java | 8 +- .../engine/util/DynamicCompileUtils.java | 1 - .../impl/QueryTableWhereParallelTest.java | 7 +- .../impl/lang/TestQueryLanguageParser.java | 5 +- .../parquet/base/ColumnChunkReaderImpl.java | 4 +- .../pagestore/topage/ChunkDictionary.java | 6 +- .../replicators/ReplicateCachingSupplier.java | 8 +- 80 files changed, 333 insertions(+), 530 deletions(-) rename Util/src/main/java/io/deephaven/util/datastructures/{LazyCachingFunction.java => SoftCachingFunction.java} (89%) rename Util/src/main/java/io/deephaven/util/datastructures/{LazyCachingSupplier.java => SoftCachingSupplier.java} (91%) diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java index 87eeaf57151..0564f9dbf01 100644 --- a/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java @@ -5,8 +5,8 @@ import org.jetbrains.annotations.NotNull; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.Future; import java.util.function.BiConsumer; @@ -21,31 +21,40 @@ */ public interface CompletionStageFuture extends Future, CompletionStage { + /** + * Create a new incomplete future. + * + * @param The result type returned by this future's {@code join} + * @return a resolver for the future + */ + static Resolver make() { + return new CompletionStageFutureImpl().new ResolverImpl(); + } + /** * Returns a new CompletionStageFuture that is already completed with the given value. * * @param value the value * @param the type of the value * @return the completed CompletionStageFuture - * @see CompletableFuture#completedFuture(Object) + * @see java.util.concurrent.CompletableFuture#completedFuture(Object) */ static CompletionStageFuture completedFuture(U value) { - final CompletionStageFutureImpl.Resolver resolver = CompletionStageFutureImpl.make(); + final CompletionStageFutureImpl.Resolver resolver = CompletionStageFuture.make(); resolver.complete(value); return resolver.getFuture(); } /** - * Returns a new CompletableFuture that is already completed exceptionally with the given exception. + * Returns a new CompletionStageFuture that is already completed exceptionally with the given exception. * * @param ex the exception * @param the type of the value - * @return the exceptionally completed CompletableFuture - * @since 9 - * @see CompletableFuture#failedFuture(Throwable) + * @return the exceptionally completed CompletionStageFuture + * @see java.util.concurrent.CompletableFuture#failedFuture(Throwable) */ static CompletionStageFuture failedFuture(Throwable ex) { - final CompletionStageFutureImpl.Resolver resolver = CompletionStageFutureImpl.make(); + final CompletionStageFutureImpl.Resolver resolver = CompletionStageFuture.make(); resolver.completeExceptionally(ex); return resolver.getFuture(); } @@ -56,7 +65,7 @@ interface Resolver { * If not already completed, sets the value returned by {@link #get()} and related methods to the given value. * * @param value the result value - * @return {@code true} if this invocation caused this SafeCompletableFuture to transition to a completed state, + * @return {@code true} if this invocation caused this CompletionStageFuture to transition to a completed state, * else {@code false} * @see java.util.concurrent.CompletableFuture#complete(Object) */ @@ -64,17 +73,18 @@ interface Resolver { /** * If not already completed, causes invocations of {@link #get()} and related methods to throw the given - * exception. + * exception wrapped in an {@link ExecutionException}. * * @param ex the exception - * @return {@code true} if this invocation caused this SafeCompletableFuture to transition to a completed state, + * @return {@code true} if this invocation caused this CompletionStageFuture to transition to a completed state, * else {@code false} * @see java.util.concurrent.CompletableFuture#completeExceptionally(Throwable) */ boolean completeExceptionally(@NotNull Throwable ex); /** - * @return the underlying future to provide to the recipient + * @return the underlying future to provide to the recipient; implementations must ensure that this method + * always returns an identical result for a given Resolver instance */ CompletionStageFuture getFuture(); } diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java index 59884161404..df86fb942b0 100644 --- a/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java @@ -20,20 +20,10 @@ @SuppressWarnings("unchecked") public class CompletionStageFutureImpl extends CompletableFuture implements CompletionStageFuture { - /** - * Create a new incomplete future. - * - * @param The result type returned by this future's {@code join} - * @return a resolver for the future - */ - public static Resolver make() { - return new CompletionStageFutureImpl().new ResolverImpl(); - } - /** * A resolver for this future implementation. */ - private class ResolverImpl implements CompletionStageFuture.Resolver { + class ResolverImpl implements CompletionStageFuture.Resolver { public boolean complete(final T value) { return safelyComplete(value); } diff --git a/Util/src/main/java/io/deephaven/util/MultiException.java b/Util/src/main/java/io/deephaven/util/MultiException.java index d8f275e5aaa..f2dcc13a90c 100644 --- a/Util/src/main/java/io/deephaven/util/MultiException.java +++ b/Util/src/main/java/io/deephaven/util/MultiException.java @@ -8,7 +8,7 @@ import java.util.List; /** - * An exception to use when a series of operations mus all be executed, but may all throw exceptions themselves. This + * An exception to use when a series of operations must all be executed, but may all throw exceptions themselves. This * allows for retention of all exception data. */ public class MultiException extends Exception { diff --git a/Util/src/main/java/io/deephaven/util/datastructures/LazyCachingFunction.java b/Util/src/main/java/io/deephaven/util/datastructures/SoftCachingFunction.java similarity index 89% rename from Util/src/main/java/io/deephaven/util/datastructures/LazyCachingFunction.java rename to Util/src/main/java/io/deephaven/util/datastructures/SoftCachingFunction.java index 05b654fc1bc..e3521335875 100644 --- a/Util/src/main/java/io/deephaven/util/datastructures/LazyCachingFunction.java +++ b/Util/src/main/java/io/deephaven/util/datastructures/SoftCachingFunction.java @@ -2,7 +2,7 @@ // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending // // ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY -// ****** Edit LazyCachingSupplier and run "./gradlew replicateCachingSupplier" to regenerate +// ****** Edit SoftCachingSupplier and run "./gradlew replicateCachingSupplier" to regenerate // // @formatter:off package io.deephaven.util.datastructures; @@ -18,7 +18,7 @@ * * @param the type of results supplied by this function */ -public final class LazyCachingFunction implements Function { +public final class SoftCachingFunction implements Function { private final Function internalFunction; @@ -29,7 +29,7 @@ public final class LazyCachingFunction implements Funct * * @param internalFunction The {@link Function} to wrap. Must be safely repeatable and must not return {@code null}. */ - public LazyCachingFunction(@NotNull final Function internalFunction) { + public SoftCachingFunction(@NotNull final Function internalFunction) { this.internalFunction = internalFunction; } diff --git a/Util/src/main/java/io/deephaven/util/datastructures/LazyCachingSupplier.java b/Util/src/main/java/io/deephaven/util/datastructures/SoftCachingSupplier.java similarity index 91% rename from Util/src/main/java/io/deephaven/util/datastructures/LazyCachingSupplier.java rename to Util/src/main/java/io/deephaven/util/datastructures/SoftCachingSupplier.java index 478d33c0901..788a61c954e 100644 --- a/Util/src/main/java/io/deephaven/util/datastructures/LazyCachingSupplier.java +++ b/Util/src/main/java/io/deephaven/util/datastructures/SoftCachingSupplier.java @@ -14,7 +14,7 @@ * * @param the type of results supplied by this supplier */ -public final class LazyCachingSupplier implements Supplier { +public final class SoftCachingSupplier implements Supplier { private final Supplier internalSupplier; @@ -25,7 +25,7 @@ public final class LazyCachingSupplier implements Supplier internalSupplier) { + public SoftCachingSupplier(@NotNull final Supplier internalSupplier) { this.internalSupplier = internalSupplier; } diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 0302ea55d9a..1152569987c 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -14,7 +14,6 @@ import io.deephaven.io.logger.Logger; import io.deephaven.util.ByteUtils; import io.deephaven.util.CompletionStageFuture; -import io.deephaven.util.CompletionStageFutureImpl; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; @@ -216,7 +215,7 @@ public void setParentClassLoader(final ClassLoader parentClassLoader) { * @param request The compilation request */ public Class compile(@NotNull final QueryCompilerRequest request) { - final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + final CompletionStageFuture.Resolver> resolver = CompletionStageFuture.make(); compile(request, resolver); try { return resolver.getFuture().get(); @@ -227,7 +226,7 @@ public Class compile(@NotNull final QueryCompilerRequest request) { } throw new UncheckedDeephavenException("Error while compiling class", cause); } catch (InterruptedException e) { - throw new UncheckedDeephavenException("Interrupted while compile class", e); + throw new UncheckedDeephavenException("Interrupted while compiling class", e); } } @@ -235,6 +234,7 @@ public Class compile(@NotNull final QueryCompilerRequest request) { * Compile a class. * * @param request The compilation request + * @param resolver The resolver to use for delivering compilation results */ public void compile( @NotNull final QueryCompilerRequest request, @@ -247,6 +247,7 @@ public void compile( * Compiles all requests. * * @param requests The compilation requests + * @param resolvers The resolvers to use for delivering compilation results */ public void compile( @NotNull final QueryCompilerRequest[] requests, @@ -459,8 +460,8 @@ private String getClassPath() { } private static class CompilationState { - int next_pi; - boolean compiled; + int nextProbeIndex; + boolean complete; String packageName; String fqClassName; } @@ -481,7 +482,7 @@ private void compileHelper( requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); } - int numCompiled = 0; + int numComplete = 0; final CompilationState[] states = new CompilationState[requests.size()]; for (int ii = 0; ii < requests.size(); ++ii) { states[ii] = new CompilationState(); @@ -489,21 +490,21 @@ private void compileHelper( /* * @formatter:off - * 1. try to resolve CFs without compiling; retain next hash to try + * 1. try to resolve without compiling; retain next hash to try * 2. compile all remaining with a single compilation task * 3. goto step 1 if any are unresolved * @formatter:on */ - while (numCompiled < requests.size()) { + while (numComplete < requests.size()) { for (int ii = 0; ii < requests.size(); ++ii) { final CompilationState state = states[ii]; - if (state.compiled) { + if (state.complete) { continue; } while (true) { - final int pi = state.next_pi++; + final int pi = state.nextProbeIndex++; final String packageNameSuffix = "c_" + basicHashText[ii] + (pi == 0 ? "" : ("p" + pi)) + "v" + JAVA_CLASS_VERSION; @@ -514,8 +515,8 @@ private void compileHelper( + request.packageNameRoot() + ", class name=" + request.className() + ", class body " + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); resolvers.get(ii).completeExceptionally(err); - state.compiled = true; - ++numCompiled; + state.complete = true; + ++numComplete; break; } @@ -533,14 +534,14 @@ private void compileHelper( if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolvers.get(ii), result)) { - state.compiled = true; - ++numCompiled; + state.complete = true; + ++numComplete; break; } } } - if (numCompiled == requests.size()) { + if (numComplete == requests.size()) { return; } @@ -548,7 +549,7 @@ private void compileHelper( final List compilationRequestAttempts = new ArrayList<>(); for (int ii = 0; ii < requests.size(); ++ii) { final CompilationState state = states[ii]; - if (!state.compiled) { + if (!state.complete) { final QueryCompilerRequest request = requests.get(ii); compilationRequestAttempts.add(new CompilationRequestAttempt( request, @@ -558,22 +559,22 @@ private void compileHelper( } } - maybeCreateClass(compilationRequestAttempts); + maybeCreateClasses(compilationRequestAttempts); // We could be running on a screwy filesystem that is slow (e.g. NFS). If we wrote a file and can't load it // ... then give the filesystem some time. All requests should use the same deadline. final long deadline = System.currentTimeMillis() + CODEGEN_TIMEOUT_MS - CODEGEN_LOOP_DELAY_MS; for (int ii = 0; ii < requests.size(); ++ii) { final CompilationState state = states[ii]; - if (state.compiled) { + if (state.complete) { continue; } final QueryCompilerRequest request = requests.get(ii); final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); if (resolver.getFuture().isDone()) { - state.compiled = true; - ++numCompiled; + state.complete = true; + ++numComplete; continue; } @@ -599,8 +600,8 @@ private void compileHelper( } if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolver, clazz)) { - state.compiled = true; - ++numCompiled; + state.complete = true; + ++numComplete; } } } @@ -796,7 +797,7 @@ public JavaSourceFromString makeSource() { } } - private void maybeCreateClass( + private void maybeCreateClasses( @NotNull final List requests) { // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. @@ -841,8 +842,6 @@ private void maybeCreateClass( int parallelismFactor = operationInitializer.parallelismFactor(); int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); - log.info().append("Compiling with parallelismFactor = ").append(parallelismFactor) - .append(" requestsPerTask = ").append(requestsPerTask).endl(); if (parallelismFactor == 1 || requestsPerTask >= requests.size()) { maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, 0, requests.size()); @@ -895,6 +894,9 @@ private void maybeCreateClassHelper( final int startInclusive, final int endExclusive) { final List toRetry = new ArrayList<>(); + // If any of our requests fail to compile then the JavaCompiler will not write any class files at all. The + // non-failing requests will be retried in a second pass that is expected to succeed. This enables us to + // fulfill futures independent of each other; otherwise a single failure would taint all requests in a batch. final boolean wantRetry = maybeCreateClassHelper2(compiler, fileManager, requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive, toRetry); if (!wantRetry) { diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index aa19e47dc45..3a42aabfa9c 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -9,7 +9,6 @@ import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.time.DateTimeUtils; import io.deephaven.util.CompletionStageFuture; -import io.deephaven.util.CompletionStageFutureImpl; import io.deephaven.util.SafeCloseable; import org.junit.After; import org.junit.Before; @@ -295,8 +294,8 @@ public void testMultiCompileWithFailure() throws ExecutionException, Interrupted // noinspection unchecked CompletionStageFuture.Resolver>[] resolvers = (CompletionStageFuture.Resolver>[]) new CompletionStageFuture.Resolver[] { - CompletionStageFutureImpl.make(), - CompletionStageFutureImpl.make(), + CompletionStageFuture.make(), + CompletionStageFuture.make(), }; try { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java index c87b539058a..b2328798e78 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java @@ -20,7 +20,6 @@ import java.util.*; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -47,14 +46,12 @@ public DeferredViewTable(@NotNull final TableDefinition definition, this.deferredViewColumns = deferredViewColumns == null ? SelectColumn.ZERO_LENGTH_SELECT_COLUMN_ARRAY : deferredViewColumns; final TableDefinition parentDefinition = tableReference.getDefinition(); - final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); SelectAndViewAnalyzer.initializeSelectColumns( - parentDefinition.getColumnNameMap(), variableSupplier, this.deferredViewColumns, compilationProcessor); + parentDefinition.getColumnNameMap(), this.deferredViewColumns, compilationProcessor); this.deferredFilters = deferredFilters == null ? WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY : deferredFilters; for (final WhereFilter sf : this.deferredFilters) { - sf.init(parentDefinition, variableSupplier, compilationProcessor); + sf.init(parentDefinition, compilationProcessor); if (sf instanceof LivenessReferent && sf.isRefreshing()) { manage((LivenessReferent) sf); setRefreshing(true); @@ -83,11 +80,9 @@ public DeferredViewTable(@NotNull final TableDefinition definition, @Override public Table where(Filter filter) { final WhereFilter[] whereFilters = WhereFilter.fromInternal(filter); - final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (WhereFilter f : whereFilters) { - f.init(definition, variableSupplier, compilationProcessor); + f.init(definition, compilationProcessor); } compilationProcessor.compile(); return getResultTableWithWhere(whereFilters); @@ -198,11 +193,9 @@ private PreAndPostFilters applyFilterRenamings(WhereFilter[] filters) { } } - final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (final WhereFilter filter : filters) { - filter.init(definition, variableSupplier, compilationProcessor); + filter.init(definition, compilationProcessor); final boolean isPostView = Stream.of(filter.getColumns(), filter.getColumnArrays()) .flatMap(Collection::stream) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java index f219651845a..4adde99c515 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java @@ -3,7 +3,6 @@ // package io.deephaven.engine.table.impl; -import io.deephaven.api.ColumnName; import io.deephaven.api.Selectable; import io.deephaven.api.filter.Filter; import io.deephaven.base.verify.Assert; @@ -25,7 +24,6 @@ import java.util.*; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -286,11 +284,9 @@ private Table whereImpl(final WhereFilter[] whereFilters) { Set groupingColumnNames = groupingColumns.stream().map(ColumnDefinition::getName).collect(Collectors.toSet()); - final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition, variableSupplier, compilationProcessor); + whereFilter.init(definition, compilationProcessor); List columns = whereFilter.getColumns(); if (whereFilter instanceof ReindexingFilter) { otherFilters.add(whereFilter); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java index 08f3cac184d..4244d076df7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java @@ -4,21 +4,56 @@ package io.deephaven.engine.table.impl; import io.deephaven.UncheckedDeephavenException; +import io.deephaven.api.util.NameValidator; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.util.MultiException; import io.deephaven.util.SafeCloseable; import io.deephaven.util.CompletionStageFuture; -import io.deephaven.util.CompletionStageFutureImpl; +import io.deephaven.util.datastructures.CachingSupplier; import org.jetbrains.annotations.NotNull; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; public interface QueryCompilerRequestProcessor { + + /** + * @return An immediate QueryCompilerRequestProcessor + */ + static QueryCompilerRequestProcessor.ImmediateProcessor immediate() { + return new ImmediateProcessor(); + } + + /** + * @return A batch QueryCompilerRequestProcessor + */ + static QueryCompilerRequestProcessor.BatchProcessor batch() { + return new BatchProcessor(); + } + + /** + * @return a CachingSupplier that supplies a snapshot of the current query scope variables + */ + static CachingSupplier> newQueryScopeVariableSupplier() { + final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); + return new CachingSupplier<>(() -> Collections.unmodifiableMap( + queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name)))); + } + + /** + * @return a lazily cached snapshot of the current query scope variables + */ + Map getQueryScopeVariables(); + /** * Submit a request for compilation. The QueryCompilerRequestProcessor is not required to immediately compile this * request. @@ -31,16 +66,38 @@ public interface QueryCompilerRequestProcessor { * A QueryCompilerRequestProcessor that immediately compiles requests. */ class ImmediateProcessor implements QueryCompilerRequestProcessor { - public static final ImmediateProcessor INSTANCE = new ImmediateProcessor(); + + private final CachingSupplier> queryScopeVariableSupplier = newQueryScopeVariableSupplier(); + + private ImmediateProcessor() { + // force use of static factory method + } + + @Override + public Map getQueryScopeVariables() { + return queryScopeVariableSupplier.get(); + } @Override public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { - final String desc = "Compile " + request.description(); - final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + final String desc = "Compile: " + request.description(); + final CompletionStageFuture.Resolver> resolver = CompletionStageFuture.make(); try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(desc)) { ExecutionContext.getContext().getQueryCompiler().compile(request, resolver); } - return resolver.getFuture(); + + // The earlier we validate the future, the better. + final CompletionStageFuture> future = resolver.getFuture(); + try { + future.get(0, TimeUnit.SECONDS); + } catch (ExecutionException err) { + throw new UncheckedDeephavenException("Compilation failed", err.getCause()); + } catch (InterruptedException | TimeoutException err) { + // This should never happen since the future is already completed. + throw new UncheckedDeephavenException("Caught unexpected exception", err); + } + + return future; } } @@ -52,10 +109,20 @@ public CompletionStageFuture> submit(@NotNull final QueryCompilerReques class BatchProcessor implements QueryCompilerRequestProcessor { private final List requests = new ArrayList<>(); private final List>> resolvers = new ArrayList<>(); + private final CachingSupplier> queryScopeVariableSupplier = newQueryScopeVariableSupplier(); + + private BatchProcessor() { + // force use of static factory method + } + + @Override + public Map getQueryScopeVariables() { + return queryScopeVariableSupplier.get(); + } @Override public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { - final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + final CompletionStageFuture.Resolver> resolver = CompletionStageFuture.make(); requests.add(request); resolvers.add(resolver); return resolver.getFuture(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java index 7acd8e9187a..1a6be0e420a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java @@ -1187,11 +1187,9 @@ private QueryTable whereInternal(final WhereFilter... filters) { List selectFilters = new LinkedList<>(); List>>> shiftColPairs = new LinkedList<>(); final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); - final Supplier> variableSupplier = - SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + QueryCompilerRequestProcessor.batch(); for (final WhereFilter filter : filters) { - filter.init(getDefinition(), variableSupplier, compilationProcessor); + filter.init(getDefinition(), compilationProcessor); if (filter instanceof AbstractConditionFilter && ((AbstractConditionFilter) filter).hasConstantArrayAccess()) { shiftColPairs.add(((AbstractConditionFilter) filter).getFormulaShiftColPair()); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java index 684cb4e8a2d..819e1d196be 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java @@ -9,7 +9,6 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.select.*; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import org.jetbrains.annotations.NotNull; import java.util.*; @@ -51,11 +50,9 @@ private Table viewInternal(Collection selectables, boolean final Map> allColumns = new HashMap<>(definition.getColumnNameMap()); boolean simpleRetain = true; - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (final SelectColumn selectColumn : columns) { - List usedColumnNames = selectColumn.initDef( - allColumns, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), compilationProcessor); + List usedColumnNames = selectColumn.initDef(allColumns, compilationProcessor); usedColumnNames.addAll(selectColumn.getColumnArrays()); resultColumnsInternal.addAll(usedColumnNames.stream() .filter(usedColumnName -> !resultColumnsExternal.containsKey(usedColumnName)) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java index 97ee9353f62..b153f5822f0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java @@ -10,7 +10,6 @@ import io.deephaven.engine.rowset.*; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.*; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.liveness.LivenessArtifact; import io.deephaven.engine.table.impl.select.WhereFilter; @@ -22,7 +21,6 @@ import org.jetbrains.annotations.Nullable; import java.util.*; -import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -40,7 +38,7 @@ public class WouldMatchOperation implements QueryTable.MemoizableOperation initialize(boolean usePrev, long beforeClock) { final List dependencies = new ArrayList<>(); final Map> newColumns = new LinkedHashMap<>(parent.getColumnSourceMap()); - final Supplier> variableSupplier = - SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + QueryCompilerRequestProcessor.batch(); final WhereFilter[] filters = matchColumns.stream().map(ColumnHolder::getFilter) - .peek(holder -> holder.init(parent.getDefinition(), variableSupplier, compilationProcessor)) + .peek(holder -> holder.init(parent.getDefinition(), compilationProcessor)) .toArray(WhereFilter[]::new); compilationProcessor.compile(); @@ -332,7 +328,6 @@ public WritableRowSet match( final Object... keys) { boolean hasFalse = false; boolean hasTrue = false; - boolean hasOther = false; for (Object key : keys) { if (key instanceof Boolean) { @@ -341,8 +336,6 @@ public WritableRowSet match( } else { hasFalse = true; } - } else { - hasOther = true; } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java index 505955435d6..2b751681071 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java @@ -93,7 +93,6 @@ import io.deephaven.engine.table.impl.by.ssmcountdistinct.unique.ShortRollupUniqueOperator; import io.deephaven.engine.table.impl.by.ssmminmax.SsmChunkedMinMaxOperator; import io.deephaven.engine.table.impl.by.ssmpercentile.SsmChunkedPercentileOperator; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.ssms.SegmentedSortedMultiSet; import io.deephaven.engine.table.impl.util.freezeby.FreezeByCountOperator; @@ -281,10 +280,9 @@ public AggregationContext makeAggregationContext( switch (type) { case NORMAL: final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + QueryCompilerRequestProcessor.batch(); final AggregationContext aggContext = new NormalConverter( - table, requireStateChangeRecorder, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), - compilationProcessor, groupByColumnNames).build(); + table, requireStateChangeRecorder, compilationProcessor, groupByColumnNames).build(); compilationProcessor.compile(); return aggContext; case ROLLUP_BASE: @@ -668,17 +666,14 @@ final void addWeightedAvgOrSumOperator( * {@link AggregationContext} for standard aggregations. Accumulates state by visiting each aggregation. */ private final class NormalConverter extends Converter { - private final Supplier> queryScopeVariables; private final QueryCompilerRequestProcessor compilationProcessor; private NormalConverter( @NotNull final Table table, final boolean requireStateChangeRecorder, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor, @NotNull final String... groupByColumnNames) { super(table, requireStateChangeRecorder, groupByColumnNames); - this.queryScopeVariables = queryScopeVariables; this.compilationProcessor = compilationProcessor; } @@ -754,7 +749,7 @@ public void visit(@NotNull final AggSpecFormula formula) { final GroupByChunkedOperator groupByChunkedOperator = new GroupByChunkedOperator(table, false, null, resultPairs.stream().map(pair -> MatchPair.of((Pair) pair.input())).toArray(MatchPair[]::new)); final FormulaChunkedOperator formulaChunkedOperator = new FormulaChunkedOperator(groupByChunkedOperator, - true, formula.formula(), formula.paramToken(), queryScopeVariables, compilationProcessor, + true, formula.formula(), formula.paramToken(), compilationProcessor, MatchPair.fromPairs(resultPairs)); addNoInputOperator(formulaChunkedOperator); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java index d076711b5b9..bdf69ed3a97 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java @@ -30,7 +30,6 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; -import java.util.function.Supplier; import java.util.function.UnaryOperator; import static io.deephaven.engine.table.impl.sources.ArrayBackedColumnSource.BLOCK_SIZE; @@ -76,7 +75,6 @@ class FormulaChunkedOperator implements IterativeChunkedAggregationOperator { final boolean delegateToBy, @NotNull final String formula, @NotNull final String columnParamName, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor, @NotNull final MatchPair... resultColumnPairs) { this.groupBy = groupBy; @@ -101,7 +99,7 @@ class FormulaChunkedOperator implements IterativeChunkedAggregationOperator { .fromGenericType(inputColumnName, inputColumnSource.getType(), inputColumnSource.getComponentType()); formulaColumn.initDef(Collections.singletonMap(inputColumnName, inputColumnDefinition), - queryScopeVariables, compilationProcessor); + compilationProcessor); resultColumns[ci] = ArrayBackedColumnSource.getMemoryColumnSource( 0, formulaColumn.getReturnedType(), formulaColumn.getReturnedComponentType()); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java index 61a0b855cfc..8602715b72c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java @@ -283,7 +283,7 @@ private Stream absoluteSelectColumns() { // have no need to drop them post-sort. final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + QueryCompilerRequestProcessor.batch(); final SelectColumn[] columns = sortColumns.stream() .map(sc -> sc.column().name()) @@ -293,7 +293,7 @@ private Stream absoluteSelectColumns() { final Selectable selectable = AbsoluteSortColumnConventions.makeSelectable(cn, baseColumnName); final SelectColumn selectColumn = SelectColumn.of(selectable); selectColumn.initDef(Map.of(baseColumnName, getDefinition().getColumn(baseColumnName)), - SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), compilationProcessor); + compilationProcessor); return selectColumn; }).toArray(SelectColumn[]::new); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java index ea0814d525d..26a77957a08 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java @@ -26,7 +26,6 @@ import io.deephaven.engine.table.impl.by.AggregationRowLookup; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.WhereFilter; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; import io.deephaven.engine.table.impl.util.RowRedirection; import io.deephaven.util.type.TypeUtils; @@ -37,7 +36,6 @@ import java.util.*; import java.util.function.Function; import java.util.function.LongUnaryOperator; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -278,11 +276,9 @@ public static WhereFilter[] initializeAndValidateFilters( @NotNull final Collection filters, @NotNull final Function exceptionFactory) { final WhereFilter[] whereFilters = WhereFilter.from(filters); - final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (final WhereFilter whereFilter : whereFilters) { - whereFilter.init(source.getDefinition(), variableSupplier, compilationProcessor); + whereFilter.init(source.getDefinition(), compilationProcessor); final List invalidColumnsUsed = whereFilter.getColumns().stream().map(ColumnName::of) .filter(cn -> !groupByColumns.contains(cn)).map(ColumnName::name).collect(Collectors.toList()); if (!invalidColumnsUsed.isEmpty()) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java index d28a469286b..a40d60d69c5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java @@ -12,14 +12,11 @@ import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.WhereFilter; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.util.Collection; import java.util.List; -import java.util.Map; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -140,13 +137,11 @@ public Table where(Filter filter) { } private Stream whereFilters() { - final Supplier> variableSupplier = - SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + QueryCompilerRequestProcessor.batch(); final WhereFilter[] filters = WhereFilter.fromInternal(filter); for (final WhereFilter filter : filters) { - filter.init(getDefinition(), variableSupplier, compilationProcessor); + filter.init(getDefinition(), compilationProcessor); } compilationProcessor.compile(); return Stream.of(filters); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java index 779765c5c8e..53f5eaadb85 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java @@ -19,7 +19,6 @@ import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.chunkboxer.ChunkBoxer; import io.deephaven.engine.table.impl.select.WhereFilter; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.ArrayBackedColumnSource; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.util.SafeCloseable; @@ -32,7 +31,6 @@ import java.util.*; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Stream; /** @@ -153,11 +151,9 @@ private TreeTableFilter(@NotNull final TreeTableImpl tree, @NotNull final WhereF parentIdColumnName = tree.getParentIdentifierColumn(); sourceRowLookup = tree.getSourceRowLookup(); this.filters = filters; - final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); - Arrays.stream(filters).forEach((final WhereFilter filter) -> filter.init(source.getDefinition(), - variableSupplier, compilationProcessor)); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); + Arrays.stream(filters) + .forEach((final WhereFilter filter) -> filter.init(source.getDefinition(), compilationProcessor)); compilationProcessor.compile(); idSource = source.getColumnSource(tree.getIdentifierColumn().name()); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java index ce5d62cb06f..680b20c07a6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java @@ -16,7 +16,6 @@ import io.deephaven.engine.table.impl.by.AggregationProcessor; import io.deephaven.engine.table.impl.by.AggregationRowLookup; import io.deephaven.engine.table.impl.select.WhereFilter; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.NotNull; @@ -24,7 +23,6 @@ import java.util.*; import java.util.function.LongUnaryOperator; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -108,7 +106,7 @@ private static Table makeNullSingleColumnTable( @NotNull final Table source, @NotNull final ColumnName column, final long size) { - final ColumnDefinition columnDefinition = source.getDefinition().getColumn(column.name()); + final ColumnDefinition columnDefinition = source.getDefinition().getColumn(column.name()); final TableDefinition columnOnlyTableDefinition = TableDefinition.of(columnDefinition); // noinspection resource return new QueryTable(columnOnlyTableDefinition, RowSetFactory.flat(size).toTracking(), @@ -156,11 +154,9 @@ public TreeTable withFilter(@NotNull Filter filter) { if (whereFilters.length == 0) { return noopResult(); } - final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); final Map> nodeSuitabilityToFilters = Stream.of(whereFilters) - .peek(wf -> wf.init(source.getDefinition(), variableSupplier, compilationProcessor)) + .peek(wf -> wf.init(source.getDefinition(), compilationProcessor)) .collect(Collectors.partitioningBy(wf -> { // Node-level filters have only node-filter columns and use no column arrays return wf.getColumns().stream().map(ColumnName::of).allMatch(nodeFilterColumns::contains) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java index 6d854abef0b..22c1fc2e8d5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java @@ -30,7 +30,6 @@ public final Class getReturnedType() { @Override public Class getReturnedComponentType() { - // Table does not have a component type return null; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java index 6111dbf6cea..2dfe17bd3eb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java @@ -11,7 +11,6 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -22,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.function.BinaryOperator; -import java.util.function.Supplier; /** * {@link SelectColumn} implementation to wrap transformer functions for {@link PartitionedTable#partitionedTransform @@ -59,10 +57,7 @@ public List initInputs( } @Override - public List initDef( - @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + public List initDef(@NotNull final Map> columnDefinitionMap) { validateInputColumnDefinition(inputOutputColumnName, columnDefinitionMap); validateInputColumnDefinition(secondInputColumnName, columnDefinitionMap); return getColumns(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java index a12dd7ace2f..449488af308 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java @@ -10,7 +10,6 @@ import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.LongSingleValueSource; @@ -21,7 +20,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Supplier; /** * {@link SelectColumn} implementation to assign a constant {@code long} value. @@ -46,10 +44,7 @@ public List initInputs( } @Override - public List initDef( - @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + public List initDef(@NotNull final Map> columnDefinitionMap) { return getColumns(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java index 591447a0cd6..e93342b475a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java @@ -23,7 +23,6 @@ import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.engine.table.impl.select.MatchFilter; import io.deephaven.engine.table.impl.select.WhereFilter; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; import io.deephaven.engine.table.impl.sources.UnionSourceManager; import io.deephaven.engine.table.iterators.ChunkedObjectColumnIterator; @@ -40,7 +39,6 @@ import java.lang.ref.WeakReference; import java.util.*; import java.util.function.BinaryOperator; -import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -234,11 +232,9 @@ private Map computeSharedAttributes(@NotNull final Iterator filters) { final WhereFilter[] whereFilters = WhereFilter.from(filters); - final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); final boolean invalidFilter = Arrays.stream(whereFilters).flatMap((final WhereFilter filter) -> { - filter.init(table.getDefinition(), variableSupplier, compilationProcessor); + filter.init(table.getDefinition(), compilationProcessor); return Stream.concat(filter.getColumns().stream(), filter.getColumnArrays().stream()); }).anyMatch((final String columnName) -> columnName.equals(constituentColumnName)); compilationProcessor.compile(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java index dbf38fa2359..848c0e179b4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java @@ -32,7 +32,6 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BinaryOperator; -import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -465,11 +464,9 @@ public PartitionedTable.Proxy sort(Collection columnsToSortBy) { public PartitionedTable.Proxy where(Filter filter) { final WhereFilter[] whereFilters = WhereFilter.fromInternal(filter); final TableDefinition definition = target.constituentDefinition(); - final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition, variableSupplier, compilationProcessor); + whereFilter.init(definition, compilationProcessor); } compilationProcessor.compile(); return basicTransform(ct -> ct.where(Filter.and(WhereFilter.copyFrom(whereFilters)))); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java index fa3c7c24c32..cf796fe5852 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java @@ -12,7 +12,6 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -22,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.function.Function; -import java.util.function.Supplier; /** * {@link SelectColumn} implementation to wrap transformer functions for {@link PartitionedTable#transform @@ -54,10 +52,7 @@ public List initInputs( } @Override - public List initDef( - @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + public List initDef(@NotNull final Map> columnDefinitionMap) { validateInputColumnDefinition(inputOutputColumnName, columnDefinitionMap); return getColumns(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java index f73f3e21c68..686e4ab7130 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java @@ -17,16 +17,13 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.table.impl.select.WhereFilter; import io.deephaven.engine.table.impl.select.WhereFilterImpl; import org.jetbrains.annotations.NotNull; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_DOUBLE; import static io.deephaven.util.QueryConstants.NULL_FLOAT; @@ -53,10 +50,7 @@ public List getColumnArrays() { } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeSupplier, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { final ColumnDefinition columnDefinition = tableDefinition.getColumn(columnName.name()); if (columnDefinition == null) { throw new IllegalArgumentException(String.format("Missing expected input column %s", diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java index 0cd05704618..bdd46c1fb29 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java @@ -26,7 +26,6 @@ import java.lang.reflect.InvocationTargetException; import java.net.MalformedURLException; import java.util.*; -import java.util.function.Supplier; import java.util.stream.Collectors; import static io.deephaven.engine.table.impl.select.DhFormulaColumn.COLUMN_SUFFIX; @@ -74,10 +73,14 @@ public List getColumnArrays() { .collect(Collectors.toList()); } + @Override + public void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, QueryCompilerRequestProcessor.immediate()); + } + @Override public synchronized void init( @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (initialized) { return; @@ -89,7 +92,7 @@ public synchronized void init( final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( timeConversionResult, tableDefinition.getColumnNameMap(), outerToInnerNames, - queryScopeVariables.get(), unboxArguments); + compilationProcessor.getQueryScopeVariables(), unboxArguments); formulaShiftColPair = result.getFormulaShiftColPair(); if (formulaShiftColPair != null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java index 318e99ef80b..03fd59fdd77 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java @@ -10,6 +10,7 @@ import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.table.impl.vector.*; @@ -46,7 +47,7 @@ public abstract class AbstractFormulaColumn implements FormulaColumn { @NotNull protected final String columnName; - protected Future formulaFactory; + protected Future formulaFactoryFuture; private Formula formula; protected QueryScopeParam[] params; protected Map> columnSources; @@ -94,9 +95,7 @@ public List initInputs( return usedColumns; } - // we'll have to assume that initDef has already been invoked if we could have grouped compilation requests - // otherwise this call will compile immediately if necessary - return initDef(extractDefinitions(columnsOfInterest)); + return initDef(extractDefinitions(columnsOfInterest), QueryCompilerRequestProcessor.immediate()); } @Override @@ -153,7 +152,7 @@ protected void applyUsedVariables( } protected void onCopy(final AbstractFormulaColumn copy) { - copy.formulaFactory = formulaFactory; + copy.formulaFactoryFuture = formulaFactoryFuture; copy.columnDefinitions = columnDefinitions; copy.params = params; copy.usedColumns = usedColumns; @@ -239,12 +238,12 @@ private Formula getFormula(boolean initLazyMap, QueryScopeParam... params) { try { // the future must already be completed or else it is an error - formula = formulaFactory.get(0, TimeUnit.SECONDS).createFormula( + formula = formulaFactoryFuture.get(0, TimeUnit.SECONDS).createFormula( StringEscapeUtils.escapeJava(columnName), rowSet, initLazyMap, columnsToData, params); - } catch (TimeoutException e) { + } catch (InterruptedException | TimeoutException e) { throw new IllegalStateException("Formula factory not already compiled!"); - } catch (InterruptedException | ExecutionException e) { - throw new UncheckedDeephavenException("Error creating formula for " + columnName, e); + } catch (ExecutionException e) { + throw new UncheckedDeephavenException("Error creating formula for " + columnName, e.getCause()); } return formula; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java index 2e38b00e0d2..887f5a221fa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java @@ -8,7 +8,6 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import io.deephaven.util.QueryConstants; @@ -17,8 +16,6 @@ import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Supplier; /** * Base class for filters that will release more rows of a table on each UGP cycle. @@ -70,10 +67,7 @@ public List getColumnArrays() { } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { initialized = true; if (!started) { return; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java index 7ad699fe1f1..d506eee8bc5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java @@ -12,7 +12,6 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ByteRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -21,9 +20,6 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; -import java.util.Map; -import java.util.function.Supplier; - public class ByteRangeFilter extends AbstractRangeFilter { public static ByteRangeFilter lt(String columnName, byte x) { return new ByteRangeFilter(columnName, QueryConstants.NULL_BYTE, x, true, false); @@ -71,10 +67,7 @@ static WhereFilter makeByteRangeFilter(String columnName, Condition condition, S } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java index 06df9a3bc66..f5d640d13f5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java @@ -8,7 +8,6 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.CharRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -17,9 +16,6 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; -import java.util.Map; -import java.util.function.Supplier; - public class CharRangeFilter extends AbstractRangeFilter { public static CharRangeFilter lt(String columnName, char x) { return new CharRangeFilter(columnName, QueryConstants.NULL_CHAR, x, true, false); @@ -67,10 +63,7 @@ static WhereFilter makeCharRangeFilter(String columnName, Condition condition, S } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java index f1751d83be7..71a38223370 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java @@ -11,7 +11,6 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageFunctionUtils; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.updategraph.DynamicNode; @@ -25,8 +24,6 @@ import java.time.Instant; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Supplier; /** * Boilerplate super-class for various clock-oriented filters. @@ -49,10 +46,7 @@ public ClockFilter(@NotNull final String columnName, @NotNull final Clock clock, } @Override - public final void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} + public final void init(@NotNull final TableDefinition tableDefinition) {} @Override public final List getColumns() { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java index 1f76242f348..3050b8d13a3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java @@ -6,7 +6,6 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.util.compare.ObjectComparisons; import io.deephaven.engine.table.ColumnSource; @@ -18,9 +17,6 @@ import io.deephaven.util.annotations.TestUseOnly; import org.jetbrains.annotations.NotNull; -import java.util.Map; -import java.util.function.Supplier; - public class ComparableRangeFilter extends AbstractRangeFilter { private final Comparable upper; private final Comparable lower; @@ -45,10 +41,7 @@ public static ComparableRangeFilter makeForTest(String columnName, Comparable } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java index cde86f837a6..b5a21c52357 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java @@ -12,7 +12,6 @@ import org.jetbrains.annotations.NotNull; import java.util.*; -import java.util.function.Supplier; import java.util.stream.Stream; public abstract class ComposedFilter extends WhereFilterLivenessArtifactImpl implements DependencyStreamProvider { @@ -58,13 +57,19 @@ public List getColumnArrays() { return new ArrayList<>(result); } + @Override + public void init(@NotNull TableDefinition tableDefinition) { + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); + init(tableDefinition, compilationProcessor); + compilationProcessor.compile(); + } + @Override public void init( @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor) { for (WhereFilter filter : componentFilters) { - filter.init(tableDefinition, queryScopeVariables, compilationProcessor); + filter.init(tableDefinition, compilationProcessor); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java index 6bad2b7a70d..fc065f6b052 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java @@ -49,7 +49,7 @@ public class ConditionFilter extends AbstractConditionFilter { public static final int CHUNK_SIZE = 4096; - private Future> filterKernelClass = null; + private Future> filterKernelClassFuture = null; private List>> usedInputs; // that is columns and special variables private String classBody; private Filter filter = null; @@ -70,7 +70,7 @@ public static WhereFilter createConditionFilter(@NotNull String formula, Formula case Numba: throw new UnsupportedOperationException("Python condition filter should be created from python"); default: - throw new UnsupportedOperationException("Unknow parser type " + parser); + throw new UnsupportedOperationException("Unknown parser type " + parser); } } @@ -416,7 +416,7 @@ protected void generateFilterCode( this.classBody = classBody.toString(); - filterKernelClass = compilationProcessor.submit(QueryCompilerRequest.builder() + filterKernelClassFuture = compilationProcessor.submit(QueryCompilerRequest.builder() .description("Filter Expression: " + formula) .className("GeneratedFilterKernel") .classBody(this.classBody) @@ -430,7 +430,7 @@ private StringBuilder getClassBody( @NotNull final TableDefinition tableDefinition, @NotNull final TimeLiteralReplacedExpression timeConversionResult, @NotNull final QueryLanguageParser.Result result) { - if (filterKernelClass != null) { + if (filterKernelClassFuture != null) { return null; } usedInputs = new ArrayList<>(); @@ -590,7 +590,7 @@ protected Filter getFilter(Table table, RowSet fullSet) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { if (filter == null) { try { - final FilterKernel filterKernel = (FilterKernel) filterKernelClass + final FilterKernel filterKernel = (FilterKernel) filterKernelClassFuture .get(0, TimeUnit.SECONDS) .getConstructor(Table.class, RowSet.class, QueryScopeParam[].class) .newInstance(table, fullSet, (Object) params); @@ -601,10 +601,10 @@ protected Filter getFilter(Table table, RowSet fullSet) // note this filter is not valid for use in other contexts, as it captures references from the source // table filterValidForCopy = false; - } catch (TimeoutException e) { + } catch (InterruptedException | TimeoutException e) { throw new IllegalStateException("Formula factory not already compiled!"); - } catch (InterruptedException | ExecutionException e) { - throw new FormulaCompilationException("Formula compilation error for: " + formula, e); + } catch (ExecutionException e) { + throw new FormulaCompilationException("Formula compilation error for: " + formula, e.getCause()); } } return filter; @@ -620,7 +620,7 @@ public ConditionFilter copy() { final ConditionFilter copy = new ConditionFilter(formula, outerToInnerNames); onCopy(copy); if (initialized) { - copy.filterKernelClass = filterKernelClass; + copy.filterKernelClassFuture = filterKernelClassFuture; copy.usedInputs = usedInputs; copy.classBody = classBody; if (filterValidForCopy) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index ec5200ab9dc..9326466fe90 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -48,7 +48,6 @@ import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.Function; -import java.util.function.Supplier; import static io.deephaven.engine.util.IterableUtils.makeCommaSeparatedList; @@ -180,12 +179,16 @@ public static Class getVectorType(Class declaredType) { } } + @Override + public List initDef(@NotNull final Map> columnDefinitionMap) { + return initDef(columnDefinitionMap, QueryCompilerRequestProcessor.immediate()); + } + @Override public List initDef( @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { - if (formulaFactory != null) { + if (formulaFactoryFuture != null) { validateColumnDefinition(columnDefinitionMap); return formulaColumnPython != null ? formulaColumnPython.usedColumns : usedColumns; } @@ -194,7 +197,8 @@ public List initDef( final TimeLiteralReplacedExpression timeConversionResult = TimeLiteralReplacedExpression.convertExpression(formulaString); final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( - timeConversionResult, columnDefinitionMap, Collections.emptyMap(), queryScopeVariables.get(), true); + timeConversionResult, columnDefinitionMap, Collections.emptyMap(), + compilationRequestProcessor.getQueryScopeVariables(), true); analyzedFormula = FormulaAnalyzer.analyze(formulaString, columnDefinitionMap, timeConversionResult, result); hasConstantValue = result.isConstantValueExpression(); @@ -211,13 +215,13 @@ public List initDef( formulaString = result.getConvertedExpression(); // check if this is a column to be created with a Python vectorizable function - checkAndInitializeVectorization(columnDefinitionMap, queryScopeVariables, compilationRequestProcessor); + checkAndInitializeVectorization(columnDefinitionMap, compilationRequestProcessor); } catch (Exception e) { throw new FormulaCompilationException("Formula compilation error for: " + originalFormulaString, e); } if (useKernelFormulasProperty) { - formulaFactory = createKernelFormulaFactory(getFormulaKernelFactory(compilationRequestProcessor)); + formulaFactoryFuture = createKernelFormulaFactory(getFormulaKernelFactory(compilationRequestProcessor)); } else { compileFormula(compilationRequestProcessor); } @@ -226,7 +230,6 @@ public List initDef( private void checkAndInitializeVectorization( @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection SuspiciousToArrayCall final PyCallableWrapperJpyImpl[] cws = Arrays.stream(params) @@ -248,7 +251,7 @@ private void checkAndInitializeVectorization( pyCallableWrapper.getReturnType(), this.analyzedFormula.sourceDescriptor.sources, argumentsChunked, true)); - formulaColumnPython.initDef(columnDefinitionMap, queryScopeVariables, compilationRequestProcessor); + formulaColumnPython.initDef(columnDefinitionMap, compilationRequestProcessor); } } @@ -750,7 +753,7 @@ private CompletionStageFuture invokeKernelBuilder( @NotNull String generateKernelClassBody() { try { - return invokeKernelBuilder(QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE).get().classBody; + return invokeKernelBuilder(QueryCompilerRequestProcessor.immediate()).get().classBody; } catch (InterruptedException | ExecutionException e) { throw new UncheckedDeephavenException("Failed to compile formula: ", e); } @@ -759,7 +762,7 @@ String generateKernelClassBody() { @Override public SelectColumn copy() { final DhFormulaColumn copy = new DhFormulaColumn(columnName, formulaString); - if (formulaFactory != null) { + if (formulaFactoryFuture != null) { copy.analyzedFormula = analyzedFormula; copy.hasConstantValue = hasConstantValue; copy.returnedType = returnedType; @@ -807,7 +810,7 @@ private void compileFormula(@NotNull final QueryCompilerRequestProcessor compila return null; }); - formulaFactory = compilationRequestProcessor.submit(QueryCompilerRequest.builder() + formulaFactoryFuture = compilationRequestProcessor.submit(QueryCompilerRequest.builder() .description("Formula Expression: " + formulaString) .className(className) .classBody(classBody) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java index c69b7100947..1b195e01b0b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java @@ -10,7 +10,6 @@ import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.compare.DoubleComparisons; import io.deephaven.engine.table.impl.chunkfilter.DoubleRangeComparator; import io.deephaven.engine.table.ColumnSource; @@ -19,9 +18,6 @@ import io.deephaven.util.QueryConstants; import org.jetbrains.annotations.NotNull; -import java.util.Map; -import java.util.function.Supplier; - public class DoubleRangeFilter extends AbstractRangeFilter { public static DoubleRangeFilter lt(String columnName, double x) { @@ -83,10 +79,7 @@ static WhereFilter makeDoubleRangeFilter(String columnName, Condition condition, } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java index 2a1ed5bc5b6..d917756a289 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java @@ -9,7 +9,6 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.time.DateTimeUtils; import io.deephaven.engine.updategraph.DynamicNode; import io.deephaven.engine.table.ColumnSource; @@ -18,8 +17,6 @@ import java.time.Instant; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Supplier; /** * Utilities for downsampling non-ticking time series data within a query. The input table must be sorted by the @@ -88,10 +85,7 @@ public List getColumnArrays() { } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java index 7ea956c163b..affa6bb2afd 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java @@ -26,7 +26,6 @@ import org.jetbrains.annotations.NotNull; import java.util.*; -import java.util.function.Supplier; /** * A where filter that extracts a set of inclusion or exclusion keys from a set table. @@ -222,10 +221,7 @@ public List getColumnArrays() { } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java index def411b61c8..2cdebf0bde7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java @@ -6,7 +6,6 @@ import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.compare.FloatComparisons; import io.deephaven.engine.table.impl.chunkfilter.FloatRangeComparator; import io.deephaven.engine.table.ColumnSource; @@ -15,9 +14,6 @@ import io.deephaven.util.QueryConstants; import org.jetbrains.annotations.NotNull; -import java.util.Map; -import java.util.function.Supplier; - public class FloatRangeFilter extends AbstractRangeFilter { public static FloatRangeFilter lt(String columnName, float x) { @@ -79,10 +75,7 @@ static WhereFilter makeFloatRangeFilter(String columnName, Condition condition, } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java index 0ddbb47682b..5854fa12440 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java @@ -8,7 +8,6 @@ import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.NoSuchColumnException; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.engine.table.impl.sources.SparseArrayColumnSource; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -26,7 +25,6 @@ import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.function.Supplier; public class FunctionalColumn implements SelectColumn { @@ -108,10 +106,7 @@ public List initInputs(TrackingRowSet rowSet, Map initDef( - @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + public List initDef(@NotNull final Map> columnDefinitionMap) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java index 3328ccab9b6..7d45254db75 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java @@ -6,7 +6,6 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; import io.deephaven.engine.table.ColumnSource; @@ -19,8 +18,6 @@ import org.jetbrains.annotations.NotNull; import java.time.Instant; -import java.util.Map; -import java.util.function.Supplier; public class InstantRangeFilter extends LongRangeFilter { @@ -40,10 +37,7 @@ public InstantRangeFilter( } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java index 3e787851bb0..ee4a345e2a2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java @@ -12,7 +12,6 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.IntRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -21,9 +20,6 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; -import java.util.Map; -import java.util.function.Supplier; - public class IntRangeFilter extends AbstractRangeFilter { public static IntRangeFilter lt(String columnName, int x) { return new IntRangeFilter(columnName, QueryConstants.NULL_INT, x, true, false); @@ -71,10 +67,7 @@ static WhereFilter makeIntRangeFilter(String columnName, Condition condition, St } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java index 9d4fad06cbe..d0718eab953 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java @@ -12,7 +12,6 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.LongRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -21,9 +20,6 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; -import java.util.Map; -import java.util.function.Supplier; - public class LongRangeFilter extends AbstractRangeFilter { public static LongRangeFilter lt(String columnName, long x) { return new LongRangeFilter(columnName, QueryConstants.NULL_LONG, x, true, false); @@ -71,10 +67,7 @@ static WhereFilter makeLongRangeFilter(String columnName, Condition condition, S } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 3ee5b9ee254..aed73c2752f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -5,14 +5,12 @@ import io.deephaven.api.literal.Literal; import io.deephaven.base.string.cache.CompressedString; -import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.preview.DisplayWrapper; -import io.deephaven.engine.context.QueryScope; import io.deephaven.time.DateTimeUtils; import io.deephaven.util.type.ArrayTypeUtils; import io.deephaven.engine.table.ColumnSource; @@ -24,7 +22,6 @@ import java.math.BigInteger; import java.time.Instant; import java.util.*; -import java.util.function.Supplier; public class MatchFilter extends WhereFilterImpl { @@ -117,10 +114,14 @@ public List getColumnArrays() { return Collections.emptyList(); } + @Override + public void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, QueryCompilerRequestProcessor.immediate()); + } + @Override public synchronized void init( @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariablesSupplier, @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (initialized) { return; @@ -135,7 +136,7 @@ public synchronized void init( return; } final List valueList = new ArrayList<>(); - final Map queryScopeVariables = queryScopeVariablesSupplier.get(); + final Map queryScopeVariables = compilationProcessor.getQueryScopeVariables(); final ColumnTypeConvertor convertor = ColumnTypeConvertorFactory.getConvertor(column.getDataType(), column.getName()); for (String strValue : strValues) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java index 8e5c1342a40..9898430313b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java @@ -9,7 +9,6 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.NoSuchColumnException; import io.deephaven.engine.table.impl.PrevColumnSource; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.engine.table.impl.sources.SparseArrayColumnSource; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -24,7 +23,6 @@ import java.util.*; import java.util.function.BiFunction; -import java.util.function.Supplier; import java.util.stream.Collectors; // TODO: Comment the heck out of this... @@ -95,10 +93,7 @@ public List initInputs(TrackingRowSet rowSet, Map initDef( - @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + public List initDef(@NotNull final Map> columnDefinitionMap) { NoSuchColumnException.throwIf(columnDefinitionMap.keySet(), sourceNames); return getColumns(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java index 7ed11fe9d1f..c897dcb7daf 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java @@ -5,7 +5,6 @@ import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.*; import io.deephaven.engine.rowset.TrackingRowSet; import org.jetbrains.annotations.NotNull; @@ -13,7 +12,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Supplier; /** * A SelectColumn implementation that can be used to replace columns with {@link NullValueColumnSource}s @@ -34,10 +32,7 @@ public List initInputs(final TrackingRowSet rowSet, } @Override - public List initDef( - @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + public List initDef(@NotNull final Map> columnDefinitionMap) { return Collections.emptyList(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java index b6c37e32dbb..f7cdfaa570a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java @@ -19,8 +19,6 @@ import java.math.BigInteger; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Supplier; /** * A filter for comparable types (including Instant) for {@link Condition} values:
@@ -131,22 +129,26 @@ public List getColumnArrays() { return Collections.emptyList(); } + @Override + public void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, QueryCompilerRequestProcessor.immediate()); + } + @Override public void init( @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (filter != null) { return; } - final ColumnDefinition def = tableDefinition.getColumn(columnName); + final ColumnDefinition def = tableDefinition.getColumn(columnName); if (def == null) { throw new RuntimeException("Column \"" + columnName + "\" doesn't exist in this table, available columns: " + tableDefinition.getColumnNames()); } - final Class colClass = def.getDataType(); + final Class colClass = def.getDataType(); if (colClass == double.class || colClass == Double.class) { filter = DoubleRangeFilter.makeDoubleRangeFilter(columnName, condition, value); @@ -185,7 +187,7 @@ public void init( } } - filter.init(tableDefinition, queryScopeVariables, compilationProcessor); + filter.init(tableDefinition, compilationProcessor); } public static char parseCharFilter(String value) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java index 968e0287c9d..3e04c8ee0bb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java @@ -13,7 +13,6 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.rowset.TrackingRowSet; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.ConvertibleTimeSource; import io.deephaven.engine.table.impl.sources.LocalDateWrapperSource; import io.deephaven.engine.table.impl.sources.LocalTimeWrapperSource; @@ -35,7 +34,6 @@ import java.util.List; import java.util.Map; import java.util.function.Function; -import java.util.function.Supplier; /** * Allows {@link ColumnSource} reinterpretation via view-type ({@link Table#view} and {@link Table#updateView}) @@ -153,10 +151,7 @@ public List initInputs(TrackingRowSet rowSet, Map initDef( - @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + public List initDef(@NotNull final Map> columnDefinitionMap) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java index 66975516e66..6459c65c704 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java @@ -9,15 +9,12 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import org.jetbrains.annotations.NotNull; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Supplier; /** * This will filter a table starting off with the first N rows, and then adding new rows to the table on each run. @@ -48,10 +45,7 @@ public List getColumnArrays() { } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java index 9728f556c1d..a725ebdcc5b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java @@ -21,8 +21,6 @@ import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; -import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; import java.util.Arrays; @@ -30,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -86,18 +83,13 @@ static Collection copyFrom(Collection selectColumns) * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. */ - @FinalDefault - default List initDef(@NotNull Map> columnDefinitionMap) { - return initDef(columnDefinitionMap, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), - QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE); - } + List initDef(@NotNull Map> columnDefinitionMap); /** * Initialize any internal column definitions from the provided initial. A compilation request consumer is provided * to allow for deferred compilation of expressions that belong to the same query. * * @param columnDefinitionMap the starting set of column definitions; valid for this call only - * @param queryScopeVariables a caching supplier of the set of query scope variables; valid for this call only * @param compilationRequestProcessor a consumer to submit compilation requests; valid for this call only * * @return a list of columns on which the result of this is dependent @@ -105,10 +97,12 @@ default List initDef(@NotNull Map> columnDef * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. */ - List initDef( - @NotNull Map> columnDefinitionMap, - @NotNull Supplier> queryScopeVariables, - @NotNull QueryCompilerRequestProcessor compilationRequestProcessor); + @SuppressWarnings("unused") + default List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + return initDef(columnDefinitionMap); + } /** * Get the data type stored in the resultant column. diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java index cedccd871c1..461707e7b7f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java @@ -12,7 +12,6 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ShortRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -21,9 +20,6 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; -import java.util.Map; -import java.util.function.Supplier; - public class ShortRangeFilter extends AbstractRangeFilter { public static ShortRangeFilter lt(String columnName, short x) { return new ShortRangeFilter(columnName, QueryConstants.NULL_SHORT, x, true, false); @@ -71,10 +67,7 @@ static WhereFilter makeShortRangeFilter(String columnName, Condition condition, } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java index 6801ff9de4d..d9a54a7758b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java @@ -6,7 +6,6 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.util.compare.ObjectComparisons; import io.deephaven.engine.table.ColumnSource; @@ -18,9 +17,6 @@ import io.deephaven.util.annotations.TestUseOnly; import org.jetbrains.annotations.NotNull; -import java.util.Map; -import java.util.function.Supplier; - public class SingleSidedComparableRangeFilter extends AbstractRangeFilter { private final Comparable pivot; private final boolean isGreaterThan; @@ -38,10 +34,7 @@ public static SingleSidedComparableRangeFilter makeForTest(String columnName, Co } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java index 0a0449c7fca..fcaa5d30bd2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java @@ -7,7 +7,6 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.NoSuchColumnException; @@ -20,7 +19,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Supplier; public class SourceColumn implements SelectColumn { @@ -63,10 +61,7 @@ public List initInputs(TrackingRowSet rowSet, Map initDef( - @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + public List initDef(@NotNull final Map> columnDefinitionMap) { sourceDefinition = columnDefinitionMap.get(sourceName); if (sourceDefinition == null) { throw new NoSuchColumnException(columnDefinitionMap.keySet(), sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java index 48caddc72a8..d380d98ed36 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java @@ -16,7 +16,6 @@ import java.util.List; import java.util.Map; -import java.util.function.Supplier; public class SwitchColumn implements SelectColumn { @@ -46,10 +45,14 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(@NotNull Map> columnDefinitionMap) { + return initDef(columnDefinitionMap, QueryCompilerRequestProcessor.immediate()); + } + @Override public List initDef( @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (realColumn == null) { if (columnDefinitionMap.get(expression) != null) { @@ -58,8 +61,7 @@ public List initDef( realColumn = FormulaColumn.createFormulaColumn(columnName, expression, parser); } } - final List usedColumns = realColumn.initDef( - columnDefinitionMap, queryScopeVariables, compilationRequestProcessor); + final List usedColumns = realColumn.initDef(columnDefinitionMap, compilationRequestProcessor); if (realColumn instanceof DhFormulaColumn) { FormulaColumnPython formulaColumnPython = ((DhFormulaColumn) realColumn).getFormulaColumnPython(); realColumn = formulaColumnPython != null ? formulaColumnPython : realColumn; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java index 2b6bf85eb8f..497957f4e45 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java @@ -12,7 +12,6 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import io.deephaven.time.DateTimeUtils; @@ -22,8 +21,6 @@ import java.time.Instant; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Supplier; /** * This will filter a table for the most recent N nanoseconds (must be on an {@link Instant} column). @@ -62,10 +59,7 @@ public List getColumnArrays() { } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java index c493d194d50..80cbcf295f2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java @@ -14,7 +14,6 @@ import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.util.annotations.FinalDefault; import io.deephaven.util.annotations.InternalUseOnly; import org.jetbrains.annotations.NotNull; @@ -22,8 +21,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Map; -import java.util.function.Supplier; /** * Interface for individual filters within a where clause. @@ -107,25 +104,22 @@ interface RecomputeListener { * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or * {@link QueryCompiler} usage needs to be resolved within init. Implementations must be idempotent. */ - @FinalDefault - default void init(@NotNull TableDefinition tableDefinition) { - init(tableDefinition, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), - QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE); - } + void init(@NotNull TableDefinition tableDefinition); /** * Initialize this select filter given the table definition * * @param tableDefinition the definition of the table that will be filtered - * @param queryScopeVariables a caching supplier of the set of query scope variables; valid for this call only * @param compilationProcessor the processor to use for compilation * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or * {@link QueryCompiler} usage needs to be resolved within init. Implementations must be idempotent. */ - void init( - @NotNull TableDefinition tableDefinition, - @NotNull Supplier> queryScopeVariables, - @NotNull QueryCompilerRequestProcessor compilationProcessor); + @SuppressWarnings("unused") + default void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + init(tableDefinition); + } /** * Validate that this {@code WhereFilter} is safe to use in the context of the provided sourceTable. diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java index e051f16e299..9792774fac1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java @@ -4,7 +4,6 @@ package io.deephaven.engine.table.impl.select; import io.deephaven.engine.liveness.LivenessArtifact; -import io.deephaven.engine.liveness.LivenessReferent; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.Table; @@ -17,9 +16,7 @@ import org.jetbrains.annotations.NotNull; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.function.Supplier; import java.util.stream.Stream; class WhereFilterInvertedImpl @@ -59,12 +56,16 @@ public List getColumnArrays() { return filter.getColumnArrays(); } + @Override + public void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, QueryCompilerRequestProcessor.immediate()); + } + @Override public void init( @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor) { - filter.init(tableDefinition, queryScopeVariables, compilationProcessor); + filter.init(tableDefinition, compilationProcessor); } @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java index f66094e9083..de19278857d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java @@ -16,15 +16,12 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter.ObjectChunkFilter; import org.jetbrains.annotations.NotNull; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.function.Supplier; final class WhereFilterPatternImpl extends WhereFilterImpl { @@ -46,10 +43,7 @@ private WhereFilterPatternImpl(FilterPattern filterPattern) { } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + public void init(@NotNull final TableDefinition tableDefinition) { final String columnName = columnName(); final ColumnDefinition column = tableDefinition.getColumn(columnName); if (column == null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java index 9b1317ee747..7a8a686ba9e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java @@ -8,13 +8,10 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import org.jetbrains.annotations.NotNull; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Supplier; /** * A Select filter that always returns an empty RowSet. @@ -36,10 +33,7 @@ public List getColumnArrays() { } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java index dc997685e2b..783a92fe62b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java @@ -3,12 +3,9 @@ // package io.deephaven.engine.table.impl.select.analyzers; -import io.deephaven.api.util.NameValidator; import io.deephaven.base.Pair; import io.deephaven.base.log.LogOutputAppendable; import io.deephaven.datastructures.util.CollectionUtil; -import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.liveness.LivenessNode; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; @@ -32,14 +29,12 @@ import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.util.SafeCloseable; import io.deephaven.util.SafeCloseablePair; -import io.deephaven.util.datastructures.CachingSupplier; import io.deephaven.vector.Vector; import org.jetbrains.annotations.Nullable; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; -import java.util.function.Supplier; import java.util.stream.Stream; public abstract class SelectAndViewAnalyzer implements LogOutputAppendable { @@ -50,29 +45,21 @@ public enum Mode { VIEW_LAZY, VIEW_EAGER, SELECT_STATIC, SELECT_REFRESHING, SELECT_REDIRECTED_REFRESHING, SELECT_REDIRECTED_STATIC } - public static Supplier> newQueryScopeVariableSupplier() { - final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); - return new CachingSupplier<>(() -> Collections.unmodifiableMap( - queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name)))); - } - public static void initializeSelectColumns( final Map> parentColumnMap, final SelectColumn[] selectColumns) { - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); - initializeSelectColumns(parentColumnMap, newQueryScopeVariableSupplier(), selectColumns, compilationProcessor); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); + initializeSelectColumns(parentColumnMap, selectColumns, QueryCompilerRequestProcessor.batch()); compilationProcessor.compile(); } public static void initializeSelectColumns( final Map> parentColumnMap, - final Supplier> queryScopeVariables, final SelectColumn[] selectColumns, final QueryCompilerRequestProcessor compilationProcessor) { final Map> targetColumnMap = new HashMap<>(parentColumnMap); for (SelectColumn column : selectColumns) { - column.initDef(targetColumnMap, queryScopeVariables, compilationProcessor); + column.initDef(targetColumnMap, compilationProcessor); final ColumnDefinition columnDefinition = ColumnDefinition.fromGenericType( column.getName(), column.getReturnedType(), column.getReturnedComponentType()); targetColumnMap.put(column.getName(), columnDefinition); @@ -120,9 +107,7 @@ public static SelectAndViewAnalyzerWrapper create( final HashSet resultColumns = new HashSet<>(); // First pass to initialize all columns and to compile formulas in one batch. - final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (Map.Entry> entry : columnSources.entrySet()) { final String name = entry.getKey(); final ColumnSource cs = entry.getValue(); @@ -136,7 +121,7 @@ public static SelectAndViewAnalyzerWrapper create( continue; } - sc.initDef(columnDefinitions, variableSupplier, compilationProcessor); + sc.initDef(columnDefinitions, compilationProcessor); final ColumnDefinition cd = ColumnDefinition.fromGenericType( sc.getName(), sc.getReturnedType(), sc.getReturnedComponentType()); columnDefinitions.put(sc.getName(), cd); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java index e60d7db35db..298abcf1702 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java @@ -5,7 +5,6 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.context.QueryScopeParam; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.table.impl.select.AbstractFormulaColumn; @@ -19,7 +18,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Supplier; import static io.deephaven.datastructures.util.CollectionUtil.ZERO_LENGTH_STRING_ARRAY; @@ -42,16 +40,13 @@ private FormulaColumnPython(String columnName, } @Override - public final List initDef( - @NotNull final Map> columnDefinitionMap, - @NotNull final Supplier> queryScopeVariables, - @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { - if (formulaFactory != null) { + public final List initDef(@NotNull final Map> columnDefinitionMap) { + if (formulaFactoryFuture != null) { validateColumnDefinition(columnDefinitionMap); } else { returnedType = dcf.getReturnedType(); applyUsedVariables(columnDefinitionMap, new LinkedHashSet<>(dcf.getColumnNames()), Map.of()); - formulaFactory = createKernelFormulaFactory(CompletionStageFuture.completedFuture(this)); + formulaFactoryFuture = createKernelFormulaFactory(CompletionStageFuture.completedFuture(this)); } return usedColumns; @@ -75,7 +70,7 @@ protected final FormulaSourceDescriptor getSourceDescriptor() { @Override public final SelectColumn copy() { final FormulaColumnPython copy = new FormulaColumnPython(columnName, dcf); - if (formulaFactory != null) { + if (formulaFactoryFuture != null) { // copy all initDef state copy.returnedType = returnedType; onCopy(copy); @@ -85,7 +80,7 @@ public final SelectColumn copy() { @Override public final FormulaKernel createInstance(Vector[] arrays, QueryScopeParam[] params) { - if (formulaFactory == null) { + if (formulaFactoryFuture == null) { throw new IllegalStateException("Must be initialized first"); } return dcf.toFormulaKernel(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java index 8a4ecb8bf35..6e551476520 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java @@ -14,7 +14,6 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.updateby.delta.*; import io.deephaven.engine.table.impl.updateby.em.*; import io.deephaven.engine.table.impl.updateby.emstd.*; @@ -40,7 +39,6 @@ import java.math.MathContext; import java.time.Instant; import java.util.*; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -88,8 +86,7 @@ final Collection getOutputColumns(@NotNull final Collection getOperators(@NotNull final Collection specs) { - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - new QueryCompilerRequestProcessor.BatchProcessor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); final OperationVisitor v = new OperationVisitor(compilationProcessor); specs.forEach(s -> s.walk(v)); @@ -290,7 +287,6 @@ public Void visit(@NotNull final ColumnUpdateOperation clause) { } private class OperationVisitor implements UpdateBySpec.Visitor, UpdateByOperation.Visitor { - private final Supplier> queryScopeVariables; private final QueryCompilerRequestProcessor compilationProcessor; private final List ops = new ArrayList<>(); private MatchPair[] pairs; @@ -301,7 +297,6 @@ private class OperationVisitor implements UpdateBySpec.Visitor, UpdateByOp OperationVisitor( @NotNull final QueryCompilerRequestProcessor compilationProcessor) { - this.queryScopeVariables = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); this.compilationProcessor = compilationProcessor; } @@ -1380,47 +1375,47 @@ private UpdateByOperator makeRollingFormulaOperator(@NotNull final MatchPair pai return new BooleanRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == byte.class || csType == Byte.class) { return new ByteRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == char.class || csType == Character.class) { return new CharRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == short.class || csType == Short.class) { return new ShortRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == int.class || csType == Integer.class) { return new IntRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == long.class || csType == Long.class) { return new LongRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == float.class || csType == Float.class) { return new FloatRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + formulaColumnMap, tableDef, compilationProcessor); } else if (csType == double.class || csType == Double.class) { return new DoubleRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + formulaColumnMap, tableDef, compilationProcessor); } return new ObjectRollingFormulaOperator<>(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + formulaColumnMap, tableDef, compilationProcessor); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java index 9f04380105a..b5e0a9560a9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; -import java.util.function.Supplier; abstract class BaseRollingFormulaOperator extends UpdateByOperator { protected final String PARAM_COLUMN_NAME = "__PARAM_COLUMN__"; @@ -98,7 +97,6 @@ public BaseRollingFormulaOperator( @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, true); this.formulaColumnMap = formulaColumnMap; @@ -117,8 +115,7 @@ public BaseRollingFormulaOperator( final ColumnDefinition inputColumnDefinition = ColumnDefinition .fromGenericType(PARAM_COLUMN_NAME, inputVectorType, inputColumnType); - tmp.initDef(Collections.singletonMap(PARAM_COLUMN_NAME, inputColumnDefinition), queryScopeVariables, - compilationProcessor); + tmp.initDef(Collections.singletonMap(PARAM_COLUMN_NAME, inputColumnDefinition), compilationProcessor); return tmp; }); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java index 4dbe63b8ae2..b54d4dcc02d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java @@ -29,7 +29,6 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; -import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -163,10 +162,9 @@ public BooleanRollingFormulaOperator( @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + paramToken, formulaColumnMap, tableDef, compilationProcessor); } protected BooleanRollingFormulaOperator( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java index bcb5da64243..4cb47097afa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java @@ -33,7 +33,6 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; -import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -166,13 +165,12 @@ public ByteRollingFormulaOperator( @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor - // region extra-constructor-args - // endregion extra-constructor-args + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java index 86a490caf42..7018c431482 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java @@ -29,7 +29,6 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; -import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -162,13 +161,12 @@ public CharRollingFormulaOperator( @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java index 161b9842cff..bb0c449e394 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java @@ -33,7 +33,6 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; -import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -166,13 +165,12 @@ public DoubleRollingFormulaOperator( @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor - // region extra-constructor-args - // endregion extra-constructor-args + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java index f994066ec0d..66a65898978 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java @@ -33,7 +33,6 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; -import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -166,13 +165,12 @@ public FloatRollingFormulaOperator( @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor - // region extra-constructor-args - // endregion extra-constructor-args + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java index 2e160f87ef4..c0b07b52af9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java @@ -33,7 +33,6 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; -import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -166,13 +165,12 @@ public IntRollingFormulaOperator( @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor - // region extra-constructor-args - // endregion extra-constructor-args + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java index 15f0f13aac8..9b3eb63de7b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java @@ -33,7 +33,6 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; -import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -166,13 +165,12 @@ public LongRollingFormulaOperator( @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor - // region extra-constructor-args - // endregion extra-constructor-args + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java index 932f1eba0b8..8b062025e52 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; -import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -162,10 +161,9 @@ public ObjectRollingFormulaOperator( @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + paramToken, formulaColumnMap, tableDef, compilationProcessor); } protected ObjectRollingFormulaOperator( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java index 7fcb68ed3d1..7cce65c7cb5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java @@ -33,7 +33,6 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; -import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -166,13 +165,12 @@ public ShortRollingFormulaOperator( @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, @NotNull final TableDefinition tableDef, - @NotNull final Supplier> queryScopeVariables, @NotNull final QueryCompilerRequestProcessor compilationProcessor - // region extra-constructor-args - // endregion extra-constructor-args + // region extra-constructor-args + // endregion extra-constructor-args ) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, - paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); + paramToken, formulaColumnMap, tableDef, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java index da14d6b0499..8a615214e71 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java @@ -6,7 +6,6 @@ import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.QueryCompilerRequest; -import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import java.util.*; import java.util.function.Supplier; diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java index cc927144403..65b27a6b380 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java @@ -21,8 +21,6 @@ import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Supplier; import static org.junit.Assert.assertEquals; @@ -59,10 +57,7 @@ public List getColumnArrays() { } @Override - public void init( - @NotNull final TableDefinition tableDefinition, - @NotNull final Supplier> queryScopeSupplier, - @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} + public void init(@NotNull final TableDefinition tableDefinition) {} @NotNull @Override diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java index 4f7597807fe..34d2db264b3 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java @@ -4,15 +4,14 @@ package io.deephaven.engine.table.impl.lang; import groovy.lang.Closure; -import io.deephaven.api.util.NameValidator; import io.deephaven.base.Pair; import io.deephaven.base.testing.BaseArrayTestCase; import io.deephaven.base.verify.Assert; import io.deephaven.base.verify.Require; import io.deephaven.engine.context.*; import io.deephaven.engine.table.Table; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser.QueryLanguageParseException; -import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.testutil.ControlledUpdateGraph; import io.deephaven.engine.util.PyCallableWrapper; import io.deephaven.util.QueryConstants; @@ -3179,7 +3178,7 @@ private void check(String expression, String resultExpression, Class resultTy final Map possibleParams; final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); if (!(queryScope instanceof PoisonedQueryScope)) { - possibleParams = SelectAndViewAnalyzer.newQueryScopeVariableSupplier().get(); + possibleParams = QueryCompilerRequestProcessor.newQueryScopeVariableSupplier().get(); } else { possibleParams = null; } diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java index e553331daa7..d32a0ea4883 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ColumnChunkReaderImpl.java @@ -9,7 +9,7 @@ import io.deephaven.parquet.compress.CompressorAdapter; import io.deephaven.parquet.compress.DeephavenCompressorAdapterFactory; import io.deephaven.util.channel.SeekableChannelContext.ContextHolder; -import io.deephaven.util.datastructures.LazyCachingFunction; +import io.deephaven.util.datastructures.SoftCachingFunction; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Dictionary; @@ -77,7 +77,7 @@ final class ColumnChunkReaderImpl implements ColumnChunkReader { } this.offsetIndex = offsetIndex; this.fieldTypes = fieldTypes; - this.dictionarySupplier = new LazyCachingFunction<>(this::getDictionary); + this.dictionarySupplier = new SoftCachingFunction<>(this::getDictionary); this.nullMaterializerFactory = PageMaterializer.factoryForType(path.getPrimitiveType().getPrimitiveTypeName()); this.numRows = numRows; this.version = version; diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ChunkDictionary.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ChunkDictionary.java index 56bccb4e4a0..17e83139c31 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ChunkDictionary.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/pagestore/topage/ChunkDictionary.java @@ -9,7 +9,7 @@ import io.deephaven.util.channel.SeekableChannelContext; import io.deephaven.stringset.LongBitmapStringSet; import io.deephaven.chunk.ObjectChunk; -import io.deephaven.util.datastructures.LazyCachingSupplier; +import io.deephaven.util.datastructures.SoftCachingSupplier; import org.apache.parquet.column.Dictionary; import org.jetbrains.annotations.NotNull; @@ -47,7 +47,7 @@ public interface Lookup { ChunkDictionary( @NotNull final Lookup lookup, @NotNull final Function dictionarySupplier) { - this.valuesSupplier = new LazyCachingSupplier<>(() -> { + this.valuesSupplier = new SoftCachingSupplier<>(() -> { // We use NULL channel context here and rely on materialization logic to provide the correct context final Dictionary dictionary = dictionarySupplier.apply(SeekableChannelContext.NULL); final T[] values = ObjectChunk.makeArray(dictionary.getMaxId() + 1); @@ -56,7 +56,7 @@ public interface Lookup { } return ObjectChunk.chunkWrap(values); }); - this.reverseMapSupplier = new LazyCachingSupplier<>(() -> { + this.reverseMapSupplier = new SoftCachingSupplier<>(() -> { final ObjectChunk values = getChunk(); final TObjectIntMap reverseMap = new TObjectIntHashMap<>(values.size()); for (int vi = 0; vi < values.size(); ++vi) { diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateCachingSupplier.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateCachingSupplier.java index ae28bf10bc5..a102cb09e85 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateCachingSupplier.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateCachingSupplier.java @@ -8,9 +8,9 @@ import static io.deephaven.replication.ReplicatePrimitiveCode.replaceAll; public class ReplicateCachingSupplier { - private static final String LAZY_CACHING_SUPPLIER_DIR = "Util/src/main/java/io/deephaven/util/datastructures/"; - private static final String LAZY_CACHING_SUPPLIER_PATH = LAZY_CACHING_SUPPLIER_DIR + "LazyCachingSupplier.java"; - private static final String LAZY_CACHING_FUNCTION_PATH = LAZY_CACHING_SUPPLIER_DIR + "LazyCachingFunction.java"; + private static final String SOFT_CACHING_SUPPLIER_DIR = "Util/src/main/java/io/deephaven/util/datastructures/"; + private static final String SOFT_CACHING_SUPPLIER_PATH = SOFT_CACHING_SUPPLIER_DIR + "SoftCachingSupplier.java"; + private static final String SOFT_CACHING_FUNCTION_PATH = SOFT_CACHING_SUPPLIER_DIR + "SoftCachingFunction.java"; private static final String[] NO_EXCEPTIONS = new String[0]; @@ -22,7 +22,7 @@ public static void main(final String[] args) throws IOException { {"Supplier", "Function"}, {"supplier", "function"}, }; - replaceAll("replicateCachingSupplier", LAZY_CACHING_SUPPLIER_PATH, LAZY_CACHING_FUNCTION_PATH, null, + replaceAll("replicateCachingSupplier", SOFT_CACHING_SUPPLIER_PATH, SOFT_CACHING_FUNCTION_PATH, null, NO_EXCEPTIONS, pairs); } } From 9527dec62f00718248cd7164d0c2ee04a567a91e Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 21 Mar 2024 17:46:42 -0600 Subject: [PATCH 07/19] Fix Merge Conflicts --- .../table/impl/PartitionAwareSourceTable.java | 37 +--------------- .../engine/table/impl/QueryTable.java | 44 +++++++------------ .../engine/table/impl/RedefinableTable.java | 7 +-- .../table/impl/WouldMatchOperation.java | 25 +++-------- .../table/impl/select/ComposedFilter.java | 3 -- .../impl/select/FunctionalColumnLong.java | 7 ++- .../engine/table/impl/select/MatchFilter.java | 9 ---- 7 files changed, 31 insertions(+), 101 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java index 023f44f35e2..2b151f0ccd6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java @@ -141,18 +141,8 @@ public Table selectDistinctInternal(Collection columns) { final Set newColumns = new HashSet<>(); for (final SelectColumn selectColumn : selectColumns) { -<<<<<<< HEAD if (!((PartitionAwareSourceTable) table).isValidAgainstColumnPartitionTable( selectColumn.getColumns(), selectColumn.getColumnArrays(), newColumns)) { -======= - try { - selectColumn.initDef(getDefinition().getColumnNameMap()); - } catch (Exception e) { - return null; - } - if (!((PartitionAwareSourceTable) table).isValidAgainstColumnPartitionTable( - selectColumn.getColumns(), selectColumn.getColumnArrays())) { ->>>>>>> upstream/main return null; } newColumns.add(selectColumn.getName()); @@ -268,22 +258,13 @@ private Table whereImpl(final WhereFilter[] whereFilters) { return prepareReturnThis(); } -<<<<<<< HEAD final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); - for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition, compilationProcessor); - List columns = whereFilter.getColumns(); - if (whereFilter instanceof ReindexingFilter) { - otherFilters.add(whereFilter); - } else if (isValidAgainstColumnPartitionTable(columns, whereFilter.getColumnArrays())) { -======= final List partitionFilters = new ArrayList<>(); final List deferredFilters = new ArrayList<>(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition); + whereFilter.init(definition, compilationProcessor); if (!(whereFilter instanceof ReindexingFilter) && isValidAgainstColumnPartitionTable(whereFilter.getColumns(), whereFilter.getColumnArrays())) { ->>>>>>> upstream/main partitionFilters.add(whereFilter); } else { deferredFilters.add(whereFilter); @@ -306,19 +287,13 @@ && isValidAgainstColumnPartitionTable(whereFilter.getColumns(), whereFilter.getC @Override public final Table selectDistinct(@NotNull final Collection columns) { final List selectColumns = Arrays.asList(SelectColumn.from(columns)); -<<<<<<< HEAD SelectAndViewAnalyzer.initializeSelectColumns( definition.getColumnNameMap(), selectColumns.toArray(SelectColumn[]::new)); final Set newColumns = new HashSet<>(); - for (SelectColumn selectColumn : selectColumns) { + for (final SelectColumn selectColumn : selectColumns) { if (!isValidAgainstColumnPartitionTable( selectColumn.getColumns(), selectColumn.getColumnArrays(), newColumns)) { -======= - for (final SelectColumn selectColumn : selectColumns) { - selectColumn.initDef(definition.getColumnNameMap()); - if (!isValidAgainstColumnPartitionTable(selectColumn.getColumns(), selectColumn.getColumnArrays())) { ->>>>>>> upstream/main // Be sure to invoke the super-class version of this method, rather than the array-based one that // delegates to this method. return super.selectDistinct(selectColumns); @@ -336,7 +311,6 @@ public final Table selectDistinct(@NotNull final Collection columnNames, @NotNull final Collection columnArrayNames) { -<<<<<<< HEAD return isValidAgainstColumnPartitionTable(columnNames, columnArrayNames, Collections.emptySet()); } @@ -350,11 +324,4 @@ private boolean isValidAgainstColumnPartitionTable( return columnNames.stream().allMatch( columnName -> partitioningColumnDefinitions.containsKey(columnName) || newColumns.contains(columnName)); } -======= - if (!columnArrayNames.isEmpty()) { - return false; - } - return columnNames.stream().allMatch(partitioningColumnDefinitions::containsKey); - } ->>>>>>> upstream/main } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java index 4802d0e4a48..694dc81f981 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java @@ -1167,13 +1167,18 @@ private void initializeAndPrioritizeFilters(@NotNull final WhereFilter... filter final DataIndexer dataIndexer = DataIndexer.existingOf(rowSet); final int numFilters = filters.length; final BitSet priorityFilterIndexes = new BitSet(numFilters); + + final QueryCompilerRequestProcessor.BatchProcessor compilationProcesor = QueryCompilerRequestProcessor.batch(); + // Initialize our filters immediately so we can examine the columns they use. Note that filter + // initialization is safe to invoke repeatedly. + for (final WhereFilter filter : filters) { + filter.init(getDefinition(), compilationProcesor); + } + compilationProcesor.compile(); + for (int fi = 0; fi < numFilters; ++fi) { final WhereFilter filter = filters[fi]; - // Initialize our filters immediately so we can examine the columns they use. Note that filter - // initialization is safe to invoke repeatedly. - filter.init(getDefinition()); - // Simple filters against indexed columns get priority if (dataIndexer != null && !(filter instanceof ReindexingFilter) @@ -1246,37 +1251,20 @@ private QueryTable whereInternal(final WhereFilter... filters) { return result; } -<<<<<<< HEAD - List selectFilters = new LinkedList<>(); - List>>> shiftColPairs = new LinkedList<>(); - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - QueryCompilerRequestProcessor.batch(); + final List whereFilters = new LinkedList<>(); + final List>>> shiftColPairs = + new LinkedList<>(); for (final WhereFilter filter : filters) { - filter.init(getDefinition(), compilationProcessor); if (filter instanceof AbstractConditionFilter && ((AbstractConditionFilter) filter).hasConstantArrayAccess()) { shiftColPairs.add(((AbstractConditionFilter) filter).getFormulaShiftColPair()); } else { - selectFilters.add(filter); -======= - { - final List whereFilters = new LinkedList<>(); - final List>>> shiftColPairs = - new LinkedList<>(); - for (final WhereFilter filter : filters) { - if (filter instanceof AbstractConditionFilter - && ((AbstractConditionFilter) filter).hasConstantArrayAccess()) { - shiftColPairs.add(((AbstractConditionFilter) filter).getFormulaShiftColPair()); - } else { - whereFilters.add(filter); - } - } - if (!shiftColPairs.isEmpty()) { - return (QueryTable) ShiftedColumnsFactory.where(this, shiftColPairs, whereFilters); ->>>>>>> upstream/main + whereFilters.add(filter); } } - compilationProcessor.compile(); + if (!shiftColPairs.isEmpty()) { + return (QueryTable) ShiftedColumnsFactory.where(this, shiftColPairs, whereFilters); + } return memoizeResult(MemoizedOperationKey.filter(filters), () -> { try (final SafeCloseable ignored = Arrays.stream(filters) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java index 12470103c22..fe479e6ea17 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java @@ -52,11 +52,8 @@ private Table viewInternal(Collection selectables, boolean final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); for (final SelectColumn selectColumn : columns) { -<<<<<<< HEAD - List usedColumnNames = selectColumn.initDef(allColumns, compilationProcessor); -======= - final List usedColumnNames = new ArrayList<>(selectColumn.initDef(allColumns)); ->>>>>>> upstream/main + final List usedColumnNames = new ArrayList<>( + selectColumn.initDef(allColumns, compilationProcessor)); usedColumnNames.addAll(selectColumn.getColumnArrays()); resultColumnsInternal.addAll(usedColumnNames.stream() .filter(usedColumnName -> !resultColumnsExternal.containsKey(usedColumnName)) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java index 81b907ea696..05ef6d8acd2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java @@ -45,11 +45,6 @@ public class WouldMatchOperation implements QueryTable.MemoizableOperation>>>>>> upstream/main private final WouldMatchPair wouldMatchPair; private final WhereFilter filter; @@ -98,9 +93,12 @@ public String getLogPrefix() { @Override public SafeCloseable beginOperation(@NotNull final QueryTable parent) { + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); + Arrays.stream(whereFilters).forEach(filter -> filter.init(parent.getDefinition(), compilationProcessor)); + compilationProcessor.compile(); + return Arrays.stream(whereFilters) .map((final WhereFilter filter) -> { - filter.init(parent.getDefinition()); // Ensure we gather the correct dependencies when building a snapshot control. return filter.beginOperation(parent); }).collect(SafeCloseableList.COLLECTOR); @@ -122,22 +120,9 @@ public Result initialize(boolean usePrev, long beforeClock) { try (final SafeCloseableList closer = new SafeCloseableList()) { final RowSet fullRowSet = usePrev ? closer.add(parent.getRowSet().copyPrev()) : parent.getRowSet(); final Map> newColumns = new LinkedHashMap<>(parent.getColumnSourceMap()); -<<<<<<< HEAD - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - QueryCompilerRequestProcessor.batch(); - final WhereFilter[] filters = matchColumns.stream().map(ColumnHolder::getFilter) - .peek(holder -> holder.init(parent.getDefinition(), compilationProcessor)) - .toArray(WhereFilter[]::new); - compilationProcessor.compile(); - - for (int ii = 0; ii < filters.length; ++ii) { - final ColumnHolder holder = matchColumns.get(ii); - final WhereFilter filter = filters[ii]; -======= matchColumns.forEach(holder -> { final WhereFilter filter = holder.getFilter(); ->>>>>>> upstream/main final WritableRowSet result = filter.filter(fullRowSet, fullRowSet, parent, usePrev); holder.column = new IndexWrapperColumnSource( holder.getColumnName(), parent, result.toTracking(), filter); @@ -151,7 +136,7 @@ public Result initialize(boolean usePrev, long beforeClock) { if (filter.isRefreshing()) { anyRefreshing.setTrue(); } - } + }); resultTable = new QueryTable(parent.getRowSet(), newColumns); transformer = diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java index d0b45af8299..d7cd648006d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java @@ -5,11 +5,8 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; -<<<<<<< HEAD import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; -======= import io.deephaven.engine.table.impl.BaseTable; ->>>>>>> upstream/main import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.liveness.LivenessArtifact; import io.deephaven.engine.table.impl.DependencyStreamProvider; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumnLong.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumnLong.java index ca72e42b00c..071858b3f92 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumnLong.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumnLong.java @@ -87,7 +87,7 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef(@NotNull final Map> columnDefinitionMap) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { @@ -106,6 +106,11 @@ public Class getReturnedType() { return long.class; } + @Override + public Class getReturnedComponentType() { + return null; + } + @Override public List getColumns() { return List.of(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 89018ffdc51..0ad27701356 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -5,28 +5,19 @@ import io.deephaven.api.literal.Literal; import io.deephaven.base.string.cache.CompressedString; -<<<<<<< HEAD -======= -import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.liveness.LivenessScopeStack; import io.deephaven.engine.rowset.RowSet; ->>>>>>> upstream/main import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.DataIndex; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; -<<<<<<< HEAD import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.preview.DisplayWrapper; -======= import io.deephaven.engine.table.impl.DependencyStreamProvider; import io.deephaven.engine.table.impl.indexer.DataIndexer; -import io.deephaven.engine.table.impl.preview.DisplayWrapper; import io.deephaven.engine.updategraph.NotificationQueue; ->>>>>>> upstream/main import io.deephaven.time.DateTimeUtils; import io.deephaven.util.SafeCloseable; import io.deephaven.util.type.ArrayTypeUtils; From c921f83197c914aeff8f67c125b058239cf927fa Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 22 Mar 2024 10:51:06 -0600 Subject: [PATCH 08/19] BugFix; don't make two CompilationProcessors --- .../table/impl/select/analyzers/SelectAndViewAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java index 783a92fe62b..78b7239fb7e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java @@ -49,7 +49,7 @@ public static void initializeSelectColumns( final Map> parentColumnMap, final SelectColumn[] selectColumns) { final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch(); - initializeSelectColumns(parentColumnMap, selectColumns, QueryCompilerRequestProcessor.batch()); + initializeSelectColumns(parentColumnMap, selectColumns, compilationProcessor); compilationProcessor.compile(); } From c9c85ee876dca2b27460306966b31ee3a76c4b7c Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 28 Mar 2024 10:04:38 -0600 Subject: [PATCH 09/19] Remaining Feedback from Ryan --- .../deephaven/engine/context/QueryCompiler.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 1152569987c..cf54d4f3924 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -5,6 +5,7 @@ import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.FileUtils; +import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; import io.deephaven.configuration.DataDir; import io.deephaven.datastructures.util.CollectionUtil; @@ -246,7 +247,7 @@ public void compile( /** * Compiles all requests. * - * @param requests The compilation requests + * @param requests The compilation requests; these must be independent of each other * @param resolvers The resolvers to use for delivering compilation results */ public void compile( @@ -304,6 +305,11 @@ public void compile( resolvers[ii].complete(allFutures[ii].get()); } catch (ExecutionException err) { resolvers[ii].completeExceptionally(err.getCause()); + } catch (InterruptedException err) { + // This can only occur if we are interrupted while waiting for the future to complete from another + // compilation request. + Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures[ii], "allFutures[ii]"); + resolvers[ii].completeExceptionally(err); } catch (Throwable err) { resolvers[ii].completeExceptionally(err); } @@ -503,7 +509,7 @@ private void compileHelper( continue; } - while (true) { + next_probe: while (true) { final int pi = state.nextProbeIndex++; final String packageNameSuffix = "c_" + basicHashText[ii] + (pi == 0 ? "" : ("p" + pi)) @@ -523,6 +529,13 @@ private void compileHelper( state.packageName = request.getPackageName(packageNameSuffix); state.fqClassName = state.packageName + "." + request.className(); + for (int jj = 0; jj < ii; ++jj) { + if (states[jj].fqClassName.equals(state.fqClassName)) { + // collision within batch + continue next_probe; + } + } + // Ask the classloader to load an existing class with this name. This might: // 1. Fail to find a class (returning null) // 2. Find a class whose body has the formula we are looking for From f76fdf2d8c93a23b7ab1d1fd06cff45807b1d19b Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 16 Apr 2024 14:28:51 -0600 Subject: [PATCH 10/19] FB from Ryan on naming from QueryCompiler.java review --- .../deephaven/util/CompletionStageFuture.java | 4 +-- .../engine/context/QueryCompiler.java | 30 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java index 0564f9dbf01..ed24c99ba4d 100644 --- a/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java @@ -17,14 +17,14 @@ /** * A safe version of CompletableFuture that does not expose the completion API. * - * @param The result type returned by this future's {@code join} + * @param The result type returned by this future's {@link #get()} */ public interface CompletionStageFuture extends Future, CompletionStage { /** * Create a new incomplete future. * - * @param The result type returned by this future's {@code join} + * @param The result type returned by this future's {@link #get()} * @return a resolver for the future */ static Resolver make() { diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index cf54d4f3924..7262efca2f4 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -856,7 +856,7 @@ private void maybeCreateClasses( int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); if (parallelismFactor == 1 || requestsPerTask >= requests.size()) { - maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + doCreateClasses(compiler, fileManager, requests, rootPathAsString, tempDirAsString, 0, requests.size()); } else { int numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; @@ -865,7 +865,7 @@ private void maybeCreateClasses( final int startInclusive = jobId * requestsPerTask; final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); tasks[jobId] = operationInitializer.submit(() -> { - maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + doCreateClasses(compiler, fileManager, requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive); }); } @@ -898,7 +898,7 @@ private void maybeCreateClasses( } } - private void maybeCreateClassHelper( + private void doCreateClasses( @NotNull final JavaCompiler compiler, @NotNull final JavaFileManager fileManager, @NotNull final List requests, @@ -910,22 +910,22 @@ private void maybeCreateClassHelper( // If any of our requests fail to compile then the JavaCompiler will not write any class files at all. The // non-failing requests will be retried in a second pass that is expected to succeed. This enables us to // fulfill futures independent of each other; otherwise a single failure would taint all requests in a batch. - final boolean wantRetry = maybeCreateClassHelper2(compiler, - fileManager, requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive, toRetry); + final boolean wantRetry = doCreateClassesSingleRound(compiler, fileManager, requests, rootPathAsString, + tempDirAsString, startInclusive, endExclusive, toRetry); if (!wantRetry) { return; } final List ignored = new ArrayList<>(); - if (maybeCreateClassHelper2(compiler, - fileManager, toRetry, rootPathAsString, tempDirAsString, 0, toRetry.size(), ignored)) { + if (doCreateClassesSingleRound(compiler, fileManager, toRetry, rootPathAsString, tempDirAsString, 0, + toRetry.size(), ignored)) { // We only retried compilation units that did not fail on the first pass, so we should not have any failures // on the second pass. throw new IllegalStateException("Unexpected failure during second pass of compilation"); } } - private boolean maybeCreateClassHelper2( + private boolean doCreateClassesSingleRound( @NotNull final JavaCompiler compiler, @NotNull final JavaFileManager fileManager, @NotNull final List requests, @@ -984,14 +984,12 @@ private boolean maybeCreateClassHelper2( // exceptionally. final boolean hasException = request.resolver.getFuture().isDone(); - if (wantRetry && !Files.exists(srcDir)) { - // The move failed and the source directory does not exist. - if (!hasException) { - // This source actually succeeded in compiling, but was not written because some other source - // failed to compile. Let's schedule this work to try again. - toRetry.add(request); - return; - } + if (wantRetry && !Files.exists(srcDir) && !hasException) { + // The move failed, the source directory does not exist, and this compilation unit actually + // succeeded. However, it was not written because some other compilation unit failed. Let's schedule + // this work to try again. + toRetry.add(request); + return; } if (!Files.exists(destDir) && !hasException) { From a21a24c49fd20f0b90dba26109619af891ad1b81 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Wed, 17 Apr 2024 13:06:47 -0600 Subject: [PATCH 11/19] Introduce QueryCompiler as Interface --- .../engine/context/PoisonedQueryCompiler.java | 6 +- .../engine/context/QueryCompiler.java | 1046 +---------------- .../engine/context/TestQueryCompiler.java | 3 +- .../engine/context/QueryCompilerImpl.java | 1041 ++++++++++++++++ .../table/impl/select/ConditionFilter.java | 4 +- .../table/impl/select/DhFormulaColumn.java | 6 +- .../impl/select/QueryScopeParamTypeUtil.java | 4 +- .../select/codegen/JavaKernelBuilder.java | 6 +- .../engine/util/AbstractScriptSession.java | 4 +- .../engine/util/DynamicCompileUtils.java | 6 +- .../engine/util/GroovyDeephavenSession.java | 15 +- .../impl/TestEventDrivenUpdateGraph.java | 3 +- .../engine/context/TestExecutionContext.java | 2 +- .../testcase/RefreshingTableTestCase.java | 6 +- .../parquet/table/TableWriteBenchmark.java | 4 +- 15 files changed, 1099 insertions(+), 1057 deletions(-) create mode 100644 engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java diff --git a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java index 1acec6cac88..fdfb59cda9d 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java @@ -9,7 +9,7 @@ import java.io.File; -public class PoisonedQueryCompiler extends QueryCompiler { +public class PoisonedQueryCompiler implements QueryCompiler { public static final PoisonedQueryCompiler INSTANCE = new PoisonedQueryCompiler(); @@ -20,12 +20,12 @@ private T fail() { } @Override - public File getFakeClassDestination() { + public File getTemporaryClassDestination() { return fail(); } @Override - public void setParentClassLoader(ClassLoader parentClassLoader) { + public void setParentClassLoader(@NotNull final ClassLoader parentClassLoader) { fail(); } diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 7262efca2f4..34f0ffea287 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -4,218 +4,40 @@ package io.deephaven.engine.context; import io.deephaven.UncheckedDeephavenException; -import io.deephaven.base.FileUtils; -import io.deephaven.base.verify.Assert; -import io.deephaven.configuration.Configuration; -import io.deephaven.configuration.DataDir; -import io.deephaven.datastructures.util.CollectionUtil; -import io.deephaven.engine.context.util.SynchronizedJavaFileManager; -import io.deephaven.engine.updategraph.OperationInitializer; -import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.logger.Logger; -import io.deephaven.util.ByteUtils; import io.deephaven.util.CompletionStageFuture; -import org.apache.commons.lang3.mutable.MutableInt; -import org.apache.commons.text.StringEscapeUtils; +import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; -import javax.tools.*; -import java.io.*; -import java.lang.reflect.Field; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.*; -import java.util.concurrent.*; -import java.util.function.Supplier; -import java.util.jar.Attributes; -import java.util.jar.Manifest; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.io.File; +import java.util.concurrent.ExecutionException; -public class QueryCompiler { +public interface QueryCompiler { - private static final Logger log = LoggerFactory.getLogger(QueryCompiler.class); /** - * We pick a number just shy of 65536, leaving a little elbow room for good luck. + * Deephaven REPL Sessions may create new classes on the fly. This method returns the directory where these classes + * should be written so that they are visible to the compiler. + * + * @return The directory where classes should be written */ - private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; - - private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); - private static final int MAX_CLASS_COLLISIONS = 128; - - private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; - - private static final String CODEGEN_TIMEOUT_PROP = "QueryCompiler.codegen.timeoutMs"; - private static final long CODEGEN_TIMEOUT_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); // 10 seconds - private static final String CODEGEN_LOOP_DELAY_PROP = "QueryCompiler.codegen.retry.delay"; - private static final long CODEGEN_LOOP_DELAY_MS_DEFAULT = 100; - private static final long CODEGEN_TIMEOUT_MS = - Configuration.getInstance().getLongWithDefault(CODEGEN_TIMEOUT_PROP, CODEGEN_TIMEOUT_MS_DEFAULT); - private static final long CODEGEN_LOOP_DELAY_MS = - Configuration.getInstance().getLongWithDefault(CODEGEN_LOOP_DELAY_PROP, CODEGEN_LOOP_DELAY_MS_DEFAULT); - - private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); - - public static final String FORMULA_PREFIX = "io.deephaven.temp"; - public static final String DYNAMIC_GROOVY_CLASS_PREFIX = "io.deephaven.dynamic"; - - public static QueryCompiler create(File cacheDirectory, ClassLoader classLoader) { - return new QueryCompiler(cacheDirectory, classLoader, true); - } - - static QueryCompiler createForUnitTests() { - final Path queryCompilerDir = DataDir.get() - .resolve("io.deephaven.engine.context.QueryCompiler.createForUnitTests"); - return new QueryCompiler(queryCompilerDir.toFile()); - } - - private final Map>> knownClasses = new HashMap<>(); - - private final String[] dynamicPatterns = new String[] {DYNAMIC_GROOVY_CLASS_PREFIX, FORMULA_PREFIX}; - - private final File classDestination; - private final boolean isCacheDirectory; - private final Set additionalClassLocations; - private volatile WritableURLClassLoader ucl; - - /** package-private constructor for {@link io.deephaven.engine.context.PoisonedQueryCompiler} */ - QueryCompiler() { - classDestination = null; - isCacheDirectory = false; - additionalClassLocations = null; - } - - private QueryCompiler(File classDestination) { - this(classDestination, null, false); - } - - private QueryCompiler( - final File classDestination, - final ClassLoader parentClassLoader, - final boolean isCacheDirectory) { - final ClassLoader parentClassLoaderToUse = parentClassLoader == null - ? QueryCompiler.class.getClassLoader() - : parentClassLoader; - this.classDestination = classDestination; - this.isCacheDirectory = isCacheDirectory; - ensureDirectories(this.classDestination, () -> "Failed to create missing class destination directory " + - classDestination.getAbsolutePath()); - additionalClassLocations = new LinkedHashSet<>(); - - URL[] urls = new URL[1]; - try { - urls[0] = (classDestination.toURI().toURL()); - } catch (MalformedURLException e) { - throw new UncheckedDeephavenException(e); - } - this.ucl = new WritableURLClassLoader(urls, parentClassLoaderToUse); - - if (isCacheDirectory) { - addClassSource(classDestination); - } - } + @Nullable + File getTemporaryClassDestination(); /** - * Enables or disables compilation logging. + * Set the parent class loader for the compiler. If the Deephaven REPL Session can create new classes on the fly, + * that class loader must provide the on-the-fly class objects used by QueryCompiler compiled classes. * - * @param logEnabled Whether logging should be enabled - * @return The value of {@code logEnabled} before calling this method. - */ - public static boolean setLogEnabled(boolean logEnabled) { - boolean original = QueryCompiler.logEnabled; - QueryCompiler.logEnabled = logEnabled; - return original; - } - - /* - * NB: This is (obviously) not thread safe if code tries to write the same className to the same - * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving - * synchronization as an external concern. - */ - public static void writeClass(final File destinationDirectory, final String className, final byte[] data) - throws IOException { - writeClass(destinationDirectory, className, data, null); - } - - /* - * NB: This is (obviously) not thread safe if code tries to write the same className to the same - * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving - * synchronization as an external concern. + * @param parentClassLoader The parent class loader */ - public static void writeClass(final File destinationDirectory, final String className, final byte[] data, - final String message) throws IOException { - final File destinationFile = new File(destinationDirectory, - className.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - - if (destinationFile.exists()) { - final byte[] existingBytes = Files.readAllBytes(destinationFile.toPath()); - if (Arrays.equals(existingBytes, data)) { - if (message == null) { - log.info().append("Ignoring pushed class ").append(className) - .append(" because it already exists in this context!").endl(); - } else { - log.info().append("Ignoring pushed class ").append(className).append(message) - .append(" because it already exists in this context!").endl(); - } - return; - } else { - if (message == null) { - log.info().append("Pushed class ").append(className) - .append(" already exists in this context, but has changed!").endl(); - } else { - log.info().append("Pushed class ").append(className).append(message) - .append(" already exists in this context, but has changed!").endl(); - } - if (!destinationFile.delete()) { - throw new IOException("Could not delete existing class file: " + destinationFile); - } - } - } - - final File parentDir = destinationFile.getParentFile(); - ensureDirectories(parentDir, - () -> "Unable to create missing destination directory " + parentDir.getAbsolutePath()); - if (!destinationFile.createNewFile()) { - throw new UncheckedDeephavenException( - "Unable to create destination file " + destinationFile.getAbsolutePath()); - } - final ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(data.length); - byteOutStream.write(data, 0, data.length); - final FileOutputStream fileOutStream = new FileOutputStream(destinationFile); - byteOutStream.writeTo(fileOutStream); - fileOutStream.close(); - } - - public File getFakeClassDestination() { - // Groovy classes need to be written out to a location where they can be found by the compiler - // (so that filters and formulae can use them). - // - // We don't want the regular runtime class loader to find them, because then they get "stuck" in there - // even if the class itself changes, and we can't forget it. So instead we use a single-use class loader - // for each formula, that will always read the class from disk. - return isCacheDirectory ? classDestination : null; - } - - public void setParentClassLoader(final ClassLoader parentClassLoader) { - // noinspection NonAtomicOperationOnVolatileField - ucl = new WritableURLClassLoader(ucl.getURLs(), parentClassLoader); - } + void setParentClassLoader(@NotNull ClassLoader parentClassLoader); /** * Compile a class. * * @param request The compilation request */ - public Class compile(@NotNull final QueryCompilerRequest request) { + @FinalDefault + default Class compile(@NotNull final QueryCompilerRequest request) { final CompletionStageFuture.Resolver> resolver = CompletionStageFuture.make(); compile(request, resolver); try { @@ -237,7 +59,8 @@ public Class compile(@NotNull final QueryCompilerRequest request) { * @param request The compilation request * @param resolver The resolver to use for delivering compilation results */ - public void compile( + @FinalDefault + default void compile( @NotNull final QueryCompilerRequest request, @NotNull final CompletionStageFuture.Resolver> resolver) { // noinspection unchecked @@ -250,832 +73,7 @@ public void compile( * @param requests The compilation requests; these must be independent of each other * @param resolvers The resolvers to use for delivering compilation results */ - public void compile( - @NotNull final QueryCompilerRequest[] requests, - @NotNull final CompletionStageFuture.Resolver>[] resolvers) { - if (requests.length == 0) { - return; - } - if (requests.length != resolvers.length) { - throw new IllegalArgumentException("Requests and resolvers must be the same length"); - } - - // noinspection unchecked - final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; - - final List newRequests = new ArrayList<>(); - final List>> newResolvers = new ArrayList<>(); - - synchronized (this) { - for (int ii = 0; ii < requests.length; ++ii) { - final QueryCompilerRequest request = requests[ii]; - final CompletionStageFuture.Resolver> resolver = resolvers[ii]; - - CompletionStageFuture> future = - knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); - if (future == null) { - newRequests.add(request); - newResolvers.add(resolver); - future = resolver.getFuture(); - } - allFutures[ii] = future; - } - } - - if (!newResolvers.isEmpty()) { - // It's my job to fulfill these futures. - try { - compileHelper(newRequests, newResolvers); - } catch (RuntimeException e) { - // These failures are not applicable to a single request, so we can't just complete the future and - // leave the failure in the cache. - synchronized (this) { - for (int ii = 0; ii < newRequests.size(); ++ii) { - if (newResolvers.get(ii).completeExceptionally(e)) { - knownClasses.remove(newRequests.get(ii).classBody()); - } - } - } - throw e; - } - } - - for (int ii = 0; ii < requests.length; ++ii) { - try { - resolvers[ii].complete(allFutures[ii].get()); - } catch (ExecutionException err) { - resolvers[ii].completeExceptionally(err.getCause()); - } catch (InterruptedException err) { - // This can only occur if we are interrupted while waiting for the future to complete from another - // compilation request. - Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures[ii], "allFutures[ii]"); - resolvers[ii].completeExceptionally(err); - } catch (Throwable err) { - resolvers[ii].completeExceptionally(err); - } - } - } - - private static void ensureDirectories(final File file, final Supplier runtimeErrMsg) { - // File.mkdirs() checks for existence on entry, in which case it returns false. - // It may also return false on a failure to create. - // Also note, two separate threads or JVMs may be running this code in parallel. It's possible that we could - // lose the race - // (and therefore mkdirs() would return false), but still get the directory we need (and therefore exists() - // would return true) - if (!file.mkdirs() && !file.isDirectory()) { - throw new UncheckedDeephavenException(runtimeErrMsg.get()); - } - } - - private ClassLoader getClassLoaderForFormula(final Map> parameterClasses) { - return new URLClassLoader(ucl.getURLs(), ucl) { - // Once we find a class that is missing, we should not attempt to load it again, - // otherwise we can end up with a StackOverflow Exception - final HashSet missingClasses = new HashSet<>(); - - @Override - protected Class findClass(String name) throws ClassNotFoundException { - // If we have a parameter that uses this class, return it - final Class paramClass = parameterClasses.get(name); - if (paramClass != null) { - return paramClass; - } - - // Unless we are looking for a formula or Groovy class, we should use the default behavior - if (!isFormulaClass(name)) { - return super.findClass(name); - } - - // if it is a groovy class, always try to use the instance in the shell - if (name.startsWith(DYNAMIC_GROOVY_CLASS_PREFIX)) { - try { - return ucl.getParent().loadClass(name); - } catch (final ClassNotFoundException ignored) { - // we'll try to load it otherwise - } - } - - // We've already not found this class, so we should not try to search again - if (missingClasses.contains(name)) { - return super.findClass(name); - } - - final byte[] bytes; - try { - bytes = loadClassData(name); - } catch (IOException ioe) { - missingClasses.add(name); - return super.loadClass(name); - } - return defineClass(name, bytes, 0, bytes.length); - } - - @SuppressWarnings("BooleanMethodIsAlwaysInverted") - private boolean isFormulaClass(String name) { - return Arrays.stream(dynamicPatterns).anyMatch(name::startsWith); - } - - @Override - public Class loadClass(String name) throws ClassNotFoundException { - if (!isFormulaClass(name)) { - return super.loadClass(name); - } - return findClass(name); - } - - private byte[] loadClassData(String name) throws IOException { - final File destFile = new File(classDestination, - name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - if (destFile.exists()) { - return Files.readAllBytes(destFile.toPath()); - } - - for (File location : additionalClassLocations) { - final File checkFile = new File(location, - name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); - if (checkFile.exists()) { - return Files.readAllBytes(checkFile.toPath()); - } - } - - throw new FileNotFoundException(name); - } - }; - } - - private static class WritableURLClassLoader extends URLClassLoader { - private WritableURLClassLoader(URL[] urls, ClassLoader parent) { - super(urls, parent); - } - - @Override - protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { - Class clazz = findLoadedClass(name); - if (clazz != null) { - return clazz; - } - - try { - clazz = findClass(name); - } catch (ClassNotFoundException e) { - if (getParent() != null) { - clazz = getParent().loadClass(name); - } - } - - if (resolve) { - resolveClass(clazz); - } - return clazz; - } - - @Override - public synchronized void addURL(URL url) { - super.addURL(url); - } - } - - private void addClassSource(File classSourceDirectory) { - synchronized (additionalClassLocations) { - if (additionalClassLocations.contains(classSourceDirectory)) { - return; - } - additionalClassLocations.add(classSourceDirectory); - } - try { - ucl.addURL(classSourceDirectory.toURI().toURL()); - } catch (MalformedURLException e) { - throw new UncheckedDeephavenException(e); - } - } - - private File getClassDestination() { - return classDestination; - } - - private String getClassPath() { - StringBuilder sb = new StringBuilder(); - sb.append(classDestination.getAbsolutePath()); - synchronized (additionalClassLocations) { - for (File classLoc : additionalClassLocations) { - sb.append(File.pathSeparatorChar).append(classLoc.getAbsolutePath()); - } - } - return sb.toString(); - } - - private static class CompilationState { - int nextProbeIndex; - boolean complete; - String packageName; - String fqClassName; - } - - private void compileHelper( - @NotNull final List requests, - @NotNull final List>> resolvers) { - final MessageDigest digest; - try { - digest = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); - } - - final String[] basicHashText = new String[requests.size()]; - for (int ii = 0; ii < requests.size(); ++ii) { - basicHashText[ii] = ByteUtils.byteArrToHex(digest.digest( - requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); - } - - int numComplete = 0; - final CompilationState[] states = new CompilationState[requests.size()]; - for (int ii = 0; ii < requests.size(); ++ii) { - states[ii] = new CompilationState(); - } - - /* - * @formatter:off - * 1. try to resolve without compiling; retain next hash to try - * 2. compile all remaining with a single compilation task - * 3. goto step 1 if any are unresolved - * @formatter:on - */ - - while (numComplete < requests.size()) { - for (int ii = 0; ii < requests.size(); ++ii) { - final CompilationState state = states[ii]; - if (state.complete) { - continue; - } - - next_probe: while (true) { - final int pi = state.nextProbeIndex++; - final String packageNameSuffix = "c_" + basicHashText[ii] - + (pi == 0 ? "" : ("p" + pi)) - + "v" + JAVA_CLASS_VERSION; - - final QueryCompilerRequest request = requests.get(ii); - if (pi >= MAX_CLASS_COLLISIONS) { - Exception err = new IllegalStateException("Found too many collisions for package name root " - + request.packageNameRoot() + ", class name=" + request.className() + ", class body " - + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); - resolvers.get(ii).completeExceptionally(err); - state.complete = true; - ++numComplete; - break; - } - - state.packageName = request.getPackageName(packageNameSuffix); - state.fqClassName = state.packageName + "." + request.className(); - - for (int jj = 0; jj < ii; ++jj) { - if (states[jj].fqClassName.equals(state.fqClassName)) { - // collision within batch - continue next_probe; - } - } - - // Ask the classloader to load an existing class with this name. This might: - // 1. Fail to find a class (returning null) - // 2. Find a class whose body has the formula we are looking for - // 3. Find a class whose body has a different formula (hash collision) - Class result = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); - if (result == null) { - break; // we'll try to compile it - } - - if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolvers.get(ii), - result)) { - state.complete = true; - ++numComplete; - break; - } - } - } - - if (numComplete == requests.size()) { - return; - } - - // Couldn't resolve at least one of the requests, so try a round of compilation. - final List compilationRequestAttempts = new ArrayList<>(); - for (int ii = 0; ii < requests.size(); ++ii) { - final CompilationState state = states[ii]; - if (!state.complete) { - final QueryCompilerRequest request = requests.get(ii); - compilationRequestAttempts.add(new CompilationRequestAttempt( - request, - state.packageName, - state.fqClassName, - resolvers.get(ii))); - } - } - - maybeCreateClasses(compilationRequestAttempts); - - // We could be running on a screwy filesystem that is slow (e.g. NFS). If we wrote a file and can't load it - // ... then give the filesystem some time. All requests should use the same deadline. - final long deadline = System.currentTimeMillis() + CODEGEN_TIMEOUT_MS - CODEGEN_LOOP_DELAY_MS; - for (int ii = 0; ii < requests.size(); ++ii) { - final CompilationState state = states[ii]; - if (state.complete) { - continue; - } - - final QueryCompilerRequest request = requests.get(ii); - final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); - if (resolver.getFuture().isDone()) { - state.complete = true; - ++numComplete; - continue; - } - - // This request may have: - // A. succeeded - // B. Lost a race to another process on the same file system which is compiling the identical formula - // C. Lost a race to another process on the same file system compiling a different formula that collides - - Class clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); - try { - while (clazz == null && System.currentTimeMillis() < deadline) { - // noinspection BusyWait - Thread.sleep(CODEGEN_LOOP_DELAY_MS); - clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); - } - } catch (final InterruptedException ie) { - throw new UncheckedDeephavenException("Interrupted while waiting for codegen", ie); - } - - // However, regardless of A-C, there will be *some* class being found - if (clazz == null) { - throw new IllegalStateException("Should have been able to load *some* class here"); - } - - if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolver, clazz)) { - state.complete = true; - ++numComplete; - } - } - } - } - - private boolean completeIfResultMatchesQueryCompilerRequest( - final String packageName, - final QueryCompilerRequest request, - final CompletionStageFuture.Resolver> resolver, - final Class result) { - final String identifyingFieldValue = loadIdentifyingField(result); - if (!request.classBody().equals(identifyingFieldValue)) { - return false; - } - - // If the caller wants a textual copy of the code we either made, or just found in the cache. - request.codeLog() - .ifPresent(sb -> sb.append(makeFinalCode(request.className(), request.classBody(), packageName))); - - // If the class we found was indeed the class we were looking for, then complete the future and return it. - resolver.complete(result); - - synchronized (this) { - // Note we are doing something kind of subtle here. We are removing an entry whose key was matched - // by value equality and replacing it with a value-equal but reference-different string that is a - // static member of the class we just loaded. This should be easier on the garbage collector because - // we are replacing a calculated value with a classloaded value and so in effect we are - // "canonicalizing" the string. This is important because these long strings stay in knownClasses - // forever. - knownClasses.remove(identifyingFieldValue); - knownClasses.put(identifyingFieldValue, resolver.getFuture()); - } - - return true; - } - - private Class tryLoadClassByFqName(String fqClassName, Map> parameterClasses) { - try { - return getClassLoaderForFormula(parameterClasses).loadClass(fqClassName); - } catch (ClassNotFoundException cnfe) { - return null; - } - } - - private static String loadIdentifyingField(Class c) { - try { - final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); - return (String) field.get(null); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new IllegalStateException("Malformed class in cache", e); - } - } - - private static String makeFinalCode(String className, String classBody, String packageName) { - final String joinedEscapedBody = createEscapedJoinedString(classBody); - classBody = classBody.replaceAll("\\$CLASSNAME\\$", className); - classBody = classBody.substring(0, classBody.lastIndexOf("}")); - classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; - return "package " + packageName + ";\n" + classBody; - } - - /** - * Transform a string into the corresponding Java source code that compiles into that string. This involves escaping - * special characters, surrounding it with quotes, and (if the string is larger than the max string length for Java - * literals), splitting it into substrings and constructing a call to String.join() that combines those substrings. - */ - public static String createEscapedJoinedString(final String originalString) { - return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); - } - - public static String createEscapedJoinedString(final String originalString, int maxStringLength) { - final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); - - // Turn each split into a Java source string by escaping it and surrounding it with " - for (int ii = 0; ii < splits.length; ++ii) { - final String escaped = StringEscapeUtils.escapeJava(splits[ii]); - splits[ii] = "\"" + escaped + "\""; - - } - assert splits.length > 0; - if (splits.length == 1) { - return splits[0]; - } - final String formattedInnards = String.join(",\n", splits); - return "String.join(\"\", " + formattedInnards + ")"; - } - - private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { - final List splits = new ArrayList<>(); - // exclusive end position of the previous substring. - int previousEnd = 0; - // Number of bytes in the "modified UTF-8" representation of the substring we are currently scanning. - int currentByteCount = 0; - for (int ii = 0; ii < originalString.length(); ++ii) { - final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); - if (currentByteCount + bytesConsumed > maxBytes) { - // This character won't fit in this string, so we flush the buffer. - splits.add(originalString.substring(previousEnd, ii)); - previousEnd = ii; - currentByteCount = 0; - } - currentByteCount += bytesConsumed; - } - // At the end of the loop, either - // 1. there are one or more characters that still need to be added to splits - // 2. originalString was empty and so splits is empty and we need to add a single empty string to splits - splits.add(originalString.substring(previousEnd)); - return splits.toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY); - } - - private static int calcBytesConsumed(final char ch) { - if (ch == 0) { - return 2; - } - if (ch <= 0x7f) { - return 1; - } - if (ch <= 0x7ff) { - return 2; - } - return 3; - } - - private static class JavaSourceFromString extends SimpleJavaFileObject { - final String description; - final String code; - final CompletionStageFuture.Resolver> resolver; - - JavaSourceFromString( - final String description, - final String name, - final String code, - final CompletionStageFuture.Resolver> resolver) { - super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); - this.description = description; - this.code = code; - this.resolver = resolver; - } - - public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return code; - } - } - - private static class CompilationRequestAttempt { - final String description; - final String fqClassName; - final String finalCode; - final String packageName; - final String[] splitPackageName; - final QueryCompilerRequest request; - final CompletionStageFuture.Resolver> resolver; - - private CompilationRequestAttempt( - @NotNull final QueryCompilerRequest request, - @NotNull final String packageName, - @NotNull final String fqClassName, - @NotNull final CompletionStageFuture.Resolver> resolver) { - this.description = request.description(); - this.fqClassName = fqClassName; - this.resolver = resolver; - this.packageName = packageName; - this.request = request; - - finalCode = makeFinalCode(request.className(), request.classBody(), packageName); - - if (logEnabled) { - log.info().append("Generating code ").append(finalCode).endl(); - } - - splitPackageName = packageName.split("\\."); - if (splitPackageName.length == 0) { - final Exception err = new UncheckedDeephavenException(String.format( - "packageName %s expected to have at least one .", packageName)); - resolver.completeExceptionally(err); - } - } - - public void ensureDirectories(@NotNull final String rootPath) { - if (splitPackageName.length == 0) { - // we've already failed - return; - } - - final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); - final Path rootPathWithPackage = Paths.get(rootPath, truncatedSplitPackageName); - final File rpf = rootPathWithPackage.toFile(); - QueryCompiler.ensureDirectories(rpf, () -> "Couldn't create package directories: " + rootPathWithPackage); - } - - public JavaSourceFromString makeSource() { - return new JavaSourceFromString(description, fqClassName, finalCode, resolver); - } - } - - private void maybeCreateClasses( - @NotNull final List requests) { - // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package - // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. - // Also create a temp directory e.g. /tmp/workspace/cache/classes/temporaryCompilationDirectory12345 - // This temp directory will be where the compiler drops files into, e.g. - // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/Formula.class - // Foreshadowing: we will eventually atomically move cm12862183232603186v52_0 from the above to - // /tmp/workspace/cache/classes/io/deephaven/test - // Note: for this atomic move to work, this temp directory must be on the same file system as the destination - // directory. - final String rootPathAsString; - final String tempDirAsString; - try { - rootPathAsString = getClassDestination().getAbsolutePath(); - final Path tempPath = - Files.createTempDirectory(Paths.get(rootPathAsString), "temporaryCompilationDirectory"); - tempDirAsString = tempPath.toFile().getAbsolutePath(); - - for (final CompilationRequestAttempt request : requests) { - request.ensureDirectories(rootPathAsString); - } - } catch (IOException ioe) { - Exception err = new UncheckedIOException(ioe); - for (final CompilationRequestAttempt request : requests) { - request.resolver.completeExceptionally(err); - } - return; - } - - - final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); - } - - final JavaFileManager fileManager = new SynchronizedJavaFileManager( - compiler.getStandardFileManager(null, null, null)); - - boolean exceptionCaught = false; - try { - final OperationInitializer operationInitializer = ExecutionContext.getContext().getOperationInitializer(); - int parallelismFactor = operationInitializer.parallelismFactor(); - - int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); - if (parallelismFactor == 1 || requestsPerTask >= requests.size()) { - doCreateClasses(compiler, fileManager, requests, rootPathAsString, tempDirAsString, - 0, requests.size()); - } else { - int numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; - final Future[] tasks = new Future[numTasks]; - for (int jobId = 0; jobId < numTasks; ++jobId) { - final int startInclusive = jobId * requestsPerTask; - final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); - tasks[jobId] = operationInitializer.submit(() -> { - doCreateClasses(compiler, fileManager, requests, rootPathAsString, tempDirAsString, - startInclusive, endExclusive); - }); - } - for (int jobId = 0; jobId < numTasks; ++jobId) { - try { - tasks[jobId].get(); - } catch (Exception err) { - throw new UncheckedDeephavenException("Exception waiting for compilation task", err); - } - } - } - } catch (final Throwable t) { - exceptionCaught = true; - throw t; - } finally { - try { - FileUtils.deleteRecursively(new File(tempDirAsString)); - } catch (Exception e) { - // ignore errors here - } - - try { - fileManager.close(); - } catch (IOException ioe) { - if (!exceptionCaught) { - // noinspection ThrowFromFinallyBlock - throw new UncheckedIOException("Could not close JavaFileManager", ioe); - } - } - } - } - - private void doCreateClasses( - @NotNull final JavaCompiler compiler, - @NotNull final JavaFileManager fileManager, - @NotNull final List requests, - @NotNull final String rootPathAsString, - @NotNull final String tempDirAsString, - final int startInclusive, - final int endExclusive) { - final List toRetry = new ArrayList<>(); - // If any of our requests fail to compile then the JavaCompiler will not write any class files at all. The - // non-failing requests will be retried in a second pass that is expected to succeed. This enables us to - // fulfill futures independent of each other; otherwise a single failure would taint all requests in a batch. - final boolean wantRetry = doCreateClassesSingleRound(compiler, fileManager, requests, rootPathAsString, - tempDirAsString, startInclusive, endExclusive, toRetry); - if (!wantRetry) { - return; - } - - final List ignored = new ArrayList<>(); - if (doCreateClassesSingleRound(compiler, fileManager, toRetry, rootPathAsString, tempDirAsString, 0, - toRetry.size(), ignored)) { - // We only retried compilation units that did not fail on the first pass, so we should not have any failures - // on the second pass. - throw new IllegalStateException("Unexpected failure during second pass of compilation"); - } - } - - private boolean doCreateClassesSingleRound( - @NotNull final JavaCompiler compiler, - @NotNull final JavaFileManager fileManager, - @NotNull final List requests, - @NotNull final String rootPathAsString, - @NotNull final String tempDirAsString, - final int startInclusive, - final int endExclusive, - List toRetry) { - final StringWriter compilerOutput = new StringWriter(); - - final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); - final List compilerOptions = Arrays.asList( - "-d", tempDirAsString, - "-cp", classPathAsString, - // this option allows the compiler to attempt to process all source files even if some of them fail - "--should-stop=ifError=GENERATE"); - - final MutableInt numFailures = new MutableInt(0); - compiler.getTask(compilerOutput, - fileManager, - diagnostic -> { - if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { - return; - } - - final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); - final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " - + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); - if (source.resolver.completeExceptionally(err)) { - // only count the first failure for each source - numFailures.increment(); - } - }, - compilerOptions, - null, - requests.subList(startInclusive, endExclusive).stream() - .map(CompilationRequestAttempt::makeSource) - .collect(Collectors.toList())) - .call(); - - final boolean wantRetry = numFailures.intValue() > 0 && numFailures.intValue() != endExclusive - startInclusive; - - // The above has compiled into e.g. - // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various - // class files} - // We want to atomically move it to e.g. - // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} - requests.subList(startInclusive, endExclusive).forEach(request -> { - final Path srcDir = Paths.get(tempDirAsString, request.splitPackageName); - final Path destDir = Paths.get(rootPathAsString, request.splitPackageName); - try { - Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException ioe) { - // The name "isDone" might be misleading here. We haven't called "complete" on the successful - // futures yet, so the only way they would be "done" at this point is if they completed - // exceptionally. - final boolean hasException = request.resolver.getFuture().isDone(); - - if (wantRetry && !Files.exists(srcDir) && !hasException) { - // The move failed, the source directory does not exist, and this compilation unit actually - // succeeded. However, it was not written because some other compilation unit failed. Let's schedule - // this work to try again. - toRetry.add(request); - return; - } - - if (!Files.exists(destDir) && !hasException) { - // Propagate an error here only if the destination does not exist; ignoring issues related to - // collisions with another process. - request.resolver.completeExceptionally(new UncheckedIOException( - "Move failed for some reason other than destination already existing", ioe)); - } - } - }); - - return wantRetry; - } - - /** - * @return the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables - */ - private static String getJavaClassPath() { - String javaClasspath; - { - final StringBuilder javaClasspathBuilder = new StringBuilder(System.getProperty("java.class.path")); - - final String teamCityWorkDir = System.getProperty("teamcity.build.workingDir"); - if (teamCityWorkDir != null) { - // We are running in TeamCity, get the classpath differently - final File[] classDirs = new File(teamCityWorkDir + "/_out_/classes").listFiles(); - if (classDirs != null) { - for (File f : classDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); - } - } - - final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); - if (testDirs != null) { - for (File f : testDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); - } - } - - final File[] jars = FileUtils.findAllFiles(new File(teamCityWorkDir + "/lib")); - for (File f : jars) { - if (f.getName().endsWith(".jar")) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); - } - } - } - javaClasspath = javaClasspathBuilder.toString(); - } - - // IntelliJ will bundle a very large class path into an empty jar with a Manifest that will define the full - // class path - // Look for this being used during compile time, so the full class path can be sent into the compile call - final String intellijClassPathJarRegex = ".*classpath[0-9]*\\.jar.*"; - if (javaClasspath.matches(intellijClassPathJarRegex)) { - try { - final Enumeration resources = - QueryCompiler.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); - final Attributes.Name createdByAttribute = new Attributes.Name("Created-By"); - final Attributes.Name classPathAttribute = new Attributes.Name("Class-Path"); - while (resources.hasMoreElements()) { - // Check all manifests -- looking for the Intellij created one - final Manifest manifest = new Manifest(resources.nextElement().openStream()); - final Attributes attributes = manifest.getMainAttributes(); - final Object createdBy = attributes.get(createdByAttribute); - if ("IntelliJ IDEA".equals(createdBy)) { - final String extendedClassPath = (String) attributes.get(classPathAttribute); - if (extendedClassPath != null) { - // Parses the files in the manifest description an changes their format to drop the "file:/" - // and - // use the default path separator - final String filePaths = Stream.of(extendedClassPath.split("file:/")) - .map(String::trim) - .filter(fileName -> !fileName.isEmpty()) - .collect(Collectors.joining(File.pathSeparator)); - - // Remove the classpath jar in question, and expand it with the files from the manifest - javaClasspath = Stream.of(javaClasspath.split(File.pathSeparator)) - .map(cp -> cp.matches(intellijClassPathJarRegex) ? filePaths : cp) - .collect(Collectors.joining(File.pathSeparator)); - } - } - } - } catch (IOException e) { - throw new UncheckedIOException("Error extract manifest file from " + javaClasspath + ".\n", e); - } - } - return javaClasspath; - } + void compile( + @NotNull QueryCompilerRequest[] requests, + @NotNull CompletionStageFuture.Resolver>[] resolvers); } diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index 3a42aabfa9c..d9bca87a113 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -71,7 +71,8 @@ public void setUp() throws IOException { executionContextClosable = ExecutionContext.newBuilder() .captureQueryLibrary() .captureQueryScope() - .setQueryCompiler(QueryCompiler.create(folder.newFolder(), TestQueryCompiler.class.getClassLoader())) + .setQueryCompiler(QueryCompilerImpl.create( + folder.newFolder(), TestQueryCompiler.class.getClassLoader())) .build() .open(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java new file mode 100644 index 00000000000..bbe30625a5b --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -0,0 +1,1041 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.context; + +import io.deephaven.UncheckedDeephavenException; +import io.deephaven.base.FileUtils; +import io.deephaven.base.verify.Assert; +import io.deephaven.configuration.Configuration; +import io.deephaven.configuration.DataDir; +import io.deephaven.datastructures.util.CollectionUtil; +import io.deephaven.engine.context.util.SynchronizedJavaFileManager; +import io.deephaven.engine.updategraph.OperationInitializer; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import io.deephaven.util.ByteUtils; +import io.deephaven.util.CompletionStageFuture; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.commons.text.StringEscapeUtils; +import org.jetbrains.annotations.NotNull; + +import javax.tools.*; +import java.io.*; +import java.lang.reflect.Field; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; +import java.util.concurrent.*; +import java.util.function.Supplier; +import java.util.jar.Attributes; +import java.util.jar.Manifest; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class QueryCompilerImpl implements QueryCompiler { + + private static final Logger log = LoggerFactory.getLogger(QueryCompilerImpl.class); + /** + * We pick a number just shy of 65536, leaving a little elbow room for good luck. + */ + private static final int DEFAULT_MAX_STRING_LITERAL_LENGTH = 65500; + + private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version").replace('.', '_'); + private static final int MAX_CLASS_COLLISIONS = 128; + + private static final String IDENTIFYING_FIELD_NAME = "_CLASS_BODY_"; + + private static final String CODEGEN_TIMEOUT_PROP = "QueryCompiler.codegen.timeoutMs"; + private static final long CODEGEN_TIMEOUT_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); // 10 seconds + private static final String CODEGEN_LOOP_DELAY_PROP = "QueryCompiler.codegen.retry.delay"; + private static final long CODEGEN_LOOP_DELAY_MS_DEFAULT = 100; + private static final long CODEGEN_TIMEOUT_MS = + Configuration.getInstance().getLongWithDefault(CODEGEN_TIMEOUT_PROP, CODEGEN_TIMEOUT_MS_DEFAULT); + private static final long CODEGEN_LOOP_DELAY_MS = + Configuration.getInstance().getLongWithDefault(CODEGEN_LOOP_DELAY_PROP, CODEGEN_LOOP_DELAY_MS_DEFAULT); + + private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); + + public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; + public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; + + public static QueryCompilerImpl create(File cacheDirectory, ClassLoader classLoader) { + return new QueryCompilerImpl(cacheDirectory, classLoader, true); + } + + static QueryCompilerImpl createForUnitTests() { + final Path queryCompilerDir = DataDir.get() + .resolve("io.deephaven.engine.context.QueryCompiler.createForUnitTests"); + return new QueryCompilerImpl(queryCompilerDir.toFile()); + } + + private final Map>> knownClasses = new HashMap<>(); + + private final String[] dynamicPatterns = new String[] {DYNAMIC_CLASS_PREFIX, FORMULA_CLASS_PREFIX}; + + private final File classDestination; + private final boolean isCacheDirectory; + private final Set additionalClassLocations; + private volatile WritableURLClassLoader ucl; + + private QueryCompilerImpl(File classDestination) { + this(classDestination, null, false); + } + + private QueryCompilerImpl( + final File classDestination, + final ClassLoader parentClassLoader, + final boolean isCacheDirectory) { + final ClassLoader parentClassLoaderToUse = parentClassLoader == null + ? QueryCompilerImpl.class.getClassLoader() + : parentClassLoader; + this.classDestination = classDestination; + this.isCacheDirectory = isCacheDirectory; + ensureDirectories(this.classDestination, () -> "Failed to create missing class destination directory " + + classDestination.getAbsolutePath()); + additionalClassLocations = new LinkedHashSet<>(); + + URL[] urls = new URL[1]; + try { + urls[0] = (classDestination.toURI().toURL()); + } catch (MalformedURLException e) { + throw new UncheckedDeephavenException(e); + } + this.ucl = new WritableURLClassLoader(urls, parentClassLoaderToUse); + + if (isCacheDirectory) { + addClassSource(classDestination); + } + } + + /** + * Enables or disables compilation logging. + * + * @param logEnabled Whether logging should be enabled + * @return The value of {@code logEnabled} before calling this method. + */ + public static boolean setLogEnabled(boolean logEnabled) { + boolean original = QueryCompilerImpl.logEnabled; + QueryCompilerImpl.logEnabled = logEnabled; + return original; + } + + /* + * NB: This is (obviously) not thread safe if code tries to write the same className to the same + * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving + * synchronization as an external concern. + */ + public static void writeClass(final File destinationDirectory, final String className, final byte[] data) + throws IOException { + writeClass(destinationDirectory, className, data, null); + } + + /* + * NB: This is (obviously) not thread safe if code tries to write the same className to the same + * destinationDirectory from multiple threads. Seeing as we don't currently have this use case, leaving + * synchronization as an external concern. + */ + public static void writeClass(final File destinationDirectory, final String className, final byte[] data, + final String message) throws IOException { + final File destinationFile = new File(destinationDirectory, + className.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); + + if (destinationFile.exists()) { + final byte[] existingBytes = Files.readAllBytes(destinationFile.toPath()); + if (Arrays.equals(existingBytes, data)) { + if (message == null) { + log.info().append("Ignoring pushed class ").append(className) + .append(" because it already exists in this context!").endl(); + } else { + log.info().append("Ignoring pushed class ").append(className).append(message) + .append(" because it already exists in this context!").endl(); + } + return; + } else { + if (message == null) { + log.info().append("Pushed class ").append(className) + .append(" already exists in this context, but has changed!").endl(); + } else { + log.info().append("Pushed class ").append(className).append(message) + .append(" already exists in this context, but has changed!").endl(); + } + if (!destinationFile.delete()) { + throw new IOException("Could not delete existing class file: " + destinationFile); + } + } + } + + final File parentDir = destinationFile.getParentFile(); + ensureDirectories(parentDir, + () -> "Unable to create missing destination directory " + parentDir.getAbsolutePath()); + if (!destinationFile.createNewFile()) { + throw new UncheckedDeephavenException( + "Unable to create destination file " + destinationFile.getAbsolutePath()); + } + final ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(data.length); + byteOutStream.write(data, 0, data.length); + final FileOutputStream fileOutStream = new FileOutputStream(destinationFile); + byteOutStream.writeTo(fileOutStream); + fileOutStream.close(); + } + + public File getTemporaryClassDestination() { + // Groovy classes need to be written out to a location where they can be found by the compiler + // (so that filters and formulae can use them). + // + // We don't want the regular runtime class loader to find them, because then they get "stuck" in there + // even if the class itself changes, and we can't forget it. So instead we use a single-use class loader + // for each formula, that will always read the class from disk. + return isCacheDirectory ? classDestination : null; + } + + public void setParentClassLoader(@NotNull final ClassLoader parentClassLoader) { + // noinspection NonAtomicOperationOnVolatileField + ucl = new WritableURLClassLoader(ucl.getURLs(), parentClassLoader); + } + + /** + * Compiles all requests. + * + * @param requests The compilation requests; these must be independent of each other + * @param resolvers The resolvers to use for delivering compilation results + */ + public void compile( + @NotNull final QueryCompilerRequest[] requests, + @NotNull final CompletionStageFuture.Resolver>[] resolvers) { + if (requests.length == 0) { + return; + } + if (requests.length != resolvers.length) { + throw new IllegalArgumentException("Requests and resolvers must be the same length"); + } + + // noinspection unchecked + final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; + + final List newRequests = new ArrayList<>(); + final List>> newResolvers = new ArrayList<>(); + + synchronized (this) { + for (int ii = 0; ii < requests.length; ++ii) { + final QueryCompilerRequest request = requests[ii]; + final CompletionStageFuture.Resolver> resolver = resolvers[ii]; + + CompletionStageFuture> future = + knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); + if (future == null) { + newRequests.add(request); + newResolvers.add(resolver); + future = resolver.getFuture(); + } + allFutures[ii] = future; + } + } + + if (!newResolvers.isEmpty()) { + // It's my job to fulfill these futures. + try { + compileHelper(newRequests, newResolvers); + } catch (RuntimeException e) { + // These failures are not applicable to a single request, so we can't just complete the future and + // leave the failure in the cache. + synchronized (this) { + for (int ii = 0; ii < newRequests.size(); ++ii) { + if (newResolvers.get(ii).completeExceptionally(e)) { + knownClasses.remove(newRequests.get(ii).classBody()); + } + } + } + throw e; + } + } + + for (int ii = 0; ii < requests.length; ++ii) { + try { + resolvers[ii].complete(allFutures[ii].get()); + } catch (ExecutionException err) { + resolvers[ii].completeExceptionally(err.getCause()); + } catch (InterruptedException err) { + // This can only occur if we are interrupted while waiting for the future to complete from another + // compilation request. + Assert.notEquals(resolvers[ii], "resolvers[ii]", allFutures[ii], "allFutures[ii]"); + resolvers[ii].completeExceptionally(err); + } catch (Throwable err) { + resolvers[ii].completeExceptionally(err); + } + } + } + + private static void ensureDirectories(final File file, final Supplier runtimeErrMsg) { + // File.mkdirs() checks for existence on entry, in which case it returns false. + // It may also return false on a failure to create. + // Also note, two separate threads or JVMs may be running this code in parallel. It's possible that we could + // lose the race + // (and therefore mkdirs() would return false), but still get the directory we need (and therefore exists() + // would return true) + if (!file.mkdirs() && !file.isDirectory()) { + throw new UncheckedDeephavenException(runtimeErrMsg.get()); + } + } + + private ClassLoader getClassLoaderForFormula(final Map> parameterClasses) { + return new URLClassLoader(ucl.getURLs(), ucl) { + // Once we find a class that is missing, we should not attempt to load it again, + // otherwise we can end up with a StackOverflow Exception + final HashSet missingClasses = new HashSet<>(); + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + // If we have a parameter that uses this class, return it + final Class paramClass = parameterClasses.get(name); + if (paramClass != null) { + return paramClass; + } + + // Unless we are looking for a formula or Groovy class, we should use the default behavior + if (!isFormulaClass(name)) { + return super.findClass(name); + } + + // if it is a groovy class, always try to use the instance in the shell + if (name.startsWith(DYNAMIC_CLASS_PREFIX)) { + try { + return ucl.getParent().loadClass(name); + } catch (final ClassNotFoundException ignored) { + // we'll try to load it otherwise + } + } + + // We've already not found this class, so we should not try to search again + if (missingClasses.contains(name)) { + return super.findClass(name); + } + + final byte[] bytes; + try { + bytes = loadClassData(name); + } catch (IOException ioe) { + missingClasses.add(name); + return super.loadClass(name); + } + return defineClass(name, bytes, 0, bytes.length); + } + + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + private boolean isFormulaClass(String name) { + return Arrays.stream(dynamicPatterns).anyMatch(name::startsWith); + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + if (!isFormulaClass(name)) { + return super.loadClass(name); + } + return findClass(name); + } + + private byte[] loadClassData(String name) throws IOException { + final File destFile = new File(classDestination, + name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); + if (destFile.exists()) { + return Files.readAllBytes(destFile.toPath()); + } + + for (File location : additionalClassLocations) { + final File checkFile = new File(location, + name.replace('.', File.separatorChar) + JavaFileObject.Kind.CLASS.extension); + if (checkFile.exists()) { + return Files.readAllBytes(checkFile.toPath()); + } + } + + throw new FileNotFoundException(name); + } + }; + } + + private static class WritableURLClassLoader extends URLClassLoader { + private WritableURLClassLoader(URL[] urls, ClassLoader parent) { + super(urls, parent); + } + + @Override + protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + Class clazz = findLoadedClass(name); + if (clazz != null) { + return clazz; + } + + try { + clazz = findClass(name); + } catch (ClassNotFoundException e) { + if (getParent() != null) { + clazz = getParent().loadClass(name); + } + } + + if (resolve) { + resolveClass(clazz); + } + return clazz; + } + + @Override + public synchronized void addURL(URL url) { + super.addURL(url); + } + } + + private void addClassSource(File classSourceDirectory) { + synchronized (additionalClassLocations) { + if (additionalClassLocations.contains(classSourceDirectory)) { + return; + } + additionalClassLocations.add(classSourceDirectory); + } + try { + ucl.addURL(classSourceDirectory.toURI().toURL()); + } catch (MalformedURLException e) { + throw new UncheckedDeephavenException(e); + } + } + + private File getClassDestination() { + return classDestination; + } + + private String getClassPath() { + StringBuilder sb = new StringBuilder(); + sb.append(classDestination.getAbsolutePath()); + synchronized (additionalClassLocations) { + for (File classLoc : additionalClassLocations) { + sb.append(File.pathSeparatorChar).append(classLoc.getAbsolutePath()); + } + } + return sb.toString(); + } + + private static class CompilationState { + int nextProbeIndex; + boolean complete; + String packageName; + String fqClassName; + } + + private void compileHelper( + @NotNull final List requests, + @NotNull final List>> resolvers) { + final MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); + } + + final String[] basicHashText = new String[requests.size()]; + for (int ii = 0; ii < requests.size(); ++ii) { + basicHashText[ii] = ByteUtils.byteArrToHex(digest.digest( + requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); + } + + int numComplete = 0; + final CompilationState[] states = new CompilationState[requests.size()]; + for (int ii = 0; ii < requests.size(); ++ii) { + states[ii] = new CompilationState(); + } + + /* + * @formatter:off + * 1. try to resolve without compiling; retain next hash to try + * 2. compile all remaining with a single compilation task + * 3. goto step 1 if any are unresolved + * @formatter:on + */ + + while (numComplete < requests.size()) { + for (int ii = 0; ii < requests.size(); ++ii) { + final CompilationState state = states[ii]; + if (state.complete) { + continue; + } + + next_probe: while (true) { + final int pi = state.nextProbeIndex++; + final String packageNameSuffix = "c_" + basicHashText[ii] + + (pi == 0 ? "" : ("p" + pi)) + + "v" + JAVA_CLASS_VERSION; + + final QueryCompilerRequest request = requests.get(ii); + if (pi >= MAX_CLASS_COLLISIONS) { + Exception err = new IllegalStateException("Found too many collisions for package name root " + + request.packageNameRoot() + ", class name=" + request.className() + ", class body " + + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); + resolvers.get(ii).completeExceptionally(err); + state.complete = true; + ++numComplete; + break; + } + + state.packageName = request.getPackageName(packageNameSuffix); + state.fqClassName = state.packageName + "." + request.className(); + + for (int jj = 0; jj < ii; ++jj) { + if (states[jj].fqClassName.equals(state.fqClassName)) { + // collision within batch + continue next_probe; + } + } + + // Ask the classloader to load an existing class with this name. This might: + // 1. Fail to find a class (returning null) + // 2. Find a class whose body has the formula we are looking for + // 3. Find a class whose body has a different formula (hash collision) + Class result = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); + if (result == null) { + break; // we'll try to compile it + } + + if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolvers.get(ii), + result)) { + state.complete = true; + ++numComplete; + break; + } + } + } + + if (numComplete == requests.size()) { + return; + } + + // Couldn't resolve at least one of the requests, so try a round of compilation. + final List compilationRequestAttempts = new ArrayList<>(); + for (int ii = 0; ii < requests.size(); ++ii) { + final CompilationState state = states[ii]; + if (!state.complete) { + final QueryCompilerRequest request = requests.get(ii); + compilationRequestAttempts.add(new CompilationRequestAttempt( + request, + state.packageName, + state.fqClassName, + resolvers.get(ii))); + } + } + + maybeCreateClasses(compilationRequestAttempts); + + // We could be running on a screwy filesystem that is slow (e.g. NFS). If we wrote a file and can't load it + // ... then give the filesystem some time. All requests should use the same deadline. + final long deadline = System.currentTimeMillis() + CODEGEN_TIMEOUT_MS - CODEGEN_LOOP_DELAY_MS; + for (int ii = 0; ii < requests.size(); ++ii) { + final CompilationState state = states[ii]; + if (state.complete) { + continue; + } + + final QueryCompilerRequest request = requests.get(ii); + final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); + if (resolver.getFuture().isDone()) { + state.complete = true; + ++numComplete; + continue; + } + + // This request may have: + // A. succeeded + // B. Lost a race to another process on the same file system which is compiling the identical formula + // C. Lost a race to another process on the same file system compiling a different formula that collides + + Class clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); + try { + while (clazz == null && System.currentTimeMillis() < deadline) { + // noinspection BusyWait + Thread.sleep(CODEGEN_LOOP_DELAY_MS); + clazz = tryLoadClassByFqName(state.fqClassName, request.parameterClasses()); + } + } catch (final InterruptedException ie) { + throw new UncheckedDeephavenException("Interrupted while waiting for codegen", ie); + } + + // However, regardless of A-C, there will be *some* class being found + if (clazz == null) { + throw new IllegalStateException("Should have been able to load *some* class here"); + } + + if (completeIfResultMatchesQueryCompilerRequest(state.packageName, request, resolver, clazz)) { + state.complete = true; + ++numComplete; + } + } + } + } + + private boolean completeIfResultMatchesQueryCompilerRequest( + final String packageName, + final QueryCompilerRequest request, + final CompletionStageFuture.Resolver> resolver, + final Class result) { + final String identifyingFieldValue = loadIdentifyingField(result); + if (!request.classBody().equals(identifyingFieldValue)) { + return false; + } + + // If the caller wants a textual copy of the code we either made, or just found in the cache. + request.codeLog() + .ifPresent(sb -> sb.append(makeFinalCode(request.className(), request.classBody(), packageName))); + + // If the class we found was indeed the class we were looking for, then complete the future and return it. + resolver.complete(result); + + synchronized (this) { + // Note we are doing something kind of subtle here. We are removing an entry whose key was matched + // by value equality and replacing it with a value-equal but reference-different string that is a + // static member of the class we just loaded. This should be easier on the garbage collector because + // we are replacing a calculated value with a classloaded value and so in effect we are + // "canonicalizing" the string. This is important because these long strings stay in knownClasses + // forever. + knownClasses.remove(identifyingFieldValue); + knownClasses.put(identifyingFieldValue, resolver.getFuture()); + } + + return true; + } + + private Class tryLoadClassByFqName(String fqClassName, Map> parameterClasses) { + try { + return getClassLoaderForFormula(parameterClasses).loadClass(fqClassName); + } catch (ClassNotFoundException cnfe) { + return null; + } + } + + private static String loadIdentifyingField(Class c) { + try { + final Field field = c.getDeclaredField(IDENTIFYING_FIELD_NAME); + return (String) field.get(null); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalStateException("Malformed class in cache", e); + } + } + + /** + * Transform a string into the corresponding Java source code that compiles into that string. This involves escaping + * special characters, surrounding it with quotes, and (if the string is larger than the max string length for Java + * literals), splitting it into substrings and constructing a call to String.join() that combines those substrings. + */ + public static String createEscapedJoinedString(final String originalString) { + return createEscapedJoinedString(originalString, DEFAULT_MAX_STRING_LITERAL_LENGTH); + } + + public static String createEscapedJoinedString(final String originalString, int maxStringLength) { + final String[] splits = splitByModifiedUtf8Encoding(originalString, maxStringLength); + + // Turn each split into a Java source string by escaping it and surrounding it with " + for (int ii = 0; ii < splits.length; ++ii) { + final String escaped = StringEscapeUtils.escapeJava(splits[ii]); + splits[ii] = "\"" + escaped + "\""; + + } + assert splits.length > 0; + if (splits.length == 1) { + return splits[0]; + } + final String formattedInnards = String.join(",\n", splits); + return "String.join(\"\", " + formattedInnards + ")"; + } + + private static String[] splitByModifiedUtf8Encoding(final String originalString, int maxBytes) { + final List splits = new ArrayList<>(); + // exclusive end position of the previous substring. + int previousEnd = 0; + // Number of bytes in the "modified UTF-8" representation of the substring we are currently scanning. + int currentByteCount = 0; + for (int ii = 0; ii < originalString.length(); ++ii) { + final int bytesConsumed = calcBytesConsumed(originalString.charAt(ii)); + if (currentByteCount + bytesConsumed > maxBytes) { + // This character won't fit in this string, so we flush the buffer. + splits.add(originalString.substring(previousEnd, ii)); + previousEnd = ii; + currentByteCount = 0; + } + currentByteCount += bytesConsumed; + } + // At the end of the loop, either + // 1. there are one or more characters that still need to be added to splits + // 2. originalString was empty and so splits is empty and we need to add a single empty string to splits + splits.add(originalString.substring(previousEnd)); + return splits.toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY); + } + + private static int calcBytesConsumed(final char ch) { + if (ch == 0) { + return 2; + } + if (ch <= 0x7f) { + return 1; + } + if (ch <= 0x7ff) { + return 2; + } + return 3; + } + + private static String makeFinalCode(String className, String classBody, String packageName) { + final String joinedEscapedBody = createEscapedJoinedString(classBody); + classBody = classBody.replaceAll("\\$CLASSNAME\\$", className); + classBody = classBody.substring(0, classBody.lastIndexOf("}")); + classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; + return "package " + packageName + ";\n" + classBody; + } + + private static class JavaSourceFromString extends SimpleJavaFileObject { + final String description; + final String code; + final CompletionStageFuture.Resolver> resolver; + + JavaSourceFromString( + final String description, + final String name, + final String code, + final CompletionStageFuture.Resolver> resolver) { + super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); + this.description = description; + this.code = code; + this.resolver = resolver; + } + + public CharSequence getCharContent(boolean ignoreEncodingErrors) { + return code; + } + } + + private static class CompilationRequestAttempt { + final String description; + final String fqClassName; + final String finalCode; + final String packageName; + final String[] splitPackageName; + final QueryCompilerRequest request; + final CompletionStageFuture.Resolver> resolver; + + private CompilationRequestAttempt( + @NotNull final QueryCompilerRequest request, + @NotNull final String packageName, + @NotNull final String fqClassName, + @NotNull final CompletionStageFuture.Resolver> resolver) { + this.description = request.description(); + this.fqClassName = fqClassName; + this.resolver = resolver; + this.packageName = packageName; + this.request = request; + + finalCode = makeFinalCode(request.className(), request.classBody(), packageName); + + if (logEnabled) { + log.info().append("Generating code ").append(finalCode).endl(); + } + + splitPackageName = packageName.split("\\."); + if (splitPackageName.length == 0) { + final Exception err = new UncheckedDeephavenException(String.format( + "packageName %s expected to have at least one .", packageName)); + resolver.completeExceptionally(err); + } + } + + public void ensureDirectories(@NotNull final String rootPath) { + if (splitPackageName.length == 0) { + // we've already failed + return; + } + + final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); + final Path rootPathWithPackage = Paths.get(rootPath, truncatedSplitPackageName); + final File rpf = rootPathWithPackage.toFile(); + QueryCompilerImpl.ensureDirectories(rpf, + () -> "Couldn't create package directories: " + rootPathWithPackage); + } + + public JavaSourceFromString makeSource() { + return new JavaSourceFromString(description, fqClassName, finalCode, resolver); + } + } + + private void maybeCreateClasses( + @NotNull final List requests) { + // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package + // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. + // Also create a temp directory e.g. /tmp/workspace/cache/classes/temporaryCompilationDirectory12345 + // This temp directory will be where the compiler drops files into, e.g. + // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/Formula.class + // Foreshadowing: we will eventually atomically move cm12862183232603186v52_0 from the above to + // /tmp/workspace/cache/classes/io/deephaven/test + // Note: for this atomic move to work, this temp directory must be on the same file system as the destination + // directory. + final String rootPathAsString; + final String tempDirAsString; + try { + rootPathAsString = getClassDestination().getAbsolutePath(); + final Path tempPath = + Files.createTempDirectory(Paths.get(rootPathAsString), "temporaryCompilationDirectory"); + tempDirAsString = tempPath.toFile().getAbsolutePath(); + + for (final CompilationRequestAttempt request : requests) { + request.ensureDirectories(rootPathAsString); + } + } catch (IOException ioe) { + Exception err = new UncheckedIOException(ioe); + for (final CompilationRequestAttempt request : requests) { + request.resolver.completeExceptionally(err); + } + return; + } + + + final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + if (compiler == null) { + throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); + } + + final JavaFileManager fileManager = new SynchronizedJavaFileManager( + compiler.getStandardFileManager(null, null, null)); + + boolean exceptionCaught = false; + try { + final OperationInitializer operationInitializer = ExecutionContext.getContext().getOperationInitializer(); + int parallelismFactor = operationInitializer.parallelismFactor(); + + int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); + if (parallelismFactor == 1 || requestsPerTask >= requests.size()) { + doCreateClasses(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + 0, requests.size()); + } else { + int numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; + final Future[] tasks = new Future[numTasks]; + for (int jobId = 0; jobId < numTasks; ++jobId) { + final int startInclusive = jobId * requestsPerTask; + final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); + tasks[jobId] = operationInitializer.submit(() -> { + doCreateClasses(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + startInclusive, endExclusive); + }); + } + for (int jobId = 0; jobId < numTasks; ++jobId) { + try { + tasks[jobId].get(); + } catch (Exception err) { + throw new UncheckedDeephavenException("Exception waiting for compilation task", err); + } + } + } + } catch (final Throwable t) { + exceptionCaught = true; + throw t; + } finally { + try { + FileUtils.deleteRecursively(new File(tempDirAsString)); + } catch (Exception e) { + // ignore errors here + } + + try { + fileManager.close(); + } catch (IOException ioe) { + if (!exceptionCaught) { + // noinspection ThrowFromFinallyBlock + throw new UncheckedIOException("Could not close JavaFileManager", ioe); + } + } + } + } + + private void doCreateClasses( + @NotNull final JavaCompiler compiler, + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, + @NotNull final String rootPathAsString, + @NotNull final String tempDirAsString, + final int startInclusive, + final int endExclusive) { + final List toRetry = new ArrayList<>(); + // If any of our requests fail to compile then the JavaCompiler will not write any class files at all. The + // non-failing requests will be retried in a second pass that is expected to succeed. This enables us to + // fulfill futures independent of each other; otherwise a single failure would taint all requests in a batch. + final boolean wantRetry = doCreateClassesSingleRound(compiler, fileManager, requests, rootPathAsString, + tempDirAsString, startInclusive, endExclusive, toRetry); + if (!wantRetry) { + return; + } + + final List ignored = new ArrayList<>(); + if (doCreateClassesSingleRound(compiler, fileManager, toRetry, rootPathAsString, tempDirAsString, 0, + toRetry.size(), ignored)) { + // We only retried compilation units that did not fail on the first pass, so we should not have any failures + // on the second pass. + throw new IllegalStateException("Unexpected failure during second pass of compilation"); + } + } + + private boolean doCreateClassesSingleRound( + @NotNull final JavaCompiler compiler, + @NotNull final JavaFileManager fileManager, + @NotNull final List requests, + @NotNull final String rootPathAsString, + @NotNull final String tempDirAsString, + final int startInclusive, + final int endExclusive, + List toRetry) { + final StringWriter compilerOutput = new StringWriter(); + + final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); + final List compilerOptions = Arrays.asList( + "-d", tempDirAsString, + "-cp", classPathAsString, + // this option allows the compiler to attempt to process all source files even if some of them fail + "--should-stop=ifError=GENERATE"); + + final MutableInt numFailures = new MutableInt(0); + compiler.getTask(compilerOutput, + fileManager, + diagnostic -> { + if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { + return; + } + + final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); + final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " + + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); + if (source.resolver.completeExceptionally(err)) { + // only count the first failure for each source + numFailures.increment(); + } + }, + compilerOptions, + null, + requests.subList(startInclusive, endExclusive).stream() + .map(CompilationRequestAttempt::makeSource) + .collect(Collectors.toList())) + .call(); + + final boolean wantRetry = numFailures.intValue() > 0 && numFailures.intValue() != endExclusive - startInclusive; + + // The above has compiled into e.g. + // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various + // class files} + // We want to atomically move it to e.g. + // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} + requests.subList(startInclusive, endExclusive).forEach(request -> { + final Path srcDir = Paths.get(tempDirAsString, request.splitPackageName); + final Path destDir = Paths.get(rootPathAsString, request.splitPackageName); + try { + Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException ioe) { + // The name "isDone" might be misleading here. We haven't called "complete" on the successful + // futures yet, so the only way they would be "done" at this point is if they completed + // exceptionally. + final boolean hasException = request.resolver.getFuture().isDone(); + + if (wantRetry && !Files.exists(srcDir) && !hasException) { + // The move failed, the source directory does not exist, and this compilation unit actually + // succeeded. However, it was not written because some other compilation unit failed. Let's schedule + // this work to try again. + toRetry.add(request); + return; + } + + if (!Files.exists(destDir) && !hasException) { + // Propagate an error here only if the destination does not exist; ignoring issues related to + // collisions with another process. + request.resolver.completeExceptionally(new UncheckedIOException( + "Move failed for some reason other than destination already existing", ioe)); + } + } + }); + + return wantRetry; + } + + /** + * @return the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables + */ + private static String getJavaClassPath() { + String javaClasspath; + { + final StringBuilder javaClasspathBuilder = new StringBuilder(System.getProperty("java.class.path")); + + final String teamCityWorkDir = System.getProperty("teamcity.build.workingDir"); + if (teamCityWorkDir != null) { + // We are running in TeamCity, get the classpath differently + final File[] classDirs = new File(teamCityWorkDir + "/_out_/classes").listFiles(); + if (classDirs != null) { + for (File f : classDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } + } + + final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); + if (testDirs != null) { + for (File f : testDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } + } + + final File[] jars = FileUtils.findAllFiles(new File(teamCityWorkDir + "/lib")); + for (File f : jars) { + if (f.getName().endsWith(".jar")) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } + } + } + javaClasspath = javaClasspathBuilder.toString(); + } + + // IntelliJ will bundle a very large class path into an empty jar with a Manifest that will define the full + // class path + // Look for this being used during compile time, so the full class path can be sent into the compile call + final String intellijClassPathJarRegex = ".*classpath[0-9]*\\.jar.*"; + if (javaClasspath.matches(intellijClassPathJarRegex)) { + try { + final Enumeration resources = + QueryCompilerImpl.class.getClassLoader().getResources("META-INF/MANIFEST.MF"); + final Attributes.Name createdByAttribute = new Attributes.Name("Created-By"); + final Attributes.Name classPathAttribute = new Attributes.Name("Class-Path"); + while (resources.hasMoreElements()) { + // Check all manifests -- looking for the Intellij created one + final Manifest manifest = new Manifest(resources.nextElement().openStream()); + final Attributes attributes = manifest.getMainAttributes(); + final Object createdBy = attributes.get(createdByAttribute); + if ("IntelliJ IDEA".equals(createdBy)) { + final String extendedClassPath = (String) attributes.get(classPathAttribute); + if (extendedClassPath != null) { + // Parses the files in the manifest description an changes their format to drop the "file:/" + // and + // use the default path separator + final String filePaths = Stream.of(extendedClassPath.split("file:/")) + .map(String::trim) + .filter(fileName -> !fileName.isEmpty()) + .collect(Collectors.joining(File.pathSeparator)); + + // Remove the classpath jar in question, and expand it with the files from the manifest + javaClasspath = Stream.of(javaClasspath.split(File.pathSeparator)) + .map(cp -> cp.matches(intellijClassPathJarRegex) ? filePaths : cp) + .collect(Collectors.joining(File.pathSeparator)); + } + } + } + } catch (IOException e) { + throw new UncheckedIOException("Error extract manifest file from " + javaClasspath + ".\n", e); + } + } + return javaClasspath; + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java index fc065f6b052..f08995981e0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java @@ -5,8 +5,8 @@ import io.deephaven.base.Pair; import io.deephaven.chunk.attributes.Any; -import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Context; @@ -420,7 +420,7 @@ protected void generateFilterCode( .description("Filter Expression: " + formula) .className("GeneratedFilterKernel") .classBody(this.classBody) - .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) .build()); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index edc58f80a0f..e772fa8d059 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -8,7 +8,7 @@ import io.deephaven.chunk.ChunkType; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.ColumnDefinition; @@ -377,7 +377,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(formulaString)); - final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(originalFormulaString); + final String joinedFormulaString = QueryCompilerImpl.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", EVALUATION_EXCEPTION_CLASSNAME); return g.freeze(); @@ -815,7 +815,7 @@ private void compileFormula(@NotNull final QueryCompilerRequestProcessor compila .description("Formula Expression: " + formulaString) .className(className) .classBody(classBody) - .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) .build()).thenApply(clazz -> { try { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java index eec62883efc..24cb8332a46 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/QueryScopeParamTypeUtil.java @@ -4,7 +4,7 @@ package io.deephaven.engine.table.impl.select; import groovy.lang.Closure; -import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.util.type.TypeUtils; import java.lang.reflect.Modifier; @@ -48,7 +48,7 @@ private static void visitParameterClass(final Map> found, Class } final String name = cls.getName(); - if (!name.startsWith(QueryCompiler.DYNAMIC_GROOVY_CLASS_PREFIX)) { + if (!name.startsWith(QueryCompilerImpl.DYNAMIC_CLASS_PREFIX)) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java index 29e61b54fdb..6ad2eb4d31e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java @@ -3,8 +3,8 @@ // package io.deephaven.engine.table.impl.select.codegen; -import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.CompletionStageFuture; @@ -49,7 +49,7 @@ public static CompletionStageFuture create( .description("FormulaKernel: " + originalFormulaString) .className("Formula") .classBody(classBody) - .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) .build()).thenApply(clazz -> { final FormulaKernelFactory fkf; try { @@ -239,7 +239,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(cookedFormulaString)); - final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(originalFormulaString); + final String joinedFormulaString = QueryCompilerImpl.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", FormulaEvaluationException.class.getCanonicalName()); return g.freeze(); diff --git a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java index 26388fe5546..c31fca971cc 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java @@ -58,7 +58,7 @@ private static void createOrClearDirectory(final File directory) { private final ObjectTypeLookup objectTypeLookup; private final Listener changeListener; - private final File classCacheDirectory; + protected final File classCacheDirectory; private final ScriptSessionQueryScope queryScope; protected final ExecutionContext executionContext; @@ -80,7 +80,7 @@ protected AbstractScriptSession( queryScope = new ScriptSessionQueryScope(); final QueryCompiler compilerContext = - QueryCompiler.create(classCacheDirectory, Thread.currentThread().getContextClassLoader()); + QueryCompilerImpl.create(classCacheDirectory, Thread.currentThread().getContextClassLoader()); executionContext = ExecutionContext.newBuilder() .markSystemic() diff --git a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java index 8a615214e71..9ae494a2057 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java @@ -4,7 +4,7 @@ package io.deephaven.engine.util; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryCompilerRequest; import java.util.*; @@ -58,7 +58,7 @@ public static Supplier compileSimpleFunction(final Class res .description("Simple Function: " + code) .className("Function") .classBody(classBody.toString()) - .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) .build()); try { @@ -82,7 +82,7 @@ public static Class getClassThroughCompilation(final String object) { .description("Formula: return " + object + ".class") .className("Function") .classBody(classBody.toString()) - .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .packageNameRoot(QueryCompilerImpl.FORMULA_CLASS_PREFIX) .build()); try { diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 83a3694c410..51dbdbe0da9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -82,7 +82,7 @@ public class GroovyDeephavenSession extends AbstractScriptSession fullCommand(String command) { } public static byte[] getDynamicClass(String name) { - return readClass(ExecutionContext.getContext().getQueryCompiler().getFakeClassDestination(), name); + return readClass(ExecutionContext.getContext().getQueryCompiler().getTemporaryClassDestination(), name); } private static byte[] readClass(final File rootDirectory, final String className) { @@ -635,7 +635,7 @@ private void updateClassloader(String currentCommand) { final String name = getNextScriptClassName(); CompilerConfiguration config = new CompilerConfiguration(CompilerConfiguration.DEFAULT); - config.setTargetDirectory(executionContext.getQueryCompiler().getFakeClassDestination()); + config.setTargetDirectory(executionContext.getQueryCompiler().getTemporaryClassDestination()); config.getCompilationCustomizers().add(consoleImports); final CompilationUnit cu = new CompilationUnit(config, null, groovyShell.getClassLoader()); cu.addSource(name, currentCommand); @@ -645,7 +645,8 @@ private void updateClassloader(String currentCommand) { } catch (RuntimeException e) { throw new GroovyExceptionWrapper(e); } - final File dynamicClassDestination = ExecutionContext.getContext().getQueryCompiler().getFakeClassDestination(); + final File dynamicClassDestination = + ExecutionContext.getContext().getQueryCompiler().getTemporaryClassDestination(); if (dynamicClassDestination == null) { return; } @@ -672,7 +673,7 @@ && isAnInteger(aClass.getName().substring(SCRIPT_PREFIX.length()))) { } try { - QueryCompiler.writeClass(dynamicClassDestination, entry.getKey(), entry.getValue()); + QueryCompilerImpl.writeClass(classCacheDirectory, entry.getKey(), entry.getValue()); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java b/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java index 206b6c73f16..b700c4ed006 100644 --- a/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java +++ b/engine/table/src/test/java/io/deephaven/engine/updategraph/impl/TestEventDrivenUpdateGraph.java @@ -7,6 +7,7 @@ import io.deephaven.configuration.DataDir; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.TrackingRowSet; @@ -102,7 +103,7 @@ private QueryCompiler compilerForUnitTests() { final Path queryCompilerDir = DataDir.get() .resolve("io.deephaven.engine.updategraph.impl.TestEventDrivenUpdateGraph.compilerForUnitTests"); - return QueryCompiler.create(queryCompilerDir.toFile(), getClass().getClassLoader()); + return QueryCompilerImpl.create(queryCompilerDir.toFile(), getClass().getClassLoader()); } @Test diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java index 39ac93d077e..eb4bd23c3eb 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/context/TestExecutionContext.java @@ -20,7 +20,7 @@ public static ExecutionContext createForUnitTests() { .markSystemic() .newQueryScope() .newQueryLibrary() - .setQueryCompiler(QueryCompiler.createForUnitTests()) + .setQueryCompiler(QueryCompilerImpl.createForUnitTests()) .setUpdateGraph(UPDATE_GRAPH) .setOperationInitializer(OPERATION_INITIALIZATION) .build(); diff --git a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java index a54c3357c55..d689b823ae6 100644 --- a/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java +++ b/engine/test-utils/src/main/java/io/deephaven/engine/testutil/testcase/RefreshingTableTestCase.java @@ -7,7 +7,7 @@ import io.deephaven.chunk.util.pools.ChunkPoolReleaseTracking; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.TestExecutionContext; import io.deephaven.engine.liveness.LivenessScope; import io.deephaven.engine.liveness.LivenessScopeStack; @@ -70,7 +70,7 @@ public void setUp() throws Exception { errors = null; livenessScopeCloseable = LivenessScopeStack.open(new LivenessScope(true), true); - oldLogEnabled = QueryCompiler.setLogEnabled(ENABLE_QUERY_COMPILER_LOGGING); + oldLogEnabled = QueryCompilerImpl.setLogEnabled(ENABLE_QUERY_COMPILER_LOGGING); oldSerialSafe = updateGraph.setSerialTableOperationsSafe(true); AsyncErrorLogger.init(); ChunkPoolReleaseTracking.enableStrict(); @@ -81,7 +81,7 @@ public void tearDown() throws Exception { ChunkPoolReleaseTracking.checkAndDisable(); final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast(); updateGraph.setSerialTableOperationsSafe(oldSerialSafe); - QueryCompiler.setLogEnabled(oldLogEnabled); + QueryCompilerImpl.setLogEnabled(oldLogEnabled); // reset the execution context executionContext.close(); diff --git a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java index 57f01185faa..2fbef5447c8 100644 --- a/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java +++ b/extensions/parquet/benchmark/src/benchmark/java/io/deephaven/benchmark/parquet/table/TableWriteBenchmark.java @@ -5,7 +5,7 @@ import io.deephaven.base.FileUtils; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerImpl; import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.table.Table; @@ -51,7 +51,7 @@ public void setupEnv() throws IOException { .newQueryLibrary() .newQueryScope() .setQueryCompiler( - QueryCompiler.create(rootPath.resolve("cache").toFile(), getClass().getClassLoader())) + QueryCompilerImpl.create(rootPath.resolve("cache").toFile(), getClass().getClassLoader())) .build(); exContextCloseable = context.open(); From dc3612a7ef77702a1e917fba0c3659e3a5d786ee Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Wed, 17 Apr 2024 13:17:42 -0600 Subject: [PATCH 12/19] Remove Serializable; Step Toward #4359 --- .../io/deephaven/plot/colors/ColorMaps.java | 6 +- .../util/functions/ClosureBiFunction.java | 2 +- .../ClosureDoubleBinaryOperator.java | 2 +- .../functions/ClosureDoubleUnaryOperator.java | 2 +- .../plot/util/functions/ClosureFunction.java | 4 +- .../plot/util/functions/HasClosure.java | 35 +++++++ .../util/functions/SerializableClosure.java | 96 ------------------- .../plot/util/functions/TestHasClosure.java | 37 +++++++ .../functions/TestSerializableClosure.java | 55 ----------- 9 files changed, 80 insertions(+), 159 deletions(-) create mode 100644 Plot/src/main/java/io/deephaven/plot/util/functions/HasClosure.java delete mode 100644 Plot/src/main/java/io/deephaven/plot/util/functions/SerializableClosure.java create mode 100644 Plot/src/test/java/io/deephaven/plot/util/functions/TestHasClosure.java delete mode 100644 Plot/src/test/java/io/deephaven/plot/util/functions/TestSerializableClosure.java diff --git a/Plot/src/main/java/io/deephaven/plot/colors/ColorMaps.java b/Plot/src/main/java/io/deephaven/plot/colors/ColorMaps.java index 27c7885a37a..92ae63bde93 100644 --- a/Plot/src/main/java/io/deephaven/plot/colors/ColorMaps.java +++ b/Plot/src/main/java/io/deephaven/plot/colors/ColorMaps.java @@ -6,7 +6,7 @@ import io.deephaven.base.verify.Require; import io.deephaven.gui.color.Color; import io.deephaven.gui.color.Paint; -import io.deephaven.plot.util.functions.SerializableClosure; +import io.deephaven.plot.util.functions.HasClosure; import io.deephaven.plot.util.Range; import groovy.lang.Closure; @@ -294,13 +294,13 @@ public static Function closureMap(final Map for (final Map.Entry, COLOR> e : map.entrySet()) { final Closure closure = e.getKey(); final COLOR color = e.getValue(); - final SerializableClosure serializableClosure = new SerializableClosure<>(closure); + final HasClosure hasClosure = new HasClosure<>(closure); final SerializablePredicate predicate = new SerializablePredicate() { private static final long serialVersionUID = 613420989214281949L; @Override public boolean test(Double aDouble) { - return serializableClosure.getClosure().call(aDouble); + return hasClosure.getClosure().call(aDouble); } }; diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureBiFunction.java b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureBiFunction.java index 15b61bc4c64..1a264d12097 100644 --- a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureBiFunction.java +++ b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureBiFunction.java @@ -8,7 +8,7 @@ /** * Wraps a {@link SerializableBiFunction} with the API of a function.
*/ -public class ClosureBiFunction extends SerializableClosure implements SerializableBiFunction { +public class ClosureBiFunction extends HasClosure implements SerializableBiFunction { private static final long serialVersionUID = 697974379939190730L; /** diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleBinaryOperator.java b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleBinaryOperator.java index 81fe43ddbf7..f0dd895ffc3 100644 --- a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleBinaryOperator.java +++ b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleBinaryOperator.java @@ -10,7 +10,7 @@ /** * A serializable closure which maps pair of doubles to doubles. */ -public class ClosureDoubleBinaryOperator extends SerializableClosure +public class ClosureDoubleBinaryOperator extends HasClosure implements DoubleBinaryOperator { private static final long serialVersionUID = -6533578879266557626L; diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleUnaryOperator.java b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleUnaryOperator.java index 211818dde1a..c8cb4d90919 100644 --- a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleUnaryOperator.java +++ b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureDoubleUnaryOperator.java @@ -10,7 +10,7 @@ /** * A serializable closure which maps doubles to doubles. */ -public class ClosureDoubleUnaryOperator extends SerializableClosure +public class ClosureDoubleUnaryOperator extends HasClosure implements DoubleUnaryOperator { private static final long serialVersionUID = -4092987117189101803L; diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureFunction.java b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureFunction.java index 8994468a8e9..fcc397d3f89 100644 --- a/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureFunction.java +++ b/Plot/src/main/java/io/deephaven/plot/util/functions/ClosureFunction.java @@ -6,9 +6,9 @@ import groovy.lang.Closure; /** - * Wraps a {@link SerializableClosure} with the API of a function. + * Wraps a {@link HasClosure} with the API of a function. */ -public class ClosureFunction extends SerializableClosure implements SerializableFunction { +public class ClosureFunction extends HasClosure implements SerializableFunction { private static final long serialVersionUID = 3693316124178311688L; diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/HasClosure.java b/Plot/src/main/java/io/deephaven/plot/util/functions/HasClosure.java new file mode 100644 index 00000000000..2bc28132f75 --- /dev/null +++ b/Plot/src/main/java/io/deephaven/plot/util/functions/HasClosure.java @@ -0,0 +1,35 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.plot.util.functions; + +import io.deephaven.base.verify.Require; +import groovy.lang.Closure; + +/** + * A serializable closure. + */ +public class HasClosure { + + private final Closure closure; + + /** + * Creates a SerializableClosure instance with the {@code closure}. + * + * @param closure closure + */ + public HasClosure(final Closure closure) { + Require.neqNull(closure, "closure"); + this.closure = closure.dehydrate(); + this.closure.setResolveStrategy(Closure.DELEGATE_ONLY); + } + + /** + * Gets this SerializableClosure's closure. + * + * @return this SerializableClosure's closure + */ + public Closure getClosure() { + return closure; + } +} diff --git a/Plot/src/main/java/io/deephaven/plot/util/functions/SerializableClosure.java b/Plot/src/main/java/io/deephaven/plot/util/functions/SerializableClosure.java deleted file mode 100644 index 79117fba62a..00000000000 --- a/Plot/src/main/java/io/deephaven/plot/util/functions/SerializableClosure.java +++ /dev/null @@ -1,96 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.plot.util.functions; - -import io.deephaven.base.classloaders.MapBackedClassLoader; -import io.deephaven.base.verify.Require; -import io.deephaven.engine.util.GroovyDeephavenSession; -import groovy.lang.Closure; - -import java.io.*; - -/** - * A serializable closure. - */ -public class SerializableClosure implements Serializable { - - private static final long serialVersionUID = -1438975554979636320L; - private Closure closure; - - /** - * Creates a SerializableClosure instance with the {@code closure}. - * - * @param closure closure - */ - public SerializableClosure(final Closure closure) { - Require.neqNull(closure, "closure"); - this.closure = closure.dehydrate(); - this.closure.setResolveStrategy(Closure.DELEGATE_ONLY); - } - - /** - * Gets this SerializableClosure's closure. - * - * @return this SerializableClosure's closure - */ - public Closure getClosure() { - return closure; - } - - private void writeObject(final ObjectOutputStream oos) throws IOException { - final Class c = closure.getClass(); - final String closureClassName = c.getName(); - - final byte[] closureCode = GroovyDeephavenSession.getDynamicClass(closureClassName); - - final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - final ObjectOutputStream objectStream = new ObjectOutputStream(byteStream); - objectStream.writeObject(closure); - final byte[] closureData = byteStream.toByteArray(); - - oos.writeObject(closureClassName); - oos.writeObject(closureCode); - oos.writeObject(closureData); - } - - @SuppressWarnings("unchecked") - private void readObject(final ObjectInputStream ois) throws IOException, ClassNotFoundException { - final String closureClassName = (String) ois.readObject(); - final byte[] closureCode = (byte[]) ois.readObject(); - final byte[] closureData = (byte[]) ois.readObject(); - - final MapBackedClassLoader cl = new MapBackedClassLoader(); - cl.addClassData(closureClassName, closureCode); - final ObjectInputStream objectStream = - new CustomClassLoaderObjectInputStream<>(new ByteArrayInputStream(closureData), cl); - closure = (Closure) objectStream.readObject(); - } - - private static class CustomClassLoaderObjectInputStream extends ObjectInputStream { - - private final CLT classLoader; - - public CustomClassLoaderObjectInputStream(InputStream inputStream, CLT classLoader) throws IOException { - super(inputStream); - this.classLoader = classLoader; - } - - @Override - protected Class resolveClass(ObjectStreamClass desc) throws ClassNotFoundException, IOException { - if (classLoader != null) { - try { - return Class.forName(desc.getName(), false, classLoader); - } catch (ClassNotFoundException cnfe) { - /* - * The default implementation in ObjectInputStream handles primitive types with a map from name to - * class. Rather than duplicate the functionality, we are delegating to the super method for all - * failures that may be of this kind, as well as any case where the passed in ClassLoader fails to - * find the class. - */ - } - } - return super.resolveClass(desc); - } - } -} diff --git a/Plot/src/test/java/io/deephaven/plot/util/functions/TestHasClosure.java b/Plot/src/test/java/io/deephaven/plot/util/functions/TestHasClosure.java new file mode 100644 index 00000000000..0ecfd2413cd --- /dev/null +++ b/Plot/src/test/java/io/deephaven/plot/util/functions/TestHasClosure.java @@ -0,0 +1,37 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.plot.util.functions; + +import io.deephaven.base.testing.BaseArrayTestCase; +import groovy.lang.Closure; + +public class TestHasClosure extends BaseArrayTestCase { + + private final String value = "S"; + + private final Closure closure = new Closure(null) { + @Override + public String call() { + return value; + } + + @Override + public String call(Object... args) { + return value; + } + + @Override + public String call(Object arguments) { + return value; + } + }; + + public void testSerializableClosure() { + HasClosure hasClosure = new ClosureFunction<>(closure); + + assertEquals(value, hasClosure.getClosure().call()); + assertEquals(value, hasClosure.getClosure().call("T")); + assertEquals(value, hasClosure.getClosure().call("A", "B")); + } +} diff --git a/Plot/src/test/java/io/deephaven/plot/util/functions/TestSerializableClosure.java b/Plot/src/test/java/io/deephaven/plot/util/functions/TestSerializableClosure.java deleted file mode 100644 index 07de3ddc68a..00000000000 --- a/Plot/src/test/java/io/deephaven/plot/util/functions/TestSerializableClosure.java +++ /dev/null @@ -1,55 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.plot.util.functions; - -import io.deephaven.base.testing.BaseArrayTestCase; -import groovy.lang.Closure; - -public class TestSerializableClosure extends BaseArrayTestCase { - - private final String value = "S"; - - private final Closure closure = new Closure(null) { - @Override - public String call() { - return value; - } - - @Override - public String call(Object... args) { - return value; - } - - @Override - public String call(Object arguments) { - return value; - } - }; - - public void testSerializableClosure() { - SerializableClosure serializableClosure = new ClosureFunction(closure); - - assertEquals(value, serializableClosure.getClosure().call()); - assertEquals(value, serializableClosure.getClosure().call("T")); - assertEquals(value, serializableClosure.getClosure().call("A", "B")); - - // testing serialization is too hard - /* - * Object copy = null; try { ByteArrayOutputStream bos = new ByteArrayOutputStream(); ObjectOutputStream oos = - * new ObjectOutputStream(bos); oos.writeObject(serializableClosure); oos.flush(); byte[] obis = - * bos.toByteArray(); oos.close(); bos.close(); - * - * ByteArrayInputStream bis = new ByteArrayInputStream(obis); ObjectInputStream ois = new - * ObjectInputStream(bis); copy = ois.readObject(); ois.close(); bis.close(); } catch (IOException | - * ClassNotFoundException e) { e.printStackTrace(); } - * - * serializableClosure = (SerializableClosure) copy; if(serializableClosure == null) { - * TestCase.fail("Null return from serialization"); } - * - * assertEquals(value, serializableClosure.getClosure().call()); - */ - } - - -} From 8af659b8828453c8260a9e567e6d60be0ab0a47d Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Wed, 17 Apr 2024 14:17:12 -0600 Subject: [PATCH 13/19] Refactor GroovyDeephavenSession Construction --- .../engine/context/PoisonedQueryCompiler.java | 12 -- .../engine/context/QueryCompiler.java | 19 --- .../engine/context/QueryCompilerImpl.java | 21 +-- .../engine/util/AbstractScriptSession.java | 32 +++-- .../engine/util/GroovyDeephavenSession.java | 126 ++++++++++-------- .../engine/table/impl/FuzzerTest.java | 2 +- .../scripts/TestGroovyDeephavenSession.java | 4 +- .../groovy/GroovyConsoleSessionModule.java | 2 +- .../server/appmode/ApplicationTest.java | 2 +- 9 files changed, 102 insertions(+), 118 deletions(-) diff --git a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java index fdfb59cda9d..72a78c7173a 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java @@ -7,8 +7,6 @@ import io.deephaven.util.ExecutionContextRegistrationException; import org.jetbrains.annotations.NotNull; -import java.io.File; - public class PoisonedQueryCompiler implements QueryCompiler { public static final PoisonedQueryCompiler INSTANCE = new PoisonedQueryCompiler(); @@ -19,16 +17,6 @@ private T fail() { throw ExecutionContextRegistrationException.onFailedComponentAccess("QueryCompiler"); } - @Override - public File getTemporaryClassDestination() { - return fail(); - } - - @Override - public void setParentClassLoader(@NotNull final ClassLoader parentClassLoader) { - fail(); - } - @Override public void compile( @NotNull final QueryCompilerRequest[] requests, diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 34f0ffea287..d02b2807aa9 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -7,30 +7,11 @@ import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import java.io.File; import java.util.concurrent.ExecutionException; public interface QueryCompiler { - /** - * Deephaven REPL Sessions may create new classes on the fly. This method returns the directory where these classes - * should be written so that they are visible to the compiler. - * - * @return The directory where classes should be written - */ - @Nullable - File getTemporaryClassDestination(); - - /** - * Set the parent class loader for the compiler. If the Deephaven REPL Session can create new classes on the fly, - * that class loader must provide the on-the-fly class objects used by QueryCompiler compiled classes. - * - * @param parentClassLoader The parent class loader - */ - void setParentClassLoader(@NotNull ClassLoader parentClassLoader); - /** * Compile a class. * diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index bbe30625a5b..22106606ffc 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -83,9 +83,8 @@ static QueryCompilerImpl createForUnitTests() { private final String[] dynamicPatterns = new String[] {DYNAMIC_CLASS_PREFIX, FORMULA_CLASS_PREFIX}; private final File classDestination; - private final boolean isCacheDirectory; private final Set additionalClassLocations; - private volatile WritableURLClassLoader ucl; + private final WritableURLClassLoader ucl; private QueryCompilerImpl(File classDestination) { this(classDestination, null, false); @@ -94,12 +93,11 @@ private QueryCompilerImpl(File classDestination) { private QueryCompilerImpl( final File classDestination, final ClassLoader parentClassLoader, - final boolean isCacheDirectory) { + boolean isCacheDirectory) { final ClassLoader parentClassLoaderToUse = parentClassLoader == null ? QueryCompilerImpl.class.getClassLoader() : parentClassLoader; this.classDestination = classDestination; - this.isCacheDirectory = isCacheDirectory; ensureDirectories(this.classDestination, () -> "Failed to create missing class destination directory " + classDestination.getAbsolutePath()); additionalClassLocations = new LinkedHashSet<>(); @@ -188,21 +186,6 @@ public static void writeClass(final File destinationDirectory, final String clas fileOutStream.close(); } - public File getTemporaryClassDestination() { - // Groovy classes need to be written out to a location where they can be found by the compiler - // (so that filters and formulae can use them). - // - // We don't want the regular runtime class loader to find them, because then they get "stuck" in there - // even if the class itself changes, and we can't forget it. So instead we use a single-use class loader - // for each formula, that will always read the class from disk. - return isCacheDirectory ? classDestination : null; - } - - public void setParentClassLoader(@NotNull final ClassLoader parentClassLoader) { - // noinspection NonAtomicOperationOnVolatileField - ucl = new WritableURLClassLoader(ucl.getURLs(), parentClassLoader); - } - /** * Compiles all requests. * diff --git a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java index c31fca971cc..10ba947d128 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java @@ -41,6 +41,14 @@ public abstract class AbstractScriptSession loadClass(String name, boolean resolve) throws ClassNotFoundE private final ScriptFinder scriptFinder; /** Contains imports to be applied to commands run in the console */ - private final ImportCustomizer consoleImports = new ImportCustomizer(); + private final ImportCustomizer consoleImports; /** Contains imports to be applied to .groovy files loaded from the classpath */ - private final ImportCustomizer loadedGroovyScriptImports = new ImportCustomizer(); + private final ImportCustomizer loadedGroovyScriptImports; private final Set dynamicClasses = new HashSet<>(); - private final Map bindingBackingMap = Collections.synchronizedMap(new LinkedHashMap<>()); - private final GroovyShell groovyShell; + private final Map bindingBackingMap; - private int counter; - private String script = "Script"; + private static class DeephavenGroovyShell extends GroovyShell { + private int counter; + private String script = "Script"; - private String getNextScriptClassName() { - return script + "_" + (counter + 1); + DeephavenGroovyShell( + final GroovyClassLoader loader, + final Binding binding, + final CompilerConfiguration config) { + super(loader, binding, config); + } + + @Override + protected synchronized String generateScriptName() { + return script + "_" + (++counter) + ".groovy"; + } + + private String getNextScriptClassName() { + return script + "_" + (counter + 1); + } } - public GroovyDeephavenSession( + private final DeephavenGroovyShell groovyShell; + + public static GroovyDeephavenSession of( final UpdateGraph updateGraph, final OperationInitializer operationInitializer, final ObjectTypeLookup objectTypeLookup, final RunScripts runScripts) throws IOException { - this(updateGraph, operationInitializer, objectTypeLookup, null, runScripts); + return GroovyDeephavenSession.of(updateGraph, operationInitializer, objectTypeLookup, null, runScripts); } - public GroovyDeephavenSession( + public static GroovyDeephavenSession of( final UpdateGraph updateGraph, final OperationInitializer operationInitializer, ObjectTypeLookup objectTypeLookup, @Nullable final Listener changeListener, - final RunScripts runScripts) - throws IOException { - super(updateGraph, operationInitializer, objectTypeLookup, changeListener); + final RunScripts runScripts) throws IOException { + + final ImportCustomizer consoleImports = new ImportCustomizer(); + final ImportCustomizer loadedGroovyScriptImports = new ImportCustomizer(); addDefaultImports(consoleImports); if (INCLUDE_DEFAULT_IMPORTS_IN_LOADED_GROOVY) { - addDefaultImports(this.loadedGroovyScriptImports); + addDefaultImports(loadedGroovyScriptImports); } + final File classCacheDirectory = AbstractScriptSession.newClassCacheLocation(); + // Specify a classloader to read from the classpath, with script imports CompilerConfiguration scriptConfig = new CompilerConfiguration(); scriptConfig.getCompilationCustomizers().add(loadedGroovyScriptImports); - scriptConfig.setTargetDirectory(executionContext.getQueryCompiler().getTemporaryClassDestination()); + scriptConfig.setTargetDirectory(classCacheDirectory); GroovyClassLoader scriptClassLoader = new GroovyClassLoader(STATIC_LOADER, scriptConfig); // Specify a configuration for compiling/running console commands for custom imports CompilerConfiguration consoleConfig = new CompilerConfiguration(); consoleConfig.getCompilationCustomizers().add(consoleImports); - consoleConfig.setTargetDirectory(executionContext.getQueryCompiler().getTemporaryClassDestination()); + consoleConfig.setTargetDirectory(classCacheDirectory); + Map bindingBackingMap = Collections.synchronizedMap(new LinkedHashMap<>()); Binding binding = new Binding(bindingBackingMap); - groovyShell = new GroovyShell(scriptClassLoader, binding, consoleConfig) { - protected synchronized String generateScriptName() { - return GroovyDeephavenSession.this.generateScriptName(); - } - }; + DeephavenGroovyShell groovyShell = new DeephavenGroovyShell(scriptClassLoader, binding, consoleConfig); + + + return new GroovyDeephavenSession(updateGraph, operationInitializer, objectTypeLookup, changeListener, + runScripts, classCacheDirectory, consoleImports, loadedGroovyScriptImports, bindingBackingMap, + groovyShell); + } + + private GroovyDeephavenSession( + final UpdateGraph updateGraph, + final OperationInitializer operationInitializer, + ObjectTypeLookup objectTypeLookup, + @Nullable final Listener changeListener, + final RunScripts runScripts, + final File classCacheDirectory, + final ImportCustomizer consoleImports, + final ImportCustomizer loadedGroovyScriptImports, + final Map bindingBackingMap, + final DeephavenGroovyShell groovyShell) + throws IOException { + super(updateGraph, operationInitializer, objectTypeLookup, changeListener, classCacheDirectory, + groovyShell.getClassLoader()); + + this.consoleImports = consoleImports; + this.loadedGroovyScriptImports = loadedGroovyScriptImports; + this.bindingBackingMap = bindingBackingMap; + this.groovyShell = groovyShell; this.scriptFinder = new ScriptFinder(DEFAULT_SCRIPT_PATH); groovyShell.setVariable("__groovySession", this); groovyShell.setVariable("DB_SCRIPT_PATH", DEFAULT_SCRIPT_PATH); - executionContext.getQueryCompiler().setParentClassLoader(getShell().getClassLoader()); - publishInitial(); for (final String path : runScripts.paths) { @@ -199,7 +235,7 @@ protected synchronized String generateScriptName() { /** * Adds the default imports that Groovy users assume to be present. */ - private void addDefaultImports(ImportCustomizer imports) { + private static void addDefaultImports(ImportCustomizer imports) { // TODO (core#230): Remove large list of manual text-based consoleImports // NOTE: Don't add to this list without a compelling reason!!! Use the user script import if possible. imports.addImports( @@ -244,10 +280,6 @@ private void addDefaultImports(ImportCustomizer imports) { StaticCalendarMethods.class.getName()); } - private String generateScriptName() { - return script + "_" + (++counter) + ".groovy"; - } - public static InputStream findScript(String relativePath) throws IOException { return new ScriptFinder(DEFAULT_SCRIPT_PATH).findScript(relativePath); } @@ -299,13 +331,13 @@ protected void evaluate(String command, String scriptName) { final String lastCommand = fc.second; final String commandPrefix = fc.first; - final String oldScriptName = script; + final String oldScriptName = groovyShell.script; try { if (scriptName != null) { - script = scriptName.replaceAll("[^0-9A-Za-z_]", "_").replaceAll("(^[0-9])", "_$1"); + groovyShell.script = scriptName.replaceAll("[^0-9A-Za-z_]", "_").replaceAll("(^[0-9])", "_$1"); } - final String currentScriptName = script; + final String currentScriptName = groovyShell.script; updateClassloader(lastCommand); @@ -319,7 +351,7 @@ protected void evaluate(String command, String scriptName) { throw wrapAndRewriteStackTrace(scriptName, currentScriptName, e, lastCommand, commandPrefix); } } finally { - script = oldScriptName; + groovyShell.script = oldScriptName; } } @@ -617,25 +649,11 @@ private Pair fullCommand(String command) { + "\n\n// this final true prevents Groovy from interpreting a trailing class definition as something to execute\n;\ntrue;\n"); } - public static byte[] getDynamicClass(String name) { - return readClass(ExecutionContext.getContext().getQueryCompiler().getTemporaryClassDestination(), name); - } - - private static byte[] readClass(final File rootDirectory, final String className) { - final String resourceName = className.replace('.', '/') + JavaFileObject.Kind.CLASS.extension; - final Path path = new File(rootDirectory, resourceName).toPath(); - try { - return Files.readAllBytes(path); - } catch (IOException e) { - throw new RuntimeException("Error reading path " + path + " for className " + className, e); - } - } - private void updateClassloader(String currentCommand) { - final String name = getNextScriptClassName(); + final String name = groovyShell.getNextScriptClassName(); CompilerConfiguration config = new CompilerConfiguration(CompilerConfiguration.DEFAULT); - config.setTargetDirectory(executionContext.getQueryCompiler().getTemporaryClassDestination()); + config.setTargetDirectory(classCacheDirectory); config.getCompilationCustomizers().add(consoleImports); final CompilationUnit cu = new CompilationUnit(config, null, groovyShell.getClassLoader()); cu.addSource(name, currentCommand); @@ -645,9 +663,7 @@ private void updateClassloader(String currentCommand) { } catch (RuntimeException e) { throw new GroovyExceptionWrapper(e); } - final File dynamicClassDestination = - ExecutionContext.getContext().getQueryCompiler().getTemporaryClassDestination(); - if (dynamicClassDestination == null) { + if (classCacheDirectory == null) { return; } final List classes = cu.getClasses(); diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/FuzzerTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/FuzzerTest.java index b5327549923..e673093a736 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/FuzzerTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/FuzzerTest.java @@ -76,7 +76,7 @@ private GroovyDeephavenSession getGroovySession() throws IOException { } private GroovyDeephavenSession getGroovySession(@Nullable Clock clock) throws IOException { - final GroovyDeephavenSession session = new GroovyDeephavenSession( + final GroovyDeephavenSession session = GroovyDeephavenSession.of( ExecutionContext.getContext().getUpdateGraph(), ExecutionContext.getContext().getOperationInitializer(), NoOp.INSTANCE, diff --git a/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java b/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java index 365c4505eb7..ff5e4b3cbaf 100644 --- a/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java +++ b/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java @@ -50,8 +50,8 @@ public void setup() throws IOException { livenessScope = new LivenessScope(); LivenessScopeStack.push(livenessScope); final ExecutionContext context = ExecutionContext.getContext(); - session = new GroovyDeephavenSession( - context.getUpdateGraph(), context.getOperationInitializer(), NoOp.INSTANCE, null, + session = GroovyDeephavenSession.of( + context.getUpdateGraph(), context.getOperationInitializer(), NoOp.INSTANCE, GroovyDeephavenSession.RunScripts.none()); executionContext = session.getExecutionContext().open(); } diff --git a/server/src/main/java/io/deephaven/server/console/groovy/GroovyConsoleSessionModule.java b/server/src/main/java/io/deephaven/server/console/groovy/GroovyConsoleSessionModule.java index 8497050d449..0adcd9400c2 100644 --- a/server/src/main/java/io/deephaven/server/console/groovy/GroovyConsoleSessionModule.java +++ b/server/src/main/java/io/deephaven/server/console/groovy/GroovyConsoleSessionModule.java @@ -36,7 +36,7 @@ GroovyDeephavenSession bindGroovySession( final ScriptSession.Listener listener, final RunScripts runScripts) { try { - return new GroovyDeephavenSession(updateGraph, operationInitializer, lookup, listener, runScripts); + return GroovyDeephavenSession.of(updateGraph, operationInitializer, lookup, listener, runScripts); } catch (final IOException e) { throw new UncheckedIOException(e); } diff --git a/server/src/test/java/io/deephaven/server/appmode/ApplicationTest.java b/server/src/test/java/io/deephaven/server/appmode/ApplicationTest.java index dbfd1a22029..3870fb17124 100644 --- a/server/src/test/java/io/deephaven/server/appmode/ApplicationTest.java +++ b/server/src/test/java/io/deephaven/server/appmode/ApplicationTest.java @@ -50,7 +50,7 @@ public void app00() { @Test public void app01() throws IOException { - session = new GroovyDeephavenSession( + session = GroovyDeephavenSession.of( ExecutionContext.getContext().getUpdateGraph(), ExecutionContext.getContext().getOperationInitializer(), NoOp.INSTANCE, null, From 5d50f961675d2715aa13ddd84156c5506a00846e Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Wed, 17 Apr 2024 15:06:25 -0600 Subject: [PATCH 14/19] Use JobScheduler --- .../engine/context/QueryCompilerImpl.java | 94 ++++++++++++------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index 22106606ffc..1ddd745331c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -10,7 +10,10 @@ import io.deephaven.configuration.DataDir; import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.util.SynchronizedJavaFileManager; -import io.deephaven.engine.updategraph.OperationInitializer; +import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; +import io.deephaven.engine.table.impl.util.ImmediateJobScheduler; +import io.deephaven.engine.table.impl.util.JobScheduler; +import io.deephaven.engine.table.impl.util.OperationInitializerJobScheduler; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.util.ByteUtils; @@ -35,6 +38,8 @@ import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.jar.Attributes; import java.util.jar.Manifest; @@ -792,38 +797,26 @@ private void maybeCreateClasses( final JavaFileManager fileManager = new SynchronizedJavaFileManager( compiler.getStandardFileManager(null, null, null)); - boolean exceptionCaught = false; - try { - final OperationInitializer operationInitializer = ExecutionContext.getContext().getOperationInitializer(); - int parallelismFactor = operationInitializer.parallelismFactor(); + final ExecutionContext executionContext = ExecutionContext.getContext(); + final int parallelismFactor = executionContext.getOperationInitializer().parallelismFactor(); - int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); - if (parallelismFactor == 1 || requestsPerTask >= requests.size()) { - doCreateClasses(compiler, fileManager, requests, rootPathAsString, tempDirAsString, - 0, requests.size()); - } else { - int numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; - final Future[] tasks = new Future[numTasks]; - for (int jobId = 0; jobId < numTasks; ++jobId) { - final int startInclusive = jobId * requestsPerTask; - final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); - tasks[jobId] = operationInitializer.submit(() -> { - doCreateClasses(compiler, fileManager, requests, rootPathAsString, tempDirAsString, - startInclusive, endExclusive); - }); - } - for (int jobId = 0; jobId < numTasks; ++jobId) { - try { - tasks[jobId].get(); - } catch (Exception err) { - throw new UncheckedDeephavenException("Exception waiting for compilation task", err); - } - } - } - } catch (final Throwable t) { - exceptionCaught = true; - throw t; - } finally { + final int requestsPerTask = Math.max(32, (requests.size() + parallelismFactor - 1) / parallelismFactor); + + final int numTasks; + final JobScheduler jobScheduler; + + final boolean canParallelize = executionContext.getOperationInitializer().canParallelize(); + if (!canParallelize || parallelismFactor == 1 || requestsPerTask >= requests.size()) { + numTasks = 1; + jobScheduler = new ImmediateJobScheduler(); + } else { + numTasks = (requests.size() + requestsPerTask - 1) / requestsPerTask; + jobScheduler = new OperationInitializerJobScheduler(); + } + + final AtomicReference exception = new AtomicReference<>(); + final CountDownLatch latch = new CountDownLatch(1); + final Runnable cleanup = () -> { try { FileUtils.deleteRecursively(new File(tempDirAsString)); } catch (Exception e) { @@ -833,11 +826,40 @@ private void maybeCreateClasses( try { fileManager.close(); } catch (IOException ioe) { - if (!exceptionCaught) { - // noinspection ThrowFromFinallyBlock - throw new UncheckedIOException("Could not close JavaFileManager", ioe); - } + exception.compareAndSet(null, + new UncheckedIOException("Could not close JavaFileManager", ioe)); + } + + latch.countDown(); + }; + + final Consumer onError = err -> { + if (err instanceof RuntimeException) { + exception.set((RuntimeException) err); + } else { + exception.set(new UncheckedDeephavenException("Error during compilation", err)); + } + cleanup.run(); + }; + + jobScheduler.iterateParallel(executionContext, null, JobScheduler.DEFAULT_CONTEXT_FACTORY, + 0, numTasks, (context, jobId, nestedErrorConsumer) -> { + final int startInclusive = jobId * requestsPerTask; + final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); + doCreateClasses(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + startInclusive, endExclusive); + }, cleanup, onError); + + try { + latch.await(); + QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate( + jobScheduler.getAccumulatedPerformance()); + final RuntimeException err = exception.get(); + if (err != null) { + throw err; } + } catch (final InterruptedException e) { + throw new CancellationException("interrupted while compiling"); } } From dc5a900cd30155537ab0e334645cddcc2d52d658 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Wed, 17 Apr 2024 15:57:24 -0600 Subject: [PATCH 15/19] Share JFM across all requests --- .../engine/context/QueryCompilerImpl.java | 47 ++++++++----------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index 1ddd745331c..e837f47d8f4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -10,6 +10,7 @@ import io.deephaven.configuration.DataDir; import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.util.SynchronizedJavaFileManager; +import io.deephaven.engine.table.impl.perf.BasePerformanceEntry; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.util.ImmediateJobScheduler; import io.deephaven.engine.table.impl.util.JobScheduler; @@ -758,6 +759,16 @@ public JavaSourceFromString makeSource() { } } + private static final JavaCompiler compiler; + private static final JavaFileManager fileManager; + static { + compiler = ToolProvider.getSystemJavaCompiler(); + if (compiler == null) { + throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); + } + fileManager = new SynchronizedJavaFileManager(compiler.getStandardFileManager(null, null, null)); + } + private void maybeCreateClasses( @NotNull final List requests) { // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package @@ -788,15 +799,6 @@ private void maybeCreateClasses( return; } - - final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); - } - - final JavaFileManager fileManager = new SynchronizedJavaFileManager( - compiler.getStandardFileManager(null, null, null)); - final ExecutionContext executionContext = ExecutionContext.getContext(); final int parallelismFactor = executionContext.getOperationInitializer().parallelismFactor(); @@ -823,13 +825,6 @@ private void maybeCreateClasses( // ignore errors here } - try { - fileManager.close(); - } catch (IOException ioe) { - exception.compareAndSet(null, - new UncheckedIOException("Could not close JavaFileManager", ioe)); - } - latch.countDown(); }; @@ -846,14 +841,15 @@ private void maybeCreateClasses( 0, numTasks, (context, jobId, nestedErrorConsumer) -> { final int startInclusive = jobId * requestsPerTask; final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); - doCreateClasses(compiler, fileManager, requests, rootPathAsString, tempDirAsString, - startInclusive, endExclusive); + doCreateClasses(requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive); }, cleanup, onError); try { latch.await(); - QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate( - jobScheduler.getAccumulatedPerformance()); + final BasePerformanceEntry perfEntry = jobScheduler.getAccumulatedPerformance(); + if (perfEntry != null) { + QueryPerformanceRecorder.getInstance().getEnclosingNugget().accumulate(perfEntry); + } final RuntimeException err = exception.get(); if (err != null) { throw err; @@ -864,8 +860,6 @@ private void maybeCreateClasses( } private void doCreateClasses( - @NotNull final JavaCompiler compiler, - @NotNull final JavaFileManager fileManager, @NotNull final List requests, @NotNull final String rootPathAsString, @NotNull final String tempDirAsString, @@ -875,15 +869,14 @@ private void doCreateClasses( // If any of our requests fail to compile then the JavaCompiler will not write any class files at all. The // non-failing requests will be retried in a second pass that is expected to succeed. This enables us to // fulfill futures independent of each other; otherwise a single failure would taint all requests in a batch. - final boolean wantRetry = doCreateClassesSingleRound(compiler, fileManager, requests, rootPathAsString, - tempDirAsString, startInclusive, endExclusive, toRetry); + final boolean wantRetry = doCreateClassesSingleRound(requests, rootPathAsString, tempDirAsString, + startInclusive, endExclusive, toRetry); if (!wantRetry) { return; } final List ignored = new ArrayList<>(); - if (doCreateClassesSingleRound(compiler, fileManager, toRetry, rootPathAsString, tempDirAsString, 0, - toRetry.size(), ignored)) { + if (doCreateClassesSingleRound(toRetry, rootPathAsString, tempDirAsString, 0, toRetry.size(), ignored)) { // We only retried compilation units that did not fail on the first pass, so we should not have any failures // on the second pass. throw new IllegalStateException("Unexpected failure during second pass of compilation"); @@ -891,8 +884,6 @@ private void doCreateClasses( } private boolean doCreateClassesSingleRound( - @NotNull final JavaCompiler compiler, - @NotNull final JavaFileManager fileManager, @NotNull final List requests, @NotNull final String rootPathAsString, @NotNull final String tempDirAsString, From 97fd3d9c8a6d3155e2197f18770ace042f994888 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 23 Apr 2024 00:28:10 -0600 Subject: [PATCH 16/19] Latest Round w/Shared JFM bugfix --- .../engine/context/QueryCompiler.java | 2 +- .../engine/context/QueryCompilerImpl.java | 147 ++++++++++++------ .../table/impl/PartitionAwareSourceTable.java | 17 +- .../impl/QueryCompilerRequestProcessor.java | 2 + .../table/impl/by/AggregationProcessor.java | 21 +-- .../table/impl/lang/QueryLanguageParser.java | 31 ++-- .../impl/select/AbstractConditionFilter.java | 7 +- .../table/impl/select/DhFormulaColumn.java | 10 +- .../impl/select/ReinterpretedColumn.java | 2 +- .../table/impl/select/SelectColumn.java | 3 +- .../analyzers/SelectAndViewAnalyzer.java | 2 +- .../impl/select/codegen/FormulaAnalyzer.java | 19 ++- .../engine/util/AbstractScriptSession.java | 3 +- .../engine/util/GroovyDeephavenSession.java | 48 +++--- .../impl/lang/TestQueryLanguageParser.java | 3 +- 15 files changed, 190 insertions(+), 127 deletions(-) diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index d02b2807aa9..c198d28ebf7 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -13,7 +13,7 @@ public interface QueryCompiler { /** - * Compile a class. + * Compile a class and wait for completion. * * @param request The compilation request */ diff --git a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java index e837f47d8f4..535b8acc573 100644 --- a/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/context/QueryCompilerImpl.java @@ -5,6 +5,8 @@ import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.FileUtils; +import io.deephaven.base.log.LogOutput; +import io.deephaven.base.log.LogOutputAppendable; import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; import io.deephaven.configuration.DataDir; @@ -16,6 +18,7 @@ import io.deephaven.engine.table.impl.util.JobScheduler; import io.deephaven.engine.table.impl.util.OperationInitializerJobScheduler; import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.io.logger.Logger; import io.deephaven.util.ByteUtils; import io.deephaven.util.CompletionStageFuture; @@ -39,6 +42,7 @@ import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Supplier; @@ -47,7 +51,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -public class QueryCompilerImpl implements QueryCompiler { +public class QueryCompilerImpl implements QueryCompiler, LogOutputAppendable { private static final Logger log = LoggerFactory.getLogger(QueryCompilerImpl.class); /** @@ -71,6 +75,41 @@ public class QueryCompilerImpl implements QueryCompiler { private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); + private static JavaCompiler compiler; + private static final AtomicReference fileManagerCache = new AtomicReference<>(); + + private static void ensureJavaCompiler() { + synchronized (QueryCompilerImpl.class) { + if (compiler == null) { + compiler = ToolProvider.getSystemJavaCompiler(); + if (compiler == null) { + throw new UncheckedDeephavenException( + "No Java compiler provided - are you using a JRE instead of a JDK?"); + } + } + } + } + + private static JavaFileManager acquireFileManager() { + JavaFileManager fileManager = fileManagerCache.getAndSet(null); + if (fileManager == null) { + fileManager = new SynchronizedJavaFileManager(compiler.getStandardFileManager(null, null, null)); + } + return fileManager; + } + + private static void releaseFileManager(@NotNull final JavaFileManager fileManager) { + // Reusing the file manager saves a lot of the time in the compilation process. However, we need to be careful + // to avoid keeping too many file handles open so we'll limit ourselves to just one outstanding file manager. + if (!fileManagerCache.compareAndSet(null, fileManager)) { + try { + fileManager.close(); + } catch (final IOException err) { + throw new UncheckedIOException("Could not close JavaFileManager", err); + } + } + } + public static final String FORMULA_CLASS_PREFIX = "io.deephaven.temp"; public static final String DYNAMIC_CLASS_PREFIX = "io.deephaven.dynamic"; @@ -81,7 +120,7 @@ public static QueryCompilerImpl create(File cacheDirectory, ClassLoader classLoa static QueryCompilerImpl createForUnitTests() { final Path queryCompilerDir = DataDir.get() .resolve("io.deephaven.engine.context.QueryCompiler.createForUnitTests"); - return new QueryCompilerImpl(queryCompilerDir.toFile()); + return new QueryCompilerImpl(queryCompilerDir.toFile(), QueryCompilerImpl.class.getClassLoader(), false); } private final Map>> knownClasses = new HashMap<>(); @@ -92,17 +131,12 @@ static QueryCompilerImpl createForUnitTests() { private final Set additionalClassLocations; private final WritableURLClassLoader ucl; - private QueryCompilerImpl(File classDestination) { - this(classDestination, null, false); - } - private QueryCompilerImpl( - final File classDestination, - final ClassLoader parentClassLoader, - boolean isCacheDirectory) { - final ClassLoader parentClassLoaderToUse = parentClassLoader == null - ? QueryCompilerImpl.class.getClassLoader() - : parentClassLoader; + @NotNull final File classDestination, + @NotNull final ClassLoader parentClassLoader, + boolean classDestinationIsAlsoClassSource) { + ensureJavaCompiler(); + this.classDestination = classDestination; ensureDirectories(this.classDestination, () -> "Failed to create missing class destination directory " + classDestination.getAbsolutePath()); @@ -114,13 +148,24 @@ private QueryCompilerImpl( } catch (MalformedURLException e) { throw new UncheckedDeephavenException(e); } - this.ucl = new WritableURLClassLoader(urls, parentClassLoaderToUse); + this.ucl = new WritableURLClassLoader(urls, parentClassLoader); - if (isCacheDirectory) { + if (classDestinationIsAlsoClassSource) { addClassSource(classDestination); } } + @Override + public LogOutput append(LogOutput logOutput) { + return logOutput.append("QueryCompiler{classDestination=").append(classDestination.getAbsolutePath()) + .append("}"); + } + + @Override + public String toString() { + return new LogOutputStringImpl().append(this).toString(); + } + /** * Enables or disables compilation logging. * @@ -192,12 +237,7 @@ public static void writeClass(final File destinationDirectory, final String clas fileOutStream.close(); } - /** - * Compiles all requests. - * - * @param requests The compilation requests; these must be independent of each other - * @param resolvers The resolvers to use for delivering compilation results - */ + @Override public void compile( @NotNull final QueryCompilerRequest[] requests, @NotNull final CompletionStageFuture.Resolver>[] resolvers) { @@ -616,6 +656,14 @@ private static String loadIdentifyingField(Class c) { } } + private static String makeFinalCode(String className, String classBody, String packageName) { + final String joinedEscapedBody = createEscapedJoinedString(classBody); + classBody = classBody.replaceAll("\\$CLASSNAME\\$", className); + classBody = classBody.substring(0, classBody.lastIndexOf("}")); + classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; + return "package " + packageName + ";\n" + classBody; + } + /** * Transform a string into the corresponding Java source code that compiles into that string. This involves escaping * special characters, surrounding it with quotes, and (if the string is larger than the max string length for Java @@ -678,14 +726,6 @@ private static int calcBytesConsumed(final char ch) { return 3; } - private static String makeFinalCode(String className, String classBody, String packageName) { - final String joinedEscapedBody = createEscapedJoinedString(classBody); - classBody = classBody.replaceAll("\\$CLASSNAME\\$", className); - classBody = classBody.substring(0, classBody.lastIndexOf("}")); - classBody += " public static String " + IDENTIFYING_FIELD_NAME + " = " + joinedEscapedBody + ";\n}"; - return "package " + packageName + ";\n" + classBody; - } - private static class JavaSourceFromString extends SimpleJavaFileObject { final String description; final String code; @@ -759,16 +799,6 @@ public JavaSourceFromString makeSource() { } } - private static final JavaCompiler compiler; - private static final JavaFileManager fileManager; - static { - compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); - } - fileManager = new SynchronizedJavaFileManager(compiler.getStandardFileManager(null, null, null)); - } - private void maybeCreateClasses( @NotNull final List requests) { // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package @@ -816,16 +846,30 @@ private void maybeCreateClasses( jobScheduler = new OperationInitializerJobScheduler(); } + final AtomicBoolean cleanupAlreadyRun = new AtomicBoolean(); + final JavaFileManager fileManager = acquireFileManager(); final AtomicReference exception = new AtomicReference<>(); final CountDownLatch latch = new CountDownLatch(1); final Runnable cleanup = () -> { - try { - FileUtils.deleteRecursively(new File(tempDirAsString)); - } catch (Exception e) { - // ignore errors here + if (!cleanupAlreadyRun.compareAndSet(false, true)) { + // onError could be run after cleanup if cleanup throws an exception + return; } - latch.countDown(); + try { + try { + FileUtils.deleteRecursively(new File(tempDirAsString)); + } catch (Exception e) { + // ignore errors here + } + try { + releaseFileManager(fileManager); + } catch (Exception e) { + // ignore errors here + } + } finally { + latch.countDown(); + } }; final Consumer onError = err -> { @@ -841,7 +885,8 @@ private void maybeCreateClasses( 0, numTasks, (context, jobId, nestedErrorConsumer) -> { final int startInclusive = jobId * requestsPerTask; final int endExclusive = Math.min(requests.size(), (jobId + 1) * requestsPerTask); - doCreateClasses(requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive); + doCreateClasses( + fileManager, requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive); }, cleanup, onError); try { @@ -860,6 +905,7 @@ private void maybeCreateClasses( } private void doCreateClasses( + @NotNull final JavaFileManager fileManager, @NotNull final List requests, @NotNull final String rootPathAsString, @NotNull final String tempDirAsString, @@ -869,14 +915,15 @@ private void doCreateClasses( // If any of our requests fail to compile then the JavaCompiler will not write any class files at all. The // non-failing requests will be retried in a second pass that is expected to succeed. This enables us to // fulfill futures independent of each other; otherwise a single failure would taint all requests in a batch. - final boolean wantRetry = doCreateClassesSingleRound(requests, rootPathAsString, tempDirAsString, + final boolean wantRetry = doCreateClassesSingleRound(fileManager, requests, rootPathAsString, tempDirAsString, startInclusive, endExclusive, toRetry); if (!wantRetry) { return; } final List ignored = new ArrayList<>(); - if (doCreateClassesSingleRound(toRetry, rootPathAsString, tempDirAsString, 0, toRetry.size(), ignored)) { + if (doCreateClassesSingleRound(fileManager, toRetry, rootPathAsString, tempDirAsString, 0, toRetry.size(), + ignored)) { // We only retried compilation units that did not fail on the first pass, so we should not have any failures // on the second pass. throw new IllegalStateException("Unexpected failure during second pass of compilation"); @@ -884,6 +931,7 @@ private void doCreateClasses( } private boolean doCreateClassesSingleRound( + @NotNull final JavaFileManager fileManager, @NotNull final List requests, @NotNull final String rootPathAsString, @NotNull final String tempDirAsString, @@ -996,8 +1044,8 @@ private static String getJavaClassPath() { } // IntelliJ will bundle a very large class path into an empty jar with a Manifest that will define the full - // class path - // Look for this being used during compile time, so the full class path can be sent into the compile call + // class path. Look for this being used during compile time, so the full class path can be sent into the compile + // call. final String intellijClassPathJarRegex = ".*classpath[0-9]*\\.jar.*"; if (javaClasspath.matches(intellijClassPathJarRegex)) { try { @@ -1014,8 +1062,7 @@ private static String getJavaClassPath() { final String extendedClassPath = (String) attributes.get(classPathAttribute); if (extendedClassPath != null) { // Parses the files in the manifest description an changes their format to drop the "file:/" - // and - // use the default path separator + // and use the default path separator final String filePaths = Stream.of(extendedClassPath.split("file:/")) .map(String::trim) .filter(fileName -> !fileName.isEmpty()) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java index 2b151f0ccd6..0dd5e9e1627 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java @@ -139,13 +139,13 @@ public Table selectDistinctInternal(Collection columns) { return null; } - final Set newColumns = new HashSet<>(); + final Set partitioningDerivedColumnNames = new HashSet<>(); for (final SelectColumn selectColumn : selectColumns) { if (!((PartitionAwareSourceTable) table).isValidAgainstColumnPartitionTable( - selectColumn.getColumns(), selectColumn.getColumnArrays(), newColumns)) { + selectColumn.getColumns(), selectColumn.getColumnArrays(), partitioningDerivedColumnNames)) { return null; } - newColumns.add(selectColumn.getName()); + partitioningDerivedColumnNames.add(selectColumn.getName()); } return table.selectDistinct(selectColumns); } @@ -290,15 +290,15 @@ public final Table selectDistinct(@NotNull final Collection newColumns = new HashSet<>(); + final Set partitioningDerivedColumnNames = new HashSet<>(); for (final SelectColumn selectColumn : selectColumns) { if (!isValidAgainstColumnPartitionTable( - selectColumn.getColumns(), selectColumn.getColumnArrays(), newColumns)) { + selectColumn.getColumns(), selectColumn.getColumnArrays(), partitioningDerivedColumnNames)) { // Be sure to invoke the super-class version of this method, rather than the array-based one that // delegates to this method. return super.selectDistinct(selectColumns); } - newColumns.add(selectColumn.getName()); + partitioningDerivedColumnNames.add(selectColumn.getName()); } // Ensure that the location table is available and populated with non-null, non-empty locations. @@ -317,11 +317,12 @@ private boolean isValidAgainstColumnPartitionTable( private boolean isValidAgainstColumnPartitionTable( @NotNull final Collection columnNames, @NotNull final Collection columnArrayNames, - @NotNull final Collection newColumns) { + @NotNull final Collection partitioningDerivedColumnNames) { if (!columnArrayNames.isEmpty()) { return false; } return columnNames.stream().allMatch( - columnName -> partitioningColumnDefinitions.containsKey(columnName) || newColumns.contains(columnName)); + columnName -> partitioningColumnDefinitions.containsKey(columnName) + || partitioningDerivedColumnNames.contains(columnName)); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java index 4244d076df7..ae321271be4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java @@ -15,6 +15,7 @@ import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.datastructures.CachingSupplier; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.Collections; @@ -43,6 +44,7 @@ static QueryCompilerRequestProcessor.BatchProcessor batch() { /** * @return a CachingSupplier that supplies a snapshot of the current query scope variables */ + @VisibleForTesting static CachingSupplier> newQueryScopeVariableSupplier() { final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); return new CachingSupplier<>(() -> Collections.unmodifiableMap( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java index 1e2582cef24..7bf2a38008f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java @@ -279,12 +279,7 @@ public AggregationContext makeAggregationContext( @NotNull final String... groupByColumnNames) { switch (type) { case NORMAL: - final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = - QueryCompilerRequestProcessor.batch(); - final AggregationContext aggContext = new NormalConverter( - table, requireStateChangeRecorder, compilationProcessor, groupByColumnNames).build(); - compilationProcessor.compile(); - return aggContext; + return new NormalConverter(table, requireStateChangeRecorder, groupByColumnNames).build(); case ROLLUP_BASE: return new RollupBaseConverter(table, requireStateChangeRecorder, groupByColumnNames).build(); case ROLLUP_REAGGREGATED: @@ -340,7 +335,7 @@ private Converter( isBlink = this.table.isBlink(); } - final AggregationContext build() { + AggregationContext build() { walkAllAggregations(); transformers.add(new RowLookupAttributeSetter()); return makeAggregationContext(); @@ -666,15 +661,21 @@ final void addWeightedAvgOrSumOperator( * {@link AggregationContext} for standard aggregations. Accumulates state by visiting each aggregation. */ private final class NormalConverter extends Converter { - private final QueryCompilerRequestProcessor compilationProcessor; + private final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor; private NormalConverter( @NotNull final Table table, final boolean requireStateChangeRecorder, - @NotNull final QueryCompilerRequestProcessor compilationProcessor, @NotNull final String... groupByColumnNames) { super(table, requireStateChangeRecorder, groupByColumnNames); - this.compilationProcessor = compilationProcessor; + this.compilationProcessor = QueryCompilerRequestProcessor.batch(); + } + + @Override + AggregationContext build() { + final AggregationContext resultContext = super.build(); + compilationProcessor.compile(); + return resultContext; } // ------------------------------------------------------------------------------------------------------------- diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java index ae037b986d6..1fd22940cc1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java @@ -94,6 +94,7 @@ import io.deephaven.engine.util.PyCallableWrapperJpyImpl; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; +import io.deephaven.time.TimeLiteralReplacedExpression; import io.deephaven.util.annotations.TestUseOnly; import io.deephaven.util.type.TypeUtils; import io.deephaven.vector.ByteVector; @@ -199,7 +200,7 @@ public final class QueryLanguageParser extends GenericVisitorAdapter, Q Map> variables, Map[]> variableTypeArguments) throws QueryLanguageParseException { this(expression, packageImports, classImports, staticImports, variables, - variableTypeArguments, null, null, true); + variableTypeArguments, null, null, true, null); } /** @@ -226,9 +227,10 @@ public QueryLanguageParser( Map> variables, Map[]> variableTypeArguments, @Nullable Map queryScopeVariables, - @Nullable Set columnVariables) throws QueryLanguageParseException { + @Nullable Set columnVariables, + @Nullable TimeLiteralReplacedExpression timeConversionResult) throws QueryLanguageParseException { this(expression, packageImports, classImports, staticImports, variables, - variableTypeArguments, queryScopeVariables, columnVariables, true); + variableTypeArguments, queryScopeVariables, columnVariables, true, timeConversionResult); } /** @@ -257,8 +259,8 @@ public QueryLanguageParser( Map[]> variableTypeArguments, @Nullable Map queryScopeVariables, @Nullable Set columnVariables, - boolean unboxArguments) - throws QueryLanguageParseException { + boolean unboxArguments, + @Nullable TimeLiteralReplacedExpression timeConversionResult) throws QueryLanguageParseException { this( expression, packageImports, @@ -270,7 +272,8 @@ public QueryLanguageParser( columnVariables, unboxArguments, false, - PyCallableWrapperJpyImpl.class.getName()); + PyCallableWrapperJpyImpl.class.getName(), + timeConversionResult); } QueryLanguageParser( @@ -284,7 +287,8 @@ public QueryLanguageParser( @Nullable final Set columnVariables, final boolean unboxArguments, final boolean verifyIdempotence, - @NotNull final String pyCallableWrapperImplName) throws QueryLanguageParseException { + @NotNull final String pyCallableWrapperImplName, + @Nullable final TimeLiteralReplacedExpression timeConversionResult) throws QueryLanguageParseException { this.packageImports = packageImports == null ? Collections.emptySet() : Set.copyOf(packageImports); this.classImports = classImports == null ? Collections.emptySet() : Set.copyOf(classImports); this.staticImports = staticImports == null ? Collections.emptySet() : Set.copyOf(staticImports); @@ -346,7 +350,7 @@ public QueryLanguageParser( final QueryLanguageParser validationQueryLanguageParser = new QueryLanguageParser( printedSource, packageImports, classImports, staticImports, variables, variableTypeArguments, queryScopeVariables, columnVariables, false, false, - pyCallableWrapperImplName); + pyCallableWrapperImplName, timeConversionResult); final String reparsedSource = validationQueryLanguageParser.result.source; Assert.equals( @@ -364,7 +368,7 @@ public QueryLanguageParser( } result = new Result(type, printer.builder.toString(), variablesUsed, this.queryScopeVariables, - isConstantValueExpression, formulaShiftColPair); + isConstantValueExpression, formulaShiftColPair, timeConversionResult); } catch (Throwable e) { // need to catch it and make a new one because it contains unserializable variables... final StringBuilder exceptionMessageBuilder = new StringBuilder(1024) @@ -3245,6 +3249,7 @@ public static class Result { private final Map possibleParams; private final boolean isConstantValueExpression; private final Pair>> formulaShiftColPair; + private final TimeLiteralReplacedExpression timeConversionResult; Result( Class type, @@ -3252,13 +3257,15 @@ public static class Result { HashSet variablesUsed, Map possibleParams, boolean isConstantValueExpression, - Pair>> formulaShiftColPair) { + Pair>> formulaShiftColPair, + TimeLiteralReplacedExpression timeConversionResult) { this.type = Objects.requireNonNull(type, "type"); this.source = source; this.variablesUsed = variablesUsed; this.possibleParams = possibleParams; this.isConstantValueExpression = isConstantValueExpression; this.formulaShiftColPair = formulaShiftColPair; + this.timeConversionResult = timeConversionResult; } public Class getType() { @@ -3284,6 +3291,10 @@ public Map getPossibleParams() { public Pair>> getFormulaShiftColPair() { return formulaShiftColPair; } + + public TimeLiteralReplacedExpression getTimeConversionResult() { + return timeConversionResult; + } } public static class VisitArgs { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java index d55e3b84aa7..e96bb529f1e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java @@ -87,11 +87,8 @@ public synchronized void init( } try { - final TimeLiteralReplacedExpression timeConversionResult = - TimeLiteralReplacedExpression.convertExpression(formula); - final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( - timeConversionResult, tableDefinition.getColumnNameMap(), outerToInnerNames, + formula, tableDefinition.getColumnNameMap(), outerToInnerNames, compilationProcessor.getQueryScopeVariables(), unboxArguments); formulaShiftColPair = result.getFormulaShiftColPair(); @@ -156,7 +153,7 @@ public synchronized void init( final Class resultType = result.getType(); checkReturnType(result, resultType); - generateFilterCode(tableDefinition, timeConversionResult, result, compilationProcessor); + generateFilterCode(tableDefinition, result.getTimeConversionResult(), result, compilationProcessor); initialized = true; } } catch (Exception e) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index e772fa8d059..31a80d3a4e2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -32,7 +32,6 @@ import io.deephaven.engine.util.caching.C14nUtil; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; -import io.deephaven.time.TimeLiteralReplacedExpression; import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.type.TypeUtils; import io.deephaven.vector.ObjectVector; @@ -194,13 +193,10 @@ public List initDef( } try { - final TimeLiteralReplacedExpression timeConversionResult = - TimeLiteralReplacedExpression.convertExpression(formulaString); final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( - timeConversionResult, columnDefinitionMap, Collections.emptyMap(), - compilationRequestProcessor.getQueryScopeVariables(), true); - analyzedFormula = FormulaAnalyzer.analyze(formulaString, columnDefinitionMap, - timeConversionResult, result); + formulaString, columnDefinitionMap, Collections.emptyMap(), + compilationRequestProcessor.getQueryScopeVariables()); + analyzedFormula = FormulaAnalyzer.analyze(formulaString, columnDefinitionMap, result); hasConstantValue = result.isConstantValueExpression(); formulaShiftColPair = result.getFormulaShiftColPair(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java index 3e04c8ee0bb..c4ecb9a0394 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java @@ -171,7 +171,7 @@ public Class getReturnedType() { @Override public Class getReturnedComponentType() { - // we don't support reinterpretting column types with components + // we don't support reinterpreting column types with components return null; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java index a725ebdcc5b..fe05a0675ca 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java @@ -88,6 +88,8 @@ static Collection copyFrom(Collection selectColumns) /** * Initialize any internal column definitions from the provided initial. A compilation request consumer is provided * to allow for deferred compilation of expressions that belong to the same query. + *

+ * Compilations must be resolved before using this {@code SelectColumn}. * * @param columnDefinitionMap the starting set of column definitions; valid for this call only * @param compilationRequestProcessor a consumer to submit compilation requests; valid for this call only @@ -97,7 +99,6 @@ static Collection copyFrom(Collection selectColumns) * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. */ - @SuppressWarnings("unused") default List initDef( @NotNull final Map> columnDefinitionMap, @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java index 78b7239fb7e..b2a5907e32e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java @@ -281,7 +281,7 @@ public static SelectAndViewAnalyzerWrapper create( case SELECT_REFRESHING: { // We need to call newDestInstance because only newDestInstance has the knowledge to endow our // created array with the proper componentType (in the case of Vectors). - // TODO(kosak): use DeltaAwareColumnSource + // TODO: use DeltaAwareColumnSource WritableColumnSource scs = sc.newDestInstance(targetDestinationCapacity); WritableColumnSource underlyingSource = null; if (rowRedirection != null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java index c38bcff1ebc..eecd97ae009 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java @@ -31,7 +31,6 @@ public class FormulaAnalyzer { public static Result analyze(final String rawFormulaString, final Map> columnDefinitionMap, - final TimeLiteralReplacedExpression timeConversionResult, final QueryLanguageParser.Result queryLanguageResult) throws Exception { log.debug().append("Expression (after language conversion) : ") @@ -62,7 +61,7 @@ public static Result analyze(final String rawFormulaString, returnedType = Boolean.class; } final String cookedFormulaString = queryLanguageResult.getConvertedExpression(); - final String timeInstanceVariables = timeConversionResult.getInstanceVariablesString(); + final String timeInstanceVariables = queryLanguageResult.getTimeConversionResult().getInstanceVariablesString(); return new Result(returnedType, usedColumns.toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY), usedColumnArrays.toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY), @@ -74,7 +73,7 @@ public static Result analyze(final String rawFormulaString, /** * Get the compiled formula for a given formula string. * - * @param timeConversionResult The formula preprocessed to extract time literals + * @param formulaString The raw formula string * @param availableColumns The columns available for use in the formula * @param columnRenames Outer to inner column name mapping * @param queryScopeVariables The query scope variables @@ -82,17 +81,17 @@ public static Result analyze(final String rawFormulaString, * @throws Exception If the formula cannot be parsed */ public static QueryLanguageParser.Result parseFormula( - @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final String formulaString, @NotNull final Map> availableColumns, @NotNull final Map columnRenames, @NotNull final Map queryScopeVariables) throws Exception { - return parseFormula(timeConversionResult, availableColumns, columnRenames, queryScopeVariables, true); + return parseFormula(formulaString, availableColumns, columnRenames, queryScopeVariables, true); } /** * Get the compiled formula for a given formula string. * - * @param timeConversionResult The formula preprocessed to extract time literals + * @param formulaString The raw formula string * @param availableColumns The columns available for use in the formula * @param columnRenames Outer to inner column name mapping * @param queryScopeVariables The query scope variables @@ -101,11 +100,15 @@ public static QueryLanguageParser.Result parseFormula( * @throws Exception If the formula cannot be parsed */ public static QueryLanguageParser.Result parseFormula( - @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final String formulaString, @NotNull final Map> availableColumns, @NotNull final Map columnRenames, @NotNull final Map queryScopeVariables, final boolean unboxArguments) throws Exception { + + final TimeLiteralReplacedExpression timeConversionResult = + TimeLiteralReplacedExpression.convertExpression(formulaString); + final Map> possibleVariables = new HashMap<>(); possibleVariables.put("i", int.class); possibleVariables.put("ii", long.class); @@ -204,7 +207,7 @@ public static QueryLanguageParser.Result parseFormula( classImports.add(WritableColumnSource.class); return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), queryLibrary.getPackageImports(), classImports, queryLibrary.getStaticImports(), possibleVariables, possibleVariableParameterizedTypes, - queryScopeVariables, columnVariables, unboxArguments).getResult(); + queryScopeVariables, columnVariables, unboxArguments, timeConversionResult).getResult(); } public static class Result { diff --git a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java index 10ba947d128..51640f22d05 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java @@ -95,8 +95,7 @@ protected AbstractScriptSession( this.classCacheDirectory = classCacheDirectory; queryScope = new ScriptSessionQueryScope(); - final QueryCompiler compilerContext = - QueryCompilerImpl.create(classCacheDirectory, parentClassLoader); + final QueryCompiler compilerContext = QueryCompilerImpl.create(classCacheDirectory, parentClassLoader); executionContext = ExecutionContext.newBuilder() .markSystemic() diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index cbc7ebb03c1..bdb9e4d5389 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -37,6 +37,7 @@ import io.deephaven.time.DateTimeUtils; import io.deephaven.time.calendar.StaticCalendarMethods; import io.deephaven.util.QueryConstants; +import io.deephaven.util.SafeCloseable; import io.deephaven.util.annotations.VisibleForTesting; import io.deephaven.util.type.ArrayTypeUtils; import io.deephaven.util.type.TypeUtils; @@ -66,6 +67,7 @@ import java.time.ZonedDateTime; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -128,12 +130,12 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE /** Contains imports to be applied to .groovy files loaded from the classpath */ private final ImportCustomizer loadedGroovyScriptImports; - private final Set dynamicClasses = new HashSet<>(); + private final Set dynamicClasses; private final Map bindingBackingMap; private static class DeephavenGroovyShell extends GroovyShell { - private int counter; - private String script = "Script"; + private final AtomicInteger counter = new AtomicInteger(); + private volatile String scriptPrefix = "Script"; DeephavenGroovyShell( final GroovyClassLoader loader, @@ -143,12 +145,19 @@ private static class DeephavenGroovyShell extends GroovyShell { } @Override - protected synchronized String generateScriptName() { - return script + "_" + (++counter) + ".groovy"; + protected String generateScriptName() { + return scriptPrefix + "_" + (counter.incrementAndGet()) + ".groovy"; } private String getNextScriptClassName() { - return script + "_" + (counter + 1); + return scriptPrefix + "_" + (counter.get() + 1); + } + + public SafeCloseable setScriptPrefix(final String newPrefix) { + scriptPrefix = newPrefix; + return () -> { + scriptPrefix = newPrefix; + }; } } @@ -215,12 +224,15 @@ private GroovyDeephavenSession( super(updateGraph, operationInitializer, objectTypeLookup, changeListener, classCacheDirectory, groovyShell.getClassLoader()); + this.scriptFinder = new ScriptFinder(DEFAULT_SCRIPT_PATH); + this.consoleImports = consoleImports; this.loadedGroovyScriptImports = loadedGroovyScriptImports; + + this.dynamicClasses = new HashSet<>(); this.bindingBackingMap = bindingBackingMap; - this.groovyShell = groovyShell; - this.scriptFinder = new ScriptFinder(DEFAULT_SCRIPT_PATH); + this.groovyShell = groovyShell; groovyShell.setVariable("__groovySession", this); groovyShell.setVariable("DB_SCRIPT_PATH", DEFAULT_SCRIPT_PATH); @@ -319,10 +331,6 @@ protected T getVariable(String name) { } } - private void evaluateCommand(String command) { - groovyShell.evaluate(command); - } - @Override protected void evaluate(String command, String scriptName) { grepScriptImports(removeComments(command)); @@ -331,27 +339,23 @@ protected void evaluate(String command, String scriptName) { final String lastCommand = fc.second; final String commandPrefix = fc.first; - final String oldScriptName = groovyShell.script; - - try { - if (scriptName != null) { - groovyShell.script = scriptName.replaceAll("[^0-9A-Za-z_]", "_").replaceAll("(^[0-9])", "_$1"); - } - final String currentScriptName = groovyShell.script; + final String newScriptPrefix = scriptName == null + ? groovyShell.scriptPrefix + : scriptName.replaceAll("[^0-9A-Za-z_]", "_").replaceAll("(^[0-9])", "_$1"); + try (final SafeCloseable ignored = groovyShell.setScriptPrefix(newScriptPrefix)) { + final String currentScriptName = groovyShell.scriptPrefix; updateClassloader(lastCommand); try { ExecutionContext.getContext().getUpdateGraph().exclusiveLock() - .doLockedInterruptibly(() -> evaluateCommand(lastCommand)); + .doLockedInterruptibly(() -> groovyShell.evaluate(command)); } catch (InterruptedException e) { throw new CancellationException(e.getMessage() != null ? e.getMessage() : "Query interrupted", maybeRewriteStackTrace(scriptName, currentScriptName, e, lastCommand, commandPrefix)); } catch (Exception e) { throw wrapAndRewriteStackTrace(scriptName, currentScriptName, e, lastCommand, commandPrefix); } - } finally { - groovyShell.script = oldScriptName; } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java index 34d2db264b3..18fec0ccca1 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java @@ -14,6 +14,7 @@ import io.deephaven.engine.table.impl.lang.QueryLanguageParser.QueryLanguageParseException; import io.deephaven.engine.testutil.ControlledUpdateGraph; import io.deephaven.engine.util.PyCallableWrapper; +import io.deephaven.time.TimeLiteralReplacedExpression; import io.deephaven.util.QueryConstants; import io.deephaven.util.SafeCloseable; import io.deephaven.util.type.TypeUtils; @@ -3188,7 +3189,7 @@ private void check(String expression, String resultExpression, Class resultTy variables, variableParameterizedTypes, possibleParams, null, true, verifyIdempotence, - PyCallableWrapperDummyImpl.class.getName()).getResult(); + PyCallableWrapperDummyImpl.class.getName(), null).getResult(); assertEquals(resultType, result.getType()); assertEquals(resultExpression, result.getConvertedExpression()); From 86070cb1cded7bfadc47df2aa4587f3599c13122 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 23 Apr 2024 10:55:05 -0600 Subject: [PATCH 17/19] Bugfix GroovyDeephavenSession not in proper Package --- .../java/io/deephaven/engine/util/GroovyDeephavenSession.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index bdb9e4d5389..516458fe5ab 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -349,7 +349,7 @@ protected void evaluate(String command, String scriptName) { try { ExecutionContext.getContext().getUpdateGraph().exclusiveLock() - .doLockedInterruptibly(() -> groovyShell.evaluate(command)); + .doLockedInterruptibly(() -> groovyShell.evaluate(lastCommand)); } catch (InterruptedException e) { throw new CancellationException(e.getMessage() != null ? e.getMessage() : "Query interrupted", maybeRewriteStackTrace(scriptName, currentScriptName, e, lastCommand, commandPrefix)); From 94312449c951b44b15abaa456a187b5165e8c799 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 23 Apr 2024 12:33:15 -0600 Subject: [PATCH 18/19] Last Rnd Ryan Review --- .../table/impl/lang/QueryLanguageParser.java | 72 ++++++------------- .../engine/util/GroovyDeephavenSession.java | 10 +-- 2 files changed, 27 insertions(+), 55 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java index 1fd22940cc1..36b10f914fa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java @@ -110,6 +110,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.VisibleForTesting; import org.jpy.PyObject; import java.lang.reflect.Array; @@ -189,32 +190,7 @@ public final class QueryLanguageParser extends GenericVisitorAdapter, Q * imported. * @param variables A map of the names of scope variables to their types * @param variableTypeArguments A map of the names of scope variables to their type arguments - * @throws QueryLanguageParseException If any exception or error is encountered - */ - @TestUseOnly - QueryLanguageParser( - String expression, - Collection packageImports, - Collection> classImports, - Collection> staticImports, - Map> variables, - Map[]> variableTypeArguments) throws QueryLanguageParseException { - this(expression, packageImports, classImports, staticImports, variables, - variableTypeArguments, null, null, true, null); - } - - /** - * Create a QueryLanguageParser and parse the given {@code expression}. After construction, the - * {@link QueryLanguageParser.Result result} of parsing the {@code expression} is available with the - * {@link #getResult()}} method. - * - * @param expression The query language expression to parse - * @param packageImports Wildcard package imports - * @param classImports Individual class imports - * @param staticImports Wildcard static imports. All static variables and methods for the given classes are - * imported. - * @param variables A map of the names of scope variables to their types - * @param variableTypeArguments A map of the names of scope variables to their type arguments + * @param unboxArguments If true it will unbox the query scope arguments * @param queryScopeVariables A mutable map of the names of query scope variables to their values * @param columnVariables A set of column variable names * @throws QueryLanguageParseException If any exception or error is encountered @@ -228,9 +204,21 @@ public QueryLanguageParser( Map[]> variableTypeArguments, @Nullable Map queryScopeVariables, @Nullable Set columnVariables, + boolean unboxArguments, @Nullable TimeLiteralReplacedExpression timeConversionResult) throws QueryLanguageParseException { - this(expression, packageImports, classImports, staticImports, variables, - variableTypeArguments, queryScopeVariables, columnVariables, true, timeConversionResult); + this( + expression, + packageImports, + classImports, + staticImports, + variables, + variableTypeArguments, + queryScopeVariables, + columnVariables, + unboxArguments, + false, + PyCallableWrapperJpyImpl.class.getName(), + timeConversionResult); } /** @@ -245,37 +233,21 @@ public QueryLanguageParser( * imported. * @param variables A map of the names of scope variables to their types * @param variableTypeArguments A map of the names of scope variables to their type arguments - * @param unboxArguments If true it will unbox the query scope arguments - * @param queryScopeVariables A mutable map of the names of query scope variables to their values - * @param columnVariables A set of column variable names * @throws QueryLanguageParseException If any exception or error is encountered */ - public QueryLanguageParser( + @TestUseOnly + QueryLanguageParser( String expression, Collection packageImports, Collection> classImports, Collection> staticImports, Map> variables, - Map[]> variableTypeArguments, - @Nullable Map queryScopeVariables, - @Nullable Set columnVariables, - boolean unboxArguments, - @Nullable TimeLiteralReplacedExpression timeConversionResult) throws QueryLanguageParseException { - this( - expression, - packageImports, - classImports, - staticImports, - variables, - variableTypeArguments, - queryScopeVariables, - columnVariables, - unboxArguments, - false, - PyCallableWrapperJpyImpl.class.getName(), - timeConversionResult); + Map[]> variableTypeArguments) throws QueryLanguageParseException { + this(expression, packageImports, classImports, staticImports, variables, + variableTypeArguments, null, null, true, null); } + @VisibleForTesting QueryLanguageParser( String expression, final Collection packageImports, diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 516458fe5ab..e6291a639e9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -86,6 +86,7 @@ public class GroovyDeephavenSession extends AbstractScriptSession loadClass(String name, boolean resolve) throws ClassNotFoundE private static class DeephavenGroovyShell extends GroovyShell { private final AtomicInteger counter = new AtomicInteger(); - private volatile String scriptPrefix = "Script"; + private volatile String scriptPrefix = DEFAULT_SCRIPT_PREFIX; DeephavenGroovyShell( final GroovyClassLoader loader, @@ -339,11 +340,10 @@ protected void evaluate(String command, String scriptName) { final String lastCommand = fc.second; final String commandPrefix = fc.first; - final String newScriptPrefix = scriptName == null - ? groovyShell.scriptPrefix + final String currentScriptName = scriptName == null + ? DEFAULT_SCRIPT_PREFIX : scriptName.replaceAll("[^0-9A-Za-z_]", "_").replaceAll("(^[0-9])", "_$1"); - try (final SafeCloseable ignored = groovyShell.setScriptPrefix(newScriptPrefix)) { - final String currentScriptName = groovyShell.scriptPrefix; + try (final SafeCloseable ignored = groovyShell.setScriptPrefix(currentScriptName)) { updateClassloader(lastCommand); From 8c3827ecb28bce1dbacfc1e9c90dc8f838ab92f9 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 23 Apr 2024 12:35:46 -0600 Subject: [PATCH 19/19] unused import --- .../engine/util/scripts/TestGroovyDeephavenSession.java | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java b/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java index ff5e4b3cbaf..ae9f80e17da 100644 --- a/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java +++ b/engine/table/src/test/java/io/deephaven/engine/util/scripts/TestGroovyDeephavenSession.java @@ -16,7 +16,6 @@ import io.deephaven.function.Sort; import io.deephaven.plugin.type.ObjectTypeLookup.NoOp; import io.deephaven.util.SafeCloseable; -import io.deephaven.util.thread.ThreadInitializationFactory; import org.apache.commons.lang3.mutable.MutableInt; import org.junit.After; import org.junit.Assert;