From 07990d82bca9cc974c29ba30fbd91e913b18c845 Mon Sep 17 00:00:00 2001 From: Tim Meehan Date: Tue, 30 Jul 2024 10:11:46 -0400 Subject: [PATCH 1/5] Add SPI to customize expression optimization --- .../presto/spi/CoordinatorPlugin.java | 6 +++ .../planner/ExpressionOptimizerContext.java | 49 +++++++++++++++++++ .../planner/ExpressionOptimizerFactory.java | 25 ++++++++++ 3 files changed, 80 insertions(+) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerFactory.java diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java b/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java index d161738b67bc..ca8ea46d58aa 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java @@ -16,6 +16,7 @@ import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; import com.facebook.presto.spi.plan.PlanCheckerProviderFactory; import com.facebook.presto.spi.session.WorkerSessionPropertyProviderFactory; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; import static java.util.Collections.emptyList; @@ -40,4 +41,9 @@ default Iterable getPlanCheckerProviderFactories() { return emptyList(); } + + default Iterable getExpressionOptimizerFactories() + { + return emptyList(); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java new file mode 100644 index 000000000000..c9a6f84aaaa1 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.sql.planner; + +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; + +import static java.util.Objects.requireNonNull; + +public class ExpressionOptimizerContext +{ + private final NodeManager nodeManager; + private final FunctionMetadataManager functionMetadataManager; + private final StandardFunctionResolution functionResolution; + + public ExpressionOptimizerContext(NodeManager nodeManager, FunctionMetadataManager functionMetadataManager, StandardFunctionResolution functionResolution) + { + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + } + + public NodeManager getNodeManager() + { + return nodeManager; + } + + public FunctionMetadataManager getFunctionMetadataManager() + { + return functionMetadataManager; + } + + public StandardFunctionResolution getFunctionResolution() + { + return functionResolution; + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerFactory.java b/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerFactory.java new file mode 100644 index 000000000000..ad645e0c04a8 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerFactory.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.sql.planner; + +import com.facebook.presto.spi.relation.ExpressionOptimizer; + +import java.util.Map; + +public interface ExpressionOptimizerFactory +{ + ExpressionOptimizer createOptimizer(Map config, ExpressionOptimizerContext context); + + String getName(); +} From b6b086a775545490b797bb5c814a0017537e5838 Mon Sep 17 00:00:00 2001 From: Tim Meehan Date: Tue, 30 Jul 2024 10:37:54 -0400 Subject: [PATCH 2/5] Add DelegatingRowExpressionOptimizer --- .../presto/SystemSessionProperties.java | 11 + .../presto/cost/ScalarStatsCalculator.java | 8 +- .../facebook/presto/server/PluginManager.java | 12 +- .../facebook/presto/server/PrestoServer.java | 3 + .../presto/server/ServerMainModule.java | 4 + .../server/testing/TestingPrestoServer.java | 9 + .../presto/sql/analyzer/FeaturesConfig.java | 29 ++ .../ExpressionOptimizerManager.java | 103 +++++++ .../presto/sql/planner/PlanOptimizers.java | 16 +- .../rule/SimplifyRowExpressions.java | 37 ++- .../CteProjectionAndPredicatePushDown.java | 12 +- .../DelegatingRowExpressionOptimizer.java | 90 ++++++ .../relational/RowExpressionOptimizer.java | 17 +- .../presto/testing/LocalQueryRunner.java | 22 +- .../facebook/presto/testing/QueryRunner.java | 3 + .../sql/analyzer/TestFeaturesConfig.java | 10 +- .../sql/planner/TestLogicalPlanner.java | 34 ++- .../planner/assertions/OptimizerAssert.java | 11 +- .../iterative/rule/TestRemoveMapCastRule.java | 31 +- ...teConstantArrayContainsToInExpression.java | 66 ++-- .../rule/TestSimplifyRowExpressions.java | 22 +- .../iterative/rule/test/BaseRuleTest.java | 6 + .../iterative/rule/test/RuleTester.java | 9 + ...TestCteProjectionAndPredicatePushdown.java | 3 +- .../TestDelegatingRowExpressionOptimizer.java | 223 ++++++++++++++ .../nativeworker/ContainerQueryRunner.java | 7 + .../presto/spark/PrestoSparkModule.java | 6 + .../presto/spark/PrestoSparkQueryRunner.java | 7 + .../relation/ExpressionOptimizerProvider.java | 19 ++ .../spi/relation/RowExpressionService.java | 1 + .../tests/AbstractTestQueryFramework.java | 10 +- .../presto/tests/DistributedQueryRunner.java | 8 + .../presto/tests/StandaloneQueryRunner.java | 7 + .../presto/memory/TestMemoryManager.java | 3 +- .../TestDelegatingExpressionOptimizer.java | 146 +++++++++ .../TestExpressionInterpreter.java | 225 ++++++++++++++ .../expressions/TestExpressionOptimizers.java | 133 ++++++++ .../tests/expressions/TestExpressions.java | 286 ++++-------------- .../thrift/integration/ThriftQueryRunner.java | 7 + 39 files changed, 1366 insertions(+), 290 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/relational/DelegatingRowExpressionOptimizer.java create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/relational/TestDelegatingRowExpressionOptimizer.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizerProvider.java create mode 100644 presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestDelegatingExpressionOptimizer.java create mode 100644 presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressionInterpreter.java create mode 100644 presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressionOptimizers.java rename presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java => presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressions.java (87%) diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 8ab8d397d521..e2e0ff1eeaa0 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -326,6 +326,7 @@ public final class SystemSessionProperties public static final String INLINE_PROJECTIONS_ON_VALUES = "inline_projections_on_values"; public static final String INCLUDE_VALUES_NODE_IN_CONNECTOR_OPTIMIZER = "include_values_node_in_connector_optimizer"; public static final String SINGLE_NODE_EXECUTION_ENABLED = "single_node_execution_enabled"; + public static final String DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED = "delegating_row_expression_optimizer_enabled"; // TODO: Native execution related session properties that are temporarily put here. They will be relocated in the future. public static final String NATIVE_AGGREGATION_SPILL_ALL = "native_aggregation_spill_all"; @@ -1835,6 +1836,11 @@ public SystemSessionProperties( SINGLE_NODE_EXECUTION_ENABLED, "Enable single node execution", featuresConfig.isSingleNodeExecutionEnabled(), + false), + booleanProperty( + DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED, + "Enable delegating row optimizer", + featuresConfig.isDelegatingRowExpressionOptimizerEnabled(), false)); } @@ -3121,4 +3127,9 @@ public static int getMinColumnarEncodingChannelsToPreferRowWiseEncoding(Session { return session.getSystemProperty(NATIVE_MIN_COLUMNAR_ENCODING_CHANNELS_TO_PREFER_ROW_WISE_ENCODING, Integer.class); } + + public static boolean isDelegatingRowExpressionOptimizerEnabled(Session session) + { + return session.getSystemProperty(DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED, Boolean.class); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java index 044785ca0a44..cb5d8ec8c591 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java @@ -31,11 +31,11 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.analyzer.ExpressionAnalyzer; import com.facebook.presto.sql.analyzer.Scope; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ExpressionInterpreter; import com.facebook.presto.sql.planner.NoOpVariableResolver; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.relational.FunctionResolution; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.ArithmeticUnaryExpression; import com.facebook.presto.sql.tree.AstVisitor; @@ -78,11 +78,13 @@ public class ScalarStatsCalculator { private final Metadata metadata; + private final ExpressionOptimizerManager expressionOptimizerManager; @Inject - public ScalarStatsCalculator(Metadata metadata) + public ScalarStatsCalculator(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager) { this.metadata = requireNonNull(metadata, "metadata can not be null"); + this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager can not be null"); } @Deprecated @@ -126,7 +128,7 @@ public VariableStatsEstimate visitCall(CallExpression call, Void context) return computeArithmeticBinaryStatistics(call, context); } - RowExpression value = new RowExpressionOptimizer(metadata).optimize(call, OPTIMIZED, session); + RowExpression value = expressionOptimizerManager.getExpressionOptimizer().optimize(call, OPTIMIZED, session); if (isNull(value)) { return nullStatsEstimate(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java index f1cef40a1a3f..5a3a06b3b97e 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java @@ -43,6 +43,7 @@ import com.facebook.presto.spi.security.SystemAccessControlFactory; import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory; import com.facebook.presto.spi.session.WorkerSessionPropertyProviderFactory; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; import com.facebook.presto.spi.storage.TempStorageFactory; import com.facebook.presto.spi.tracing.TracerProvider; @@ -50,6 +51,7 @@ import com.facebook.presto.spi.ttl.NodeTtlFetcherFactory; import com.facebook.presto.sql.analyzer.AnalyzerProviderManager; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; import com.facebook.presto.storage.TempStorageManager; import com.facebook.presto.tracing.TracerProviderManager; @@ -135,6 +137,7 @@ public class PluginManager private final QueryPreparerProviderManager queryPreparerProviderManager; private final NodeStatusNotificationManager nodeStatusNotificationManager; private final PlanCheckerProviderManager planCheckerProviderManager; + private final ExpressionOptimizerManager expressionOptimizerManager; @Inject public PluginManager( @@ -157,7 +160,8 @@ public PluginManager( HistoryBasedPlanStatisticsManager historyBasedPlanStatisticsManager, TracerProviderManager tracerProviderManager, NodeStatusNotificationManager nodeStatusNotificationManager, - PlanCheckerProviderManager planCheckerProviderManager) + PlanCheckerProviderManager planCheckerProviderManager, + ExpressionOptimizerManager expressionOptimizerManager) { requireNonNull(nodeInfo, "nodeInfo is null"); requireNonNull(config, "config is null"); @@ -190,6 +194,7 @@ public PluginManager( this.queryPreparerProviderManager = requireNonNull(queryPreparerProviderManager, "queryPreparerProviderManager is null"); this.nodeStatusNotificationManager = requireNonNull(nodeStatusNotificationManager, "nodeStatusNotificationManager is null"); this.planCheckerProviderManager = requireNonNull(planCheckerProviderManager, "planCheckerProviderManager is null"); + this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionManager is null"); } public void loadPlugins() @@ -372,6 +377,11 @@ public void installCoordinatorPlugin(CoordinatorPlugin plugin) log.info("Registering plan checker provider factory %s", planCheckerProviderFactory.getName()); planCheckerProviderManager.addPlanCheckerProviderFactory(planCheckerProviderFactory); } + + for (ExpressionOptimizerFactory expressionOptimizerFactory : plugin.getExpressionOptimizerFactories()) { + log.info("Registering expression optimizer factory %s", expressionOptimizerFactory.getName()); + expressionOptimizerManager.addExpressionOptimizerFactory(expressionOptimizerFactory); + } } private URLClassLoader buildClassLoader(String plugin) diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index d8a0fe050e7a..105e740636d2 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -51,6 +51,7 @@ import com.facebook.presto.server.security.PasswordAuthenticatorManager; import com.facebook.presto.server.security.ServerSecurityModule; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; import com.facebook.presto.storage.TempStorageManager; @@ -190,6 +191,8 @@ public void run() PluginNodeManager pluginNodeManager = new PluginNodeManager(nodeManager, nodeInfo.getEnvironment()); planCheckerProviderManager.loadPlanCheckerProviders(pluginNodeManager); + injector.getInstance(ExpressionOptimizerManager.class).loadExpressionOptimizerFactory(); + startAssociatedProcesses(injector); injector.getInstance(Announcer.class).start(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index e2ab95982852..881588658003 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -194,6 +194,7 @@ import com.facebook.presto.sql.analyzer.MetadataExtractorMBean; import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -359,6 +360,9 @@ else if (serverConfig.isCoordinator()) { binder.bind(SystemSessionProperties.class).in(Scopes.SINGLETON); binder.bind(SessionPropertyDefaults.class).in(Scopes.SINGLETON); + // expression manager + binder.bind(ExpressionOptimizerManager.class).in(Scopes.SINGLETON); + // schema properties binder.bind(SchemaPropertyManager.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java index 034763c47212..e853a2846106 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java @@ -71,6 +71,7 @@ import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; @@ -168,6 +169,7 @@ public class TestingPrestoServer private final TaskManager taskManager; private final GracefulShutdownHandler gracefulShutdownHandler; private final ShutdownAction shutdownAction; + private final ExpressionOptimizerManager expressionManager; private final RequestBlocker requestBlocker; private final boolean resourceManager; private final boolean catalogServer; @@ -368,6 +370,7 @@ public TestingPrestoServer( procedureTester = injector.getInstance(ProcedureTester.class); splitManager = injector.getInstance(SplitManager.class); pageSourceManager = injector.getInstance(PageSourceManager.class); + expressionManager = injector.getInstance(ExpressionOptimizerManager.class); if (coordinator) { dispatchManager = injector.getInstance(DispatchManager.class); queryManager = injector.getInstance(QueryManager.class); @@ -385,6 +388,7 @@ public TestingPrestoServer( eventListenerManager = ((TestingEventListenerManager) injector.getInstance(EventListenerManager.class)); clusterStateProvider = null; planCheckerProviderManager = injector.getInstance(PlanCheckerProviderManager.class); + expressionManager.loadExpressionOptimizerFactory(); } else if (resourceManager) { dispatchManager = null; @@ -704,6 +708,11 @@ public ShutdownAction getShutdownAction() return shutdownAction; } + public ExpressionOptimizerManager getExpressionManager() + { + return expressionManager; + } + public boolean isCoordinator() { return coordinator; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index f7ea9174fe8e..9c45e84f3318 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -279,6 +279,8 @@ public class FeaturesConfig private boolean generateDomainFilters; private boolean printEstimatedStatsFromCache; private boolean removeCrossJoinWithSingleConstantRow = true; + private boolean delegatingRowOptimizerEnabled; + private int delegatingRowOptimizerMaxIterations = 10; private CreateView.Security defaultViewSecurityMode = DEFINER; private boolean useHistograms; @@ -2875,4 +2877,31 @@ public FeaturesConfig setSingleNodeExecutionEnabled(boolean singleNodeExecutionE this.singleNodeExecutionEnabled = singleNodeExecutionEnabled; return this; } + + public boolean isDelegatingRowExpressionOptimizerEnabled() + { + return delegatingRowOptimizerEnabled; + } + + @Config("optimizer.delegating-row-expression-optimizer-enabled") + @ConfigDescription("Enable delegating row optimizer") + public FeaturesConfig setDelegatingRowExpressionOptimizerEnabled(boolean delegatingRowOptimizerEnabled) + { + this.delegatingRowOptimizerEnabled = delegatingRowOptimizerEnabled; + return this; + } + + @Min(1) + public int getDelegatingRowExpressionOptimizerMaxIterations() + { + return delegatingRowOptimizerMaxIterations; + } + + @Config("optimizer.delegating-row-expression-optimizer-max-iterations") + @ConfigDescription("Maximum number of iterations for delegating row optimizer") + public FeaturesConfig setDelegatingRowExpressionOptimizerMaxIterations(int delegatingRowOptimizerMaxIterations) + { + this.delegatingRowOptimizerMaxIterations = delegatingRowOptimizerMaxIterations; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java b/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java new file mode 100644 index 000000000000..303a5e2abb29 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java @@ -0,0 +1,103 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.expressions; + +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.nodeManager.PluginNodeManager; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerContext; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; + +import javax.inject.Inject; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; + +import static com.facebook.presto.util.PropertiesUtil.loadProperties; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Objects.requireNonNull; + +public class ExpressionOptimizerManager + implements ExpressionOptimizerProvider +{ + private static final File EXPRESSION_MANAGER_CONFIGURATION = new File("etc/expression-manager.properties"); + public static final String EXPRESSION_MANAGER_FACTORY_NAME = "expression-manager-factory.name"; + + private final Map expressionOptimizerFactories = new ConcurrentHashMap<>(); + private final AtomicReference rowExpressionInterpreter = new AtomicReference<>(); + private final NodeManager nodeManager; + private final FunctionAndTypeManager functionAndTypeManager; + private final FunctionResolution functionResolution; + private final ExpressionOptimizer defaultExpressionOptimizer; + + @Inject + public ExpressionOptimizerManager(PluginNodeManager nodeManager, FunctionAndTypeManager functionAndTypeManager) + { + requireNonNull(nodeManager, "nodeManager is null"); + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); + this.functionResolution = new FunctionResolution(functionAndTypeManager.getFunctionAndTypeResolver()); + this.defaultExpressionOptimizer = new RowExpressionOptimizer(functionAndTypeManager); + rowExpressionInterpreter.set(defaultExpressionOptimizer); + } + + public void loadExpressionOptimizerFactory() + { + try { + if (EXPRESSION_MANAGER_CONFIGURATION.exists()) { + Map properties = new HashMap<>(loadProperties(EXPRESSION_MANAGER_CONFIGURATION)); + loadExpressionOptimizerFactory(properties); + } + } + catch (IOException e) { + throw new UncheckedIOException("Failed to load expression manager configuration", e); + } + } + + private void loadExpressionOptimizerFactory(Map properties) + { + properties = new HashMap<>(properties); + String factoryName = properties.remove(EXPRESSION_MANAGER_FACTORY_NAME); + checkArgument(!isNullOrEmpty(factoryName), "%s does not contain %s", EXPRESSION_MANAGER_CONFIGURATION, EXPRESSION_MANAGER_FACTORY_NAME); + checkArgument( + rowExpressionInterpreter.compareAndSet( + defaultExpressionOptimizer, + expressionOptimizerFactories.get(factoryName).createOptimizer(properties, new ExpressionOptimizerContext(nodeManager, functionAndTypeManager, functionResolution))), + "ExpressionManager is already loaded"); + } + + public void addExpressionOptimizerFactory(ExpressionOptimizerFactory expressionOptimizerFactory) + { + String name = expressionOptimizerFactory.getName(); + checkArgument( + this.expressionOptimizerFactories.putIfAbsent(name, expressionOptimizerFactory) == null, + "ExpressionOptimizerFactory %s is already registered", name); + } + + @Override + public ExpressionOptimizer getExpressionOptimizer() + { + return rowExpressionInterpreter.get(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index a900bb364d16..ab4b87ed9ab0 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -22,6 +22,7 @@ import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; import com.facebook.presto.sql.planner.iterative.Rule; @@ -220,7 +221,8 @@ public PlanOptimizers( CostComparator costComparator, TaskCountEstimator taskCountEstimator, PartitioningProviderManager partitioningProviderManager, - FeaturesConfig featuresConfig) + FeaturesConfig featuresConfig, + ExpressionOptimizerManager expressionOptimizerManager) { this(metadata, sqlParser, @@ -235,7 +237,8 @@ public PlanOptimizers( costComparator, taskCountEstimator, partitioningProviderManager, - featuresConfig); + featuresConfig, + expressionOptimizerManager); } @PostConstruct @@ -266,7 +269,8 @@ public PlanOptimizers( CostComparator costComparator, TaskCountEstimator taskCountEstimator, PartitioningProviderManager partitioningProviderManager, - FeaturesConfig featuresConfig) + FeaturesConfig featuresConfig, + ExpressionOptimizerManager expressionOptimizerManager) { this.exporter = exporter; ImmutableList.Builder builder = ImmutableList.builder(); @@ -321,7 +325,7 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.>builder() - .addAll(new SimplifyRowExpressions(metadata).rules()) + .addAll(new SimplifyRowExpressions(metadata, expressionOptimizerManager, featuresConfig).rules()) .add(new PruneRedundantProjectionAssignments()) .build()); @@ -487,7 +491,7 @@ public PlanOptimizers( estimatedExchangesCostCalculator, ImmutableSet.>builder() .add(new InlineProjectionsOnValues(metadata.getFunctionAndTypeManager())) - .addAll(new SimplifyRowExpressions(metadata).rules()) + .addAll(new SimplifyRowExpressions(metadata, expressionOptimizerManager, featuresConfig).rules()) .build()), new IterativeOptimizer( metadata, @@ -846,7 +850,7 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of(new PushTableWriteThroughUnion()))); // Must run before AddExchanges - builder.add(new CteProjectionAndPredicatePushDown(metadata)); // must run before PhysicalCteOptimizer + builder.add(new CteProjectionAndPredicatePushDown(metadata, expressionOptimizerManager, featuresConfig)); // must run before PhysicalCteOptimizer builder.add(new PhysicalCteOptimizer(metadata)); // Must run before AddExchanges builder.add(new StatsRecordingPlanOptimizer(optimizerStats, new AddExchanges(metadata, partitioningProviderManager, featuresConfig.isNativeExecutionEnabled()))); builder.add(new StatsRecordingPlanOptimizer(optimizerStats, new AddExchangesForSingleNodeExecution(metadata))); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java index 9705f7e836fd..9127beacb562 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java @@ -13,22 +13,27 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.Session; import com.facebook.presto.common.type.BooleanType; import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.expressions.RowExpressionRewriter; import com.facebook.presto.expressions.RowExpressionTreeRewriter; import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; -import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.relational.DelegatingRowExpressionOptimizer; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.google.common.annotations.VisibleForTesting; +import static com.facebook.presto.SystemSessionProperties.isDelegatingRowExpressionOptimizerEnabled; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.SERIALIZABLE; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND; @@ -39,44 +44,52 @@ public class SimplifyRowExpressions extends RowExpressionRewriteRuleSet { - public SimplifyRowExpressions(Metadata metadata) + public SimplifyRowExpressions(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager, FeaturesConfig featuresConfig) { - super(new Rewriter(metadata)); + super(new Rewriter(metadata, expressionOptimizerManager, featuresConfig)); } private static class Rewriter implements PlanRowExpressionRewriter { - private final RowExpressionOptimizer optimizer; + private final ExpressionOptimizer inMemoryExpressionOptimizer; + private final ExpressionOptimizer delegatingExpressionOptimizer; private final LogicalExpressionRewriter logicalExpressionRewriter; - public Rewriter(Metadata metadata) + public Rewriter(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager, FeaturesConfig featuresConfig) { requireNonNull(metadata, "metadata is null"); - this.optimizer = new RowExpressionOptimizer(metadata); + requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); + requireNonNull(featuresConfig, "featuresConfig is null"); + this.inMemoryExpressionOptimizer = new RowExpressionOptimizer(metadata); + this.delegatingExpressionOptimizer = new DelegatingRowExpressionOptimizer(metadata, expressionOptimizerManager, featuresConfig.getDelegatingRowExpressionOptimizerMaxIterations()); this.logicalExpressionRewriter = new LogicalExpressionRewriter(metadata.getFunctionAndTypeManager()); } @Override public RowExpression rewrite(RowExpression expression, Rule.Context context) { - return rewrite(expression, context.getSession().toConnectorSession()); + return rewrite(expression, context.getSession()); } - private RowExpression rewrite(RowExpression expression, ConnectorSession session) + private RowExpression rewrite(RowExpression expression, Session session) { // Rewrite RowExpression first to reduce depth of RowExpression tree by balancing AND/OR predicates. // It doesn't matter whether we rewrite/optimize first because this will be called by IterativeOptimizer. RowExpression rewritten = RowExpressionTreeRewriter.rewriteWith(logicalExpressionRewriter, expression, true); - RowExpression optimizedRowExpression = optimizer.optimize(rewritten, SERIALIZABLE, session); - return optimizedRowExpression; + if (isDelegatingRowExpressionOptimizerEnabled(session)) { + return delegatingExpressionOptimizer.optimize(rewritten, SERIALIZABLE, session.toConnectorSession()); + } + else { + return inMemoryExpressionOptimizer.optimize(rewritten, SERIALIZABLE, session.toConnectorSession()); + } } } @VisibleForTesting - public static RowExpression rewrite(RowExpression expression, Metadata metadata, ConnectorSession session) + public static RowExpression rewrite(RowExpression expression, Metadata metadata, Session session, ExpressionOptimizerManager expressionOptimizerManager, FeaturesConfig featuresConfig) { - return new Rewriter(metadata).rewrite(expression, session); + return new Rewriter(metadata, expressionOptimizerManager, featuresConfig).rewrite(expression, session); } private static class LogicalExpressionRewriter diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CteProjectionAndPredicatePushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CteProjectionAndPredicatePushDown.java index d64068aeb3c2..cb706a0d2c01 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CteProjectionAndPredicatePushDown.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CteProjectionAndPredicatePushDown.java @@ -26,6 +26,8 @@ import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.PlannerUtils; import com.facebook.presto.sql.planner.RowExpressionVariableInliner; import com.facebook.presto.sql.planner.SimplePlanVisitor; @@ -87,10 +89,14 @@ public class CteProjectionAndPredicatePushDown implements PlanOptimizer { private final Metadata metadata; + private final ExpressionOptimizerManager expressionOptimizerManager; + private final FeaturesConfig featuresConfig; - public CteProjectionAndPredicatePushDown(Metadata metadata) + public CteProjectionAndPredicatePushDown(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager, FeaturesConfig featuresConfig) { - this.metadata = metadata; + this.metadata = requireNonNull(metadata, "metadata is null"); + this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); + this.featuresConfig = requireNonNull(featuresConfig, "featuresConfig is null"); } @Override @@ -383,7 +389,7 @@ private PlanNode addFilter(PlanNode node, List predicates) resultPredicate, predicates.get(i)); } } - resultPredicate = SimplifyRowExpressions.rewrite(resultPredicate, metadata, session.toConnectorSession()); + resultPredicate = SimplifyRowExpressions.rewrite(resultPredicate, metadata, session, expressionOptimizerManager, featuresConfig); return new FilterNode(node.getSourceLocation(), idAllocator.getNextId(), node, resultPredicate); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/DelegatingRowExpressionOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/DelegatingRowExpressionOptimizer.java new file mode 100644 index 000000000000..b420adc9ed18 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/DelegatingRowExpressionOptimizer.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; +import com.facebook.presto.spi.relation.InputReferenceExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; + +import java.util.function.Function; + +import static com.facebook.presto.sql.planner.LiteralEncoder.toRowExpression; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public final class DelegatingRowExpressionOptimizer + implements ExpressionOptimizer +{ + private static final int DEFAULT_MAX_OPTIMIZATION_ATTEMPTS = 10; + private final ExpressionOptimizerProvider expressionOptimizerManager; + private final int maxOptimizationAttempts; + + public DelegatingRowExpressionOptimizer(Metadata metadata, ExpressionOptimizerProvider expressionOptimizerManager) + { + this(metadata, expressionOptimizerManager, DEFAULT_MAX_OPTIMIZATION_ATTEMPTS); + } + + public DelegatingRowExpressionOptimizer(Metadata metadata, ExpressionOptimizerProvider expressionOptimizerManager, int maxOptimizationAttempts) + { + requireNonNull(metadata, "metadata is null"); + this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); + checkArgument(maxOptimizationAttempts > 0, "maxOptimizationAttempts must be greater than 0"); + this.maxOptimizationAttempts = maxOptimizationAttempts; + } + + @Override + public RowExpression optimize(RowExpression rowExpression, Level level, ConnectorSession session) + { + ExpressionOptimizer delegate = expressionOptimizerManager.getExpressionOptimizer(); + RowExpression originalExpression; + for (int i = 0; i < maxOptimizationAttempts; i++) { + // Do not optimize ConstantExpression, and InputReferenceExpression because they cannot be optimized further + if (rowExpression instanceof ConstantExpression || rowExpression instanceof InputReferenceExpression) { + return rowExpression; + } + originalExpression = rowExpression; + rowExpression = delegate.optimize(rowExpression, level, session); + requireNonNull(rowExpression, "optimized expression is null"); + if (originalExpression.equals(rowExpression)) { + break; + } + } + return rowExpression; + } + + @Override + public Object optimize(RowExpression rowExpression, Level level, ConnectorSession session, Function variableResolver) + { + ExpressionOptimizer delegate = expressionOptimizerManager.getExpressionOptimizer(); + Object currentExpression = rowExpression; + Object originalExpression; + for (int i = 0; i < maxOptimizationAttempts; i++) { + // Do not optimize ConstantExpression, and InputReferenceExpression because they cannot be optimized further + if (currentExpression instanceof ConstantExpression || currentExpression instanceof InputReferenceExpression) { + return currentExpression; + } + originalExpression = currentExpression; + currentExpression = delegate.optimize(toRowExpression(currentExpression, rowExpression.getType()), level, session, variableResolver); + if (currentExpression == null || currentExpression.equals(originalExpression)) { + break; + } + } + return currentExpression; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java index 1ab9d2ead2e6..f4a7a067f780 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java @@ -13,8 +13,10 @@ */ package com.facebook.presto.sql.relational; +import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -24,23 +26,30 @@ import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.sql.planner.LiteralEncoder.toRowExpression; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; public final class RowExpressionOptimizer implements ExpressionOptimizer { - private final Metadata metadata; + private final FunctionAndTypeManager functionAndTypeManager; public RowExpressionOptimizer(Metadata metadata) { - this.metadata = requireNonNull(metadata, "metadata is null"); + this(requireNonNull(metadata, "metadata is null").getFunctionAndTypeManager()); + } + + public RowExpressionOptimizer(FunctionMetadataManager functionMetadataManager) + { + checkArgument(functionMetadataManager instanceof FunctionAndTypeManager, "Expected functionMetadataManager to be instance of FunctionAndTypeManager"); + this.functionAndTypeManager = (FunctionAndTypeManager) requireNonNull(functionMetadataManager, "functionMetadataManager is null"); } @Override public RowExpression optimize(RowExpression rowExpression, Level level, ConnectorSession session) { if (level.ordinal() <= OPTIMIZED.ordinal()) { - return toRowExpression(rowExpression.getSourceLocation(), new RowExpressionInterpreter(rowExpression, metadata.getFunctionAndTypeManager(), session, level).optimize(), rowExpression.getType()); + return toRowExpression(rowExpression.getSourceLocation(), new RowExpressionInterpreter(rowExpression, functionAndTypeManager, session, level).optimize(), rowExpression.getType()); } throw new IllegalArgumentException("Not supported optimization level: " + level); } @@ -48,7 +57,7 @@ public RowExpression optimize(RowExpression rowExpression, Level level, Connecto @Override public Object optimize(RowExpression expression, Level level, ConnectorSession session, Function variableResolver) { - RowExpressionInterpreter interpreter = new RowExpressionInterpreter(expression, metadata.getFunctionAndTypeManager(), session, level); + RowExpressionInterpreter interpreter = new RowExpressionInterpreter(expression, functionAndTypeManager, session, level); return interpreter.optimize(variableResolver::apply); } } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index cefcd193422d..248bd6a11ec0 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -106,6 +106,7 @@ import com.facebook.presto.metadata.SchemaPropertyManager; import com.facebook.presto.metadata.Split; import com.facebook.presto.metadata.TablePropertyManager; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.operator.Driver; import com.facebook.presto.operator.DriverContext; import com.facebook.presto.operator.DriverFactory; @@ -168,6 +169,7 @@ import com.facebook.presto.sql.analyzer.JavaFeaturesConfig; import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -353,6 +355,7 @@ public class LocalQueryRunner private static ExecutorService metadataExtractorExecutor = newCachedThreadPool(threadsNamed("query-execution-%s")); private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private ExpressionOptimizerManager expressionOptimizerManager; private List additionalOptimizer = ImmutableList.of(); @@ -450,8 +453,12 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, this.planFragmenter = new PlanFragmenter(this.metadata, this.nodePartitioningManager, new QueryManagerConfig(), featuresConfig, planCheckerProviderManager); this.joinCompiler = new JoinCompiler(metadata); this.pageIndexerFactory = new GroupByHashPageIndexerFactory(joinCompiler); + + NodeInfo nodeInfo = new NodeInfo("test"); + expressionOptimizerManager = new ExpressionOptimizerManager(new PluginNodeManager(nodeManager, nodeInfo.getEnvironment()), getFunctionAndTypeManager()); + this.statsNormalizer = new StatsNormalizer(); - this.scalarStatsCalculator = new ScalarStatsCalculator(metadata); + this.scalarStatsCalculator = new ScalarStatsCalculator(metadata, expressionOptimizerManager); this.filterStatsCalculator = new FilterStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer); this.historyBasedPlanStatisticsManager = new HistoryBasedPlanStatisticsManager(objectMapper, createTestingSessionPropertyManager(), metadata, new HistoryBasedOptimizationConfig(), featuresConfig, new NodeVersion("1")); this.fragmentStatsProvider = new FragmentStatsProvider(); @@ -466,7 +473,6 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, this.expressionCompiler = new ExpressionCompiler(metadata, pageFunctionCompiler); this.joinFilterFunctionCompiler = new JoinFilterFunctionCompiler(metadata); - NodeInfo nodeInfo = new NodeInfo("test"); NodeVersion nodeVersion = new NodeVersion("testversion"); this.connectorManager = new ConnectorManager( metadata, @@ -529,7 +535,8 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, historyBasedPlanStatisticsManager, new TracerProviderManager(new TracingConfig()), new NodeStatusNotificationManager(), - planCheckerProviderManager); + planCheckerProviderManager, + expressionOptimizerManager); connectorManager.addConnectorFactory(globalSystemConnectorFactory); connectorManager.createConnection(GlobalSystemConnector.NAME, GlobalSystemConnector.NAME, ImmutableMap.of()); @@ -707,6 +714,12 @@ public TestingAccessControlManager getAccessControl() return accessControl; } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + return expressionOptimizerManager; + } + public ExecutorService getExecutor() { return notificationExecutor; @@ -1127,7 +1140,8 @@ public List getPlanOptimizers(boolean noExchange) new CostComparator(featuresConfig), taskCountEstimator, partitioningProviderManager, - featuresConfig).getPlanningTimeOptimizers()); + featuresConfig, + expressionOptimizerManager).getPlanningTimeOptimizers()); return planOptimizers.build(); } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/QueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/QueryRunner.java index d2121ce335f8..e62447a403e9 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/QueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/QueryRunner.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.Plan; @@ -63,6 +64,8 @@ public interface QueryRunner TestingAccessControlManager getAccessControl(); + ExpressionOptimizerManager getExpressionManager(); + MaterializedResult execute(@Language("SQL") String sql); MaterializedResult execute(Session session, @Language("SQL") String sql); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index ff8154866a60..c1446678a48c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -248,7 +248,9 @@ public void testDefaults() .setEagerPlanValidationEnabled(false) .setEagerPlanValidationThreadPoolSize(20) .setPrestoSparkExecutionEnvironment(false) - .setSingleNodeExecutionEnabled(false)); + .setSingleNodeExecutionEnabled(false) + .setDelegatingRowExpressionOptimizerEnabled(false) + .setDelegatingRowExpressionOptimizerMaxIterations(10)); } @Test @@ -446,6 +448,8 @@ public void testExplicitPropertyMappings() .put("eager-plan-validation-thread-pool-size", "2") .put("presto-spark-execution-environment", "true") .put("single-node-execution-enabled", "true") + .put("optimizer.delegating-row-expression-optimizer-enabled", "true") + .put("optimizer.delegating-row-expression-optimizer-max-iterations", "5") .build(); FeaturesConfig expected = new FeaturesConfig() @@ -640,7 +644,9 @@ public void testExplicitPropertyMappings() .setEagerPlanValidationEnabled(true) .setEagerPlanValidationThreadPoolSize(2) .setPrestoSparkExecutionEnvironment(true) - .setSingleNodeExecutionEnabled(true); + .setSingleNodeExecutionEnabled(true) + .setDelegatingRowExpressionOptimizerEnabled(true) + .setDelegatingRowExpressionOptimizerMaxIterations(5); assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java index d3c644b793ca..f44809325bc0 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java @@ -14,10 +14,18 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.Session; +import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.block.SortOrder; +import com.facebook.presto.common.type.StandardTypes; import com.facebook.presto.functionNamespace.FunctionNamespaceManagerPlugin; import com.facebook.presto.functionNamespace.json.JsonFileBasedFunctionNamespaceManagerFactory; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.AggregationFunctionMetadata; +import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.Parameter; +import com.facebook.presto.spi.function.RoutineCharacteristics; +import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.FilterNode; @@ -76,8 +84,12 @@ import static com.facebook.presto.common.block.SortOrder.ASC_NULLS_LAST; import static com.facebook.presto.common.predicate.Domain.singleValue; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.VarcharType.createVarcharType; import static com.facebook.presto.spi.StandardErrorCode.INVALID_LIMIT_CLAUSE; +import static com.facebook.presto.spi.function.FunctionVersion.notVersioned; +import static com.facebook.presto.spi.function.RoutineCharacteristics.Determinism.DETERMINISTIC; +import static com.facebook.presto.spi.function.RoutineCharacteristics.Language.CPP; import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; @@ -88,8 +100,6 @@ import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; -import static com.facebook.presto.sql.TestExpressionInterpreter.AVG_UDAF_CPP; -import static com.facebook.presto.sql.TestExpressionInterpreter.SQUARE_UDF_CPP; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.any; @@ -144,6 +154,26 @@ public class TestLogicalPlanner extends BasePlanTest { + public static final SqlInvokedFunction SQUARE_UDF_CPP = new SqlInvokedFunction( + QualifiedObjectName.valueOf(new CatalogSchemaName("json", "test_schema"), "square"), + ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.BIGINT))), + parseTypeSignature(StandardTypes.BIGINT), + "Integer square", + RoutineCharacteristics.builder().setDeterminism(DETERMINISTIC).setLanguage(CPP).build(), + "", + notVersioned()); + + public static final SqlInvokedFunction AVG_UDAF_CPP = new SqlInvokedFunction( + QualifiedObjectName.valueOf(new CatalogSchemaName("json", "test_schema"), "avg"), + ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.DOUBLE))), + parseTypeSignature(StandardTypes.DOUBLE), + "Returns mean of doubles", + RoutineCharacteristics.builder().setDeterminism(DETERMINISTIC).setLanguage(CPP).build(), + "", + notVersioned(), + FunctionKind.AGGREGATE, + Optional.of(new AggregationFunctionMetadata(parseTypeSignature("ROW(double, int)"), false))); + // TODO: Use com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder#tableScan with required node/stream // partitioning to properly test aggregation, window function and join. diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java index b4ad658409b8..ddb6b0d64767 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java @@ -16,13 +16,17 @@ import com.facebook.presto.Session; import com.facebook.presto.cost.StatsAndCosts; import com.facebook.presto.cost.StatsCalculator; +import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.Optimizer; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.TypeProvider; @@ -170,7 +174,12 @@ private List getMinimalOptimizers() new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), - new SimplifyRowExpressions(metadata).rules())); + new SimplifyRowExpressions( + metadata, + new ExpressionOptimizerManager( + new PluginNodeManager(new InMemoryNodeManager()), + queryRunner.getFunctionAndTypeManager()), + new FeaturesConfig()).rules())); } private void inTransaction(Function transactionSessionConsumer) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveMapCastRule.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveMapCastRule.java index 19638181d35e..8519c3e7fddb 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveMapCastRule.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveMapCastRule.java @@ -14,12 +14,15 @@ package com.facebook.presto.sql.planner.iterative.rule; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import static com.facebook.presto.SystemSessionProperties.DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED; import static com.facebook.presto.SystemSessionProperties.REMOVE_MAP_CAST; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.DoubleType.DOUBLE; @@ -33,12 +36,24 @@ public class TestRemoveMapCastRule extends BaseRuleTest { - @Test - public void testSubscriptCast() + @DataProvider(name = "delegating-row-expression-optimizer-enabled") + public Object[][] delegatingDataProvider() + { + return new Object[][] { + {true}, + {false}, + }; + } + @Test(dataProvider = "delegating-row-expression-optimizer-enabled") + public void testSubscriptCast(boolean enableDelegatingRowExpressionOptimizer) { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll(new RemoveMapCastRule(getFunctionManager()).rules()).build()) + ImmutableSet.>builder() + .addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager(), new FeaturesConfig()).rules()) + .addAll(new RemoveMapCastRule(getFunctionManager()).rules()) + .build()) .setSystemProperty(REMOVE_MAP_CAST, "true") + .setSystemProperty(DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED, Boolean.toString(enableDelegatingRowExpressionOptimizer)) .on(p -> { VariableReferenceExpression a = p.variable("a", DOUBLE); VariableReferenceExpression feature = p.variable("feature", createMapType(getFunctionManager(), INTEGER, DOUBLE)); @@ -53,12 +68,16 @@ public void testSubscriptCast() values("feature", "key"))); } - @Test - public void testElementAtCast() + @Test(dataProvider = "delegating-row-expression-optimizer-enabled") + public void testElementAtCast(boolean enableDelegatingRowExpressionOptimizer) { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll(new RemoveMapCastRule(getFunctionManager()).rules()).build()) + ImmutableSet.>builder() + .addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager(), new FeaturesConfig()).rules()) + .addAll(new RemoveMapCastRule(getFunctionManager()).rules()) + .build()) .setSystemProperty(REMOVE_MAP_CAST, "true") + .setSystemProperty(DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED, Boolean.toString(enableDelegatingRowExpressionOptimizer)) .on(p -> { VariableReferenceExpression a = p.variable("a", DOUBLE); VariableReferenceExpression feature = p.variable("feature", createMapType(getFunctionManager(), INTEGER, DOUBLE)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteConstantArrayContainsToInExpression.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteConstantArrayContainsToInExpression.java index 47c50c31a730..69eb3b69913f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteConstantArrayContainsToInExpression.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteConstantArrayContainsToInExpression.java @@ -15,12 +15,15 @@ import com.facebook.presto.common.type.ArrayType; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import static com.facebook.presto.SystemSessionProperties.DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED; import static com.facebook.presto.SystemSessionProperties.REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; @@ -32,13 +35,25 @@ public class TestRewriteConstantArrayContainsToInExpression extends BaseRuleTest { - @Test - public void testNoNull() + @DataProvider(name = "delegating-row-expression-optimizer-enabled") + public Object[][] delegatingDataProvider() + { + return new Object[][] { + {true}, + {false}, + }; + } + + @Test(dataProvider = "delegating-row-expression-optimizer-enabled") + public void testNoNull(boolean enableDelegatingRowExpressionOptimizer) { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll( - new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()).build()) + ImmutableSet.>builder() + .addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager(), new FeaturesConfig()).rules()) + .addAll(new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()) + .build()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") + .setSystemProperty(DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED, Boolean.toString(enableDelegatingRowExpressionOptimizer)) .on(p -> { VariableReferenceExpression a = p.variable("a", BOOLEAN); VariableReferenceExpression b = p.variable("b"); @@ -52,11 +67,12 @@ public void testNoNull() values("b"))); } - @Test - public void testDoesNotFireForNestedArray() + @Test(dataProvider = "delegating-row-expression-optimizer-enabled") + public void testDoesNotFireForNestedArray(boolean enableDelegatingRowExpressionOptimizer) { tester().assertThat(new RewriteConstantArrayContainsToInExpression(getFunctionManager()).projectRowExpressionRewriteRule()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") + .setSystemProperty(DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED, Boolean.toString(enableDelegatingRowExpressionOptimizer)) .on(p -> { VariableReferenceExpression a = p.variable("a", BOOLEAN); VariableReferenceExpression b = p.variable("b", new ArrayType(BIGINT)); @@ -67,8 +83,8 @@ public void testDoesNotFireForNestedArray() .doesNotFire(); } - @Test - public void testDoesNotFireForNull() + @Test(dataProvider = "delegating-row-expression-optimizer-enabled") + public void testDoesNotFireForNull(boolean enableDelegatingRowExpressionOptimizer) { tester().assertThat(new RewriteConstantArrayContainsToInExpression(getFunctionManager()).projectRowExpressionRewriteRule()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") @@ -82,8 +98,8 @@ public void testDoesNotFireForNull() .doesNotFire(); } - @Test - public void testDoesNotFireForEmpty() + @Test(dataProvider = "delegating-row-expression-optimizer-enabled") + public void testDoesNotFireForEmpty(boolean enableDelegatingRowExpressionOptimizer) { tester().assertThat(new RewriteConstantArrayContainsToInExpression(getFunctionManager()).projectRowExpressionRewriteRule()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") @@ -97,12 +113,14 @@ public void testDoesNotFireForEmpty() .doesNotFire(); } - @Test - public void testNotFire() + @Test(dataProvider = "delegating-row-expression-optimizer-enabled") + public void testNotFire(boolean enableDelegatingRowExpressionOptimizer) { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll( - new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()).build()) + ImmutableSet.>builder() + .addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager(), new FeaturesConfig()).rules()) + .addAll(new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()) + .build()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") .on(p -> { VariableReferenceExpression a = p.variable("a", BOOLEAN); @@ -118,12 +136,14 @@ public void testNotFire() values("b", "c"))); } - @Test - public void testWithNull() + @Test(dataProvider = "delegating-row-expression-optimizer-enabled") + public void testWithNull(boolean enableDelegatingRowExpressionOptimizer) { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll( - new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()).build()) + ImmutableSet.>builder() + .addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager(), new FeaturesConfig()).rules()) + .addAll(new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()) + .build()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") .on(p -> { VariableReferenceExpression a = p.variable("a", BOOLEAN); @@ -138,12 +158,14 @@ public void testWithNull() values("b"))); } - @Test - public void testLambda() + @Test(dataProvider = "delegating-row-expression-optimizer-enabled") + public void testLambda(boolean enableDelegatingRowExpressionOptimizer) { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll( - new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()).build()) + ImmutableSet.>builder() + .addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager(), new FeaturesConfig()).rules()) + .addAll(new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()) + .build()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") .on(p -> { VariableReferenceExpression a = p.variable("a", BOOLEAN); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java index 96da619e3ccb..d81aa041bc24 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java @@ -13,15 +13,20 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.Session; import com.facebook.presto.common.type.Type; import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.expressions.RowExpressionRewriter; import com.facebook.presto.expressions.RowExpressionTreeRewriter; +import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Expression; @@ -36,6 +41,7 @@ import java.util.stream.Stream; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.SystemSessionProperties.DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; @@ -43,6 +49,8 @@ import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.OR; import static com.facebook.presto.sql.ExpressionUtils.rewriteIdentifiersToSymbolReferences; import static com.facebook.presto.sql.relational.Expressions.specialForm; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static java.lang.String.format; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; @@ -181,12 +189,24 @@ private static void assertSimplifies(String expression, String rowExpressionExpe { Expression actualExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(expression)); + InMemoryNodeManager nodeManager = new InMemoryNodeManager(); + ExpressionOptimizerManager expressionOptimizerManager = new ExpressionOptimizerManager(new PluginNodeManager(nodeManager), METADATA.getFunctionAndTypeManager()); + expressionOptimizerManager.loadExpressionOptimizerFactory(); + TestingRowExpressionTranslator translator = new TestingRowExpressionTranslator(METADATA); RowExpression actualRowExpression = translator.translate(actualExpression, TypeProvider.viewOf(TYPES)); - RowExpression simplifiedRowExpression = SimplifyRowExpressions.rewrite(actualRowExpression, METADATA, TEST_SESSION.toConnectorSession()); + RowExpression simplifiedRowExpression = SimplifyRowExpressions.rewrite(actualRowExpression, METADATA, TEST_SESSION, expressionOptimizerManager, new FeaturesConfig()); Expression expectedByRowExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(rowExpressionExpected)); RowExpression simplifiedByExpression = translator.translate(expectedByRowExpression, TypeProvider.viewOf(TYPES)); assertEquals(normalize(simplifiedRowExpression), normalize(simplifiedByExpression)); + + Session session = testSessionBuilder() + .setCatalog("tpch") + .setSchema(TINY_SCHEMA_NAME) + .setSystemProperty(DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED, "true") + .build(); + RowExpression sidecarSimplifiedExpressions = SimplifyRowExpressions.rewrite(actualRowExpression, METADATA, session, expressionOptimizerManager, new FeaturesConfig()); + assertEquals(normalize(sidecarSimplifiedExpressions), normalize(simplifiedByExpression)); } private static RowExpression normalize(RowExpression expression) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java index 04ce733e96aa..0dafb0c96b8b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java @@ -16,6 +16,7 @@ import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.Plugin; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.Plan; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; @@ -84,4 +85,9 @@ protected void assertNodePresentInPlan(Plan plan, Class nodeClass) .matches(), "Expected " + nodeClass.toString() + " in plan after optimization. "); } + + protected ExpressionOptimizerManager getExpressionManager() + { + return tester.getExpressionManager(); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java index 3c9540ef42b2..8783a6125e1e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java @@ -26,6 +26,7 @@ import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.assertions.OptimizerAssert; @@ -61,6 +62,7 @@ public class RuleTester private final PageSourceManager pageSourceManager; private final AccessControl accessControl; private final SqlParser sqlParser; + private ExpressionOptimizerManager expressionOptimizerManager; public RuleTester() { @@ -107,6 +109,8 @@ public RuleTester(List plugins, Map sessionProperties, S connectorFactory, ImmutableMap.of()); plugins.stream().forEach(queryRunner::installPlugin); + expressionOptimizerManager = queryRunner.getExpressionManager(); + expressionOptimizerManager.loadExpressionOptimizerFactory(); this.metadata = queryRunner.getMetadata(); this.transactionManager = queryRunner.getTransactionManager(); @@ -197,4 +201,9 @@ public List> getTableConstraints(TableHandle table return metadata.getTableMetadata(transactionSession, tableHandle).getMetadata().getTableConstraintsHolder().getTableConstraintsWithColumnHandles(); }); } + + public ExpressionOptimizerManager getExpressionManager() + { + return expressionOptimizerManager; + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestCteProjectionAndPredicatePushdown.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestCteProjectionAndPredicatePushdown.java index f36c3aaabcf1..0e73b6b3d551 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestCteProjectionAndPredicatePushdown.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestCteProjectionAndPredicatePushdown.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.sql.Optimizer; +import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; @@ -144,7 +145,7 @@ private void assertCtePlan(String sql, PlanMatchPattern pattern) new RemoveIdentityProjectionsBelowProjection(), new PruneRedundantProjectionAssignments())), new PruneUnreferencedOutputs(), - new CteProjectionAndPredicatePushDown(metadata)); + new CteProjectionAndPredicatePushDown(metadata, getQueryRunner().getExpressionManager(), new FeaturesConfig())); assertPlan(sql, getSession(), Optimizer.PlanStage.OPTIMIZED, pattern, optimizers); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDelegatingRowExpressionOptimizer.java b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDelegatingRowExpressionOptimizer.java new file mode 100644 index 000000000000..4d1b3606068d --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/relational/TestDelegatingRowExpressionOptimizer.java @@ -0,0 +1,223 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.relational; + +import com.facebook.presto.common.type.Type; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.EVALUATED; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.SERIALIZABLE; +import static com.facebook.presto.sql.planner.LiteralEncoder.toRowExpression; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.testing.TestingConnectorSession.SESSION; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; +import static org.testng.Assert.assertEquals; + +public class TestDelegatingRowExpressionOptimizer +{ + private DelegatingRowExpressionOptimizer optimizer; + private static final MetadataManager METADATA = MetadataManager.createTestMetadataManager(); + + private static final TestingRowExpressionTranslator TRANSLATOR = new TestingRowExpressionTranslator(METADATA); + + @BeforeClass + public void setUp() + { + optimizer = new DelegatingRowExpressionOptimizer(METADATA, InnerOptimizer::new, 3); + } + + @AfterClass(alwaysRun = true) + public void tearDown() + { + optimizer = null; + } + + @Test + public void testBasicExpressions() + { + assertEquals(optimize(expression("BIGINT'1' + 1")), expression("BIGINT'2'")); + assertEquals(optimize(expression("IF(TRUE, 1, 2)")), expression("1")); + } + + @Test + public void testVariableReference() + { + VariableReferenceExpression variable = new VariableReferenceExpression(Optional.empty(), "x", BIGINT); + assertEquals(optimize(variable), variable); + ImmutableMap typeMap = ImmutableMap.of("x", BIGINT); + assertEquals(optimize(expression("x + 1", typeMap)), expression("x + 1", typeMap)); + assertEquals(optimize(expression("x + 1", typeMap), ImmutableMap.of(variable, 1L)), expression("BIGINT'2'")); + } + + @Test + public void testComplexExpressions() + { + assertEquals(optimize(expression("IF(TRUE, 1, 2) + 3")), expression("4")); + assertEquals(optimize(expression("IF(TRUE, 1, 2) + 3 + 4")), expression("8")); + assertEquals(optimize(expression("IF(TRUE, 1, 2) + 3 + 4 + 5")), expression("8 + 5")); + + VariableReferenceExpression variable = new VariableReferenceExpression(Optional.empty(), "x", INTEGER); + ImmutableMap typeMap = ImmutableMap.of("x", INTEGER); + assertEquals(optimize(expression("IF(TRUE, 1, 2) + x", typeMap), ImmutableMap.of(variable, 3L)), expression("4")); + assertEquals(optimize(expression("IF(TRUE, 1, 2) + x + 4", typeMap), ImmutableMap.of(variable, 3L)), expression("8")); + assertEquals(optimize(expression("IF(TRUE, 1, 2) + x + 4 + 5", typeMap), ImmutableMap.of(variable, 3L)), expression("8 + 5")); + } + + @Test + public void testDifferentOptimizationLevels() + { + assertEquals(optimize(expression("rand(10) + 1 > 0"), EVALUATED), expression("true")); + assertEquals(optimize(expression("rand(10) + 1 > 0"), OPTIMIZED), expression("rand(10) + 1 > 0")); + assertEquals(optimize(expression("rand(10) + 1 > 0"), SERIALIZABLE), expression("rand(10) + 1 > 0")); + } + + private Object optimize(RowExpression expression, Map variableMap) + { + return optimize(expression, variableMap, OPTIMIZED); + } + + private Object optimize(RowExpression expression, Map variableMap, Level level) + { + return optimizer.optimize(expression, level, SESSION, variableMap::get); + } + + private RowExpression optimize(RowExpression expression) + { + return optimize(expression, OPTIMIZED); + } + + private RowExpression optimize(RowExpression expression, Level level) + { + return optimizer.optimize(expression, level, SESSION); + } + + private static RowExpression expression(String expressionSql) + { + return expression(expressionSql, ImmutableMap.of()); + } + + private static RowExpression expression(String expressionSql, Map typeMap) + { + return TRANSLATOR.translate(expressionSql, typeMap); + } + + private static class InnerOptimizer + implements ExpressionOptimizer + { + @Override + public RowExpression optimize(RowExpression rowExpression, Level level, ConnectorSession session) + { + OneLevelDeepExpressionRewriter rewriter = new OneLevelDeepExpressionRewriter(level, variable -> variable); + return rowExpression.accept(rewriter, null); + } + + @Override + public Object optimize(RowExpression expression, Level level, ConnectorSession session, Function variableResolver) + { + OneLevelDeepExpressionRewriter rewriter = new OneLevelDeepExpressionRewriter(level, variableResolver); + return expression.accept(rewriter, null); + } + } + + // This visitor will only rewrite the first expression it comes across. It is intended to be used to test + // the DelegatingRowExpressionOptimizer, which will call the inner optimizer multiple times. + private static class OneLevelDeepExpressionRewriter + implements RowExpressionVisitor + { + private final RowExpressionOptimizer innerOptimizer = new RowExpressionOptimizer(METADATA); + private final Level level; + private final Function variableResolver; + + private boolean rewritten; + + public OneLevelDeepExpressionRewriter(Level level, Function variableResolver) + { + this.level = level; + this.variableResolver = requireNonNull(variableResolver, "variableResolver is null"); + } + + @Override + public RowExpression visitExpression(RowExpression node, Void context) + { + return node; + } + + @Override + public RowExpression visitVariableReference(VariableReferenceExpression reference, Void context) + { + if (variableResolver == null) { + return reference; + } + Object value = variableResolver.apply(reference); + if (value == null) { + return reference; + } + return toRowExpression(value, reference.getType()); + } + + @Override + public RowExpression visitCall(CallExpression call, Void context) + { + List arguments = call.getArguments().stream() + .map(argument -> argument.accept(this, context)) + .collect(toImmutableList()); + if (!rewritten) { + RowExpression rewritten = toRowExpression(innerOptimizer.optimize(call, level, SESSION, variableResolver), call.getType()); + if (!rewritten.equals(call)) { + this.rewritten = true; + return rewritten; + } + } + return call(call.getDisplayName(), call.getFunctionHandle(), call.getType(), arguments); + } + + @Override + public RowExpression visitSpecialForm(SpecialFormExpression specialForm, Void context) + { + List arguments = specialForm.getArguments().stream() + .map(argument -> argument.accept(this, context)) + .collect(toImmutableList()); + if (!rewritten) { + RowExpression rewritten = toRowExpression(innerOptimizer.optimize(specialForm, OPTIMIZED, SESSION, variableResolver), specialForm.getType()); + if (!rewritten.equals(specialForm)) { + this.rewritten = true; + return rewritten; + } + } + return new SpecialFormExpression(specialForm.getForm(), specialForm.getType(), arguments); + } + } +} diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunner.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunner.java index d32eea55c392..c65afda86cf1 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunner.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunner.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; @@ -230,6 +231,12 @@ public TestingAccessControlManager getAccessControl() throw new UnsupportedOperationException(); } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + throw new UnsupportedOperationException(); + } + @Override public MaterializedResult execute(String sql) { diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java index ed017e654cce..b84557391a8b 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java @@ -80,6 +80,7 @@ import com.facebook.presto.metadata.StaticFunctionNamespaceStore; import com.facebook.presto.metadata.StaticFunctionNamespaceStoreConfig; import com.facebook.presto.metadata.TablePropertyManager; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.operator.FileFragmentResultCacheConfig; import com.facebook.presto.operator.FileFragmentResultCacheManager; import com.facebook.presto.operator.FragmentCacheStats; @@ -173,6 +174,7 @@ import com.facebook.presto.sql.analyzer.MetadataExtractorMBean; import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -346,6 +348,9 @@ protected void setup(Binder binder) binder.bind(AnalyzePropertyManager.class).in(Scopes.SINGLETON); binder.bind(QuerySessionSupplier.class).in(Scopes.SINGLETON); + // expression manager + binder.bind(ExpressionOptimizerManager.class).in(Scopes.SINGLETON); + // tracer provider managers binder.bind(TracerProviderManager.class).in(Scopes.SINGLETON); @@ -508,6 +513,7 @@ protected void setup(Binder binder) // TODO: Decouple and remove: required by ConnectorManager binder.bind(InternalNodeManager.class).toInstance(new PrestoSparkInternalNodeManager()); + binder.bind(PluginNodeManager.class); // TODO: Decouple and remove: required by PluginManager binder.bind(InternalResourceGroupManager.class).in(Scopes.SINGLETON); diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java index f85c08ef58eb..2ddde52dfa79 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java @@ -64,6 +64,7 @@ import com.facebook.presto.spi.security.PrincipalType; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; @@ -504,6 +505,12 @@ public TestingAccessControlManager getAccessControl() return testingAccessControlManager; } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + throw new UnsupportedOperationException(); + } + public HistoryBasedPlanStatisticsManager getHistoryBasedPlanStatisticsManager() { return historyBasedPlanStatisticsManager; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizerProvider.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizerProvider.java new file mode 100644 index 000000000000..3ffde03a0d6e --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizerProvider.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.relation; + +public interface ExpressionOptimizerProvider +{ + ExpressionOptimizer getExpressionOptimizer(); +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java index 12710f71b0f7..0d2a5545c2e9 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java @@ -19,6 +19,7 @@ * A set of services/utilities that are helpful for connectors to operate on row expressions */ public interface RowExpressionService + extends ExpressionOptimizerProvider { DomainTranslator getDomainTranslator(); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java index 303ae0dfe780..bedd15a5d9e1 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.tests; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.Session; import com.facebook.presto.common.type.Type; import com.facebook.presto.cost.CostCalculator; @@ -21,11 +22,14 @@ import com.facebook.presto.cost.CostComparator; import com.facebook.presto.cost.TaskCountEstimator; import com.facebook.presto.execution.QueryManagerConfig; +import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.security.AccessDeniedException; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.QueryExplainer; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.Plan; @@ -74,6 +78,7 @@ public abstract class AbstractTestQueryFramework { + private static final NodeInfo NODE_INFO = new NodeInfo("test"); private QueryRunner queryRunner; private ExpectedQueryRunner expectedQueryRunner; private SqlParser sqlParser; @@ -568,7 +573,10 @@ private QueryExplainer getQueryExplainer() new CostComparator(featuresConfig), taskCountEstimator, new PartitioningProviderManager(), - featuresConfig) + featuresConfig, + new ExpressionOptimizerManager( + new PluginNodeManager(new InMemoryNodeManager()), + queryRunner.getMetadata().getFunctionAndTypeManager())) .getPlanningTimeOptimizers(); return new QueryExplainer( optimizers, diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index 62502743809b..5674d59e752b 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -42,6 +42,7 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; @@ -626,6 +627,13 @@ public PlanCheckerProviderManager getPlanCheckerProviderManager() return coordinators.get(0).getPlanCheckerProviderManager(); } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + checkState(coordinators.size() == 1, "Expected a single coordinator"); + return coordinators.get(0).getExpressionManager(); + } + public TestingPrestoServer getCoordinator() { checkState(coordinators.size() == 1, "Expected a single coordinator"); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java index 980fb4f991bc..00993ae05495 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java @@ -27,6 +27,7 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; @@ -188,6 +189,12 @@ public TestingAccessControlManager getAccessControl() return server.getAccessControl(); } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + return server.getExpressionManager(); + } + public TestingPrestoServer getServer() { return server; diff --git a/presto-tests/src/test/java/com/facebook/presto/memory/TestMemoryManager.java b/presto-tests/src/test/java/com/facebook/presto/memory/TestMemoryManager.java index 30951ad186cb..19d22733f67a 100644 --- a/presto-tests/src/test/java/com/facebook/presto/memory/TestMemoryManager.java +++ b/presto-tests/src/test/java/com/facebook/presto/memory/TestMemoryManager.java @@ -512,7 +512,7 @@ public void clusterPoolsMultiCoordinatorCleanup() queryRunner2.close(); } - @Test(timeOut = 60_000, groups = {"clusterPoolsMultiCoordinator"}) + @Test(timeOut = 600_000, groups = {"clusterPoolsMultiCoordinator"}) public void testClusterPoolsMultiCoordinator() throws Exception { @@ -544,6 +544,7 @@ public void testClusterPoolsMultiCoordinator() generalPool = memoryManager.getClusterInfo(GENERAL_POOL); reservedPool = memoryManager.getClusterInfo(RESERVED_POOL); MILLISECONDS.sleep(10); + System.out.println("waiting"); } // Make sure the queries are blocked diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestDelegatingExpressionOptimizer.java b/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestDelegatingExpressionOptimizer.java new file mode 100644 index 000000000000..616feccc3d7c --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestDelegatingExpressionOptimizer.java @@ -0,0 +1,146 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests.expressions; + +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.relational.DelegatingRowExpressionOptimizer; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.google.common.collect.ImmutableList; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.EVALUATED; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.type.LikeFunctions.castVarcharToLikePattern; +import static com.facebook.presto.type.LikePatternType.LIKE_PATTERN; +import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.slice.Slices.wrappedBuffer; +import static org.testng.Assert.assertEquals; + +public class TestDelegatingExpressionOptimizer + extends TestExpressions +{ + private FunctionResolution resolution; + private ExpressionOptimizer expressionOptimizer; + + @BeforeClass + public void setup() + { + expressionOptimizer = new DelegatingRowExpressionOptimizer(getMetadata(), new InMemoryExpressionOptimizerProvider(getMetadata())); + resolution = new FunctionResolution(getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + } + + @Test + public void assertLikeOptimizations() + { + assertOptimizedMatches("unbound_string LIKE bound_pattern", "unbound_string LIKE CAST('%el%' AS varchar)"); + } + + @Override + protected void assertLike(byte[] value, String pattern, boolean expected) + { + CallExpression predicate = call( + "LIKE", + resolution.likeVarcharFunction(), + BOOLEAN, + ImmutableList.of( + new ConstantExpression(wrappedBuffer(value), VARCHAR), + new ConstantExpression(castVarcharToLikePattern(utf8Slice(pattern)), LIKE_PATTERN))); + assertEquals(optimizeRowExpression(predicate, EVALUATED), expected); + } + @Override + protected Object evaluate(String expression, boolean deterministic) + { + assertRoundTrip(expression); + RowExpression rowExpression = sqlToRowExpression(expression); + return optimizeRowExpression(rowExpression, EVALUATED); + } + + @Override + protected Object optimize(@Language("SQL") String expression) + { + assertRoundTrip(expression); + RowExpression parsedExpression = sqlToRowExpression(expression); + return optimizeRowExpression(parsedExpression, OPTIMIZED); + } + + @Override + protected Object optimizeRowExpression(RowExpression expression, Level level) + { + Object optimized = expressionOptimizer.optimize( + expression, + level, + TEST_SESSION.toConnectorSession(), + variable -> { + Symbol symbol = new Symbol(variable.getName()); + Object value = symbolConstant(symbol); + if (value == null) { + return new VariableReferenceExpression(Optional.empty(), symbol.getName(), SYMBOL_TYPES.get(symbol.toSymbolReference())); + } + return value; + }); + return unwrap(optimized); + } + + public Object unwrap(Object result) + { + if (result instanceof ConstantExpression) { + return ((ConstantExpression) result).getValue(); + } + else { + return result; + } + } + + @Override + protected void assertOptimizedEquals(@Language("SQL") String actual, @Language("SQL") String expected) + { + Object optimizedActual = optimize(actual); + Object optimizedExpected = optimize(expected); + assertRowExpressionEvaluationEquals(optimizedActual, optimizedExpected); + } + + @Override + protected void assertOptimizedMatches(@Language("SQL") String actual, @Language("SQL") String expected) + { + Object actualOptimized = optimize(actual); + Object expectedOptimized = optimize(expected); + assertRowExpressionEvaluationEquals( + actualOptimized, + expectedOptimized); + } + + @Override + protected void assertDoNotOptimize(@Language("SQL") String expression, Level optimizationLevel) + { + assertRoundTrip(expression); + RowExpression rowExpression = sqlToRowExpression(expression); + Object rowExpressionResult = optimizeRowExpression(rowExpression, optimizationLevel); + assertRowExpressionEvaluationEquals(rowExpressionResult, rowExpression); + } +} diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressionInterpreter.java b/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressionInterpreter.java new file mode 100644 index 000000000000..17534292b5c2 --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressionInterpreter.java @@ -0,0 +1,225 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests.expressions; + +import com.facebook.presto.common.type.Type; +import com.facebook.presto.operator.scalar.FunctionAssertions; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.facebook.presto.sql.planner.ExpressionInterpreter; +import com.facebook.presto.sql.planner.RowExpressionInterpreter; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.ExpressionRewriter; +import com.facebook.presto.sql.tree.ExpressionTreeRewriter; +import com.facebook.presto.sql.tree.FunctionCall; +import com.facebook.presto.sql.tree.LikePredicate; +import com.facebook.presto.sql.tree.NodeRef; +import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.presto.sql.tree.StringLiteral; +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; +import static com.facebook.presto.sql.ExpressionUtils.rewriteIdentifiersToSymbolReferences; +import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; +import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionInterpreter; +import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionOptimizer; +import static com.facebook.presto.sql.planner.RowExpressionInterpreter.rowExpressionInterpreter; +import static java.util.Collections.emptyMap; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestExpressionInterpreter + extends TestExpressions +{ + private final TestingRowExpressionTranslator translator = new TestingRowExpressionTranslator(getMetadata()); + + @Test + public void assertLikeOptimizations() + { + assertOptimizedEquals("unbound_string LIKE bound_pattern", "unbound_string LIKE bound_pattern"); + } + + @Override + protected void assertLike(byte[] value, String pattern, boolean expected) + { + Expression predicate = new LikePredicate( + rawStringLiteral(Slices.wrappedBuffer(value)), + new StringLiteral(pattern), + Optional.empty()); + assertEquals(evaluate(predicate, true), expected); + } + + private static StringLiteral rawStringLiteral(final Slice slice) + { + return new StringLiteral(slice.toStringUtf8()) + { + @Override + public Slice getSlice() + { + return slice; + } + }; + } + + @Override + protected void assertOptimizedEquals(@Language("SQL") String actual, @Language("SQL") String expected) + { + assertEquals(optimize(actual), optimize(expected)); + } + + @Override + protected void assertOptimizedMatches(@Language("SQL") String actual, @Language("SQL") String expected) + { + // replaces FunctionCalls to FailureFunction by fail() + Object actualOptimized = optimize(actual); + if (actualOptimized instanceof Expression) { + actualOptimized = ExpressionTreeRewriter.rewriteWith(new FailedFunctionRewriter(), (Expression) actualOptimized); + } + assertEquals( + actualOptimized, + rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(expected))); + } + + @Override + protected Object optimize(@Language("SQL") String expression) + { + assertRoundTrip(expression); + + Expression parsedExpression = expression(expression); + Object expressionResult = optimize(parsedExpression); + + RowExpression rowExpression = toRowExpression(parsedExpression); + Object rowExpressionResult = optimizeRowExpression(rowExpression, OPTIMIZED); + assertExpressionAndRowExpressionEquals(expressionResult, rowExpressionResult); + return expressionResult; + } + + @Override + protected Object optimizeRowExpression(RowExpression expression, ExpressionOptimizer.Level level) + { + RowExpressionInterpreter rowExpressionInterpreter = new RowExpressionInterpreter(expression, getMetadata(), TEST_SESSION.toConnectorSession(), level); + return rowExpressionInterpreter.optimize(variable -> { + Symbol symbol = new Symbol(variable.getName()); + Object value = symbolConstant(symbol); + if (value == null) { + return new VariableReferenceExpression(Optional.empty(), symbol.getName(), SYMBOL_TYPES.get(symbol.toSymbolReference())); + } + return value; + }); + } + + private Expression expression(String expression) + { + return FunctionAssertions.createExpression(expression, getMetadata(), SYMBOL_TYPES); + } + + private RowExpression toRowExpression(Expression expression) + { + return translator.translate(expression, SYMBOL_TYPES); + } + + private Object optimize(Expression expression) + { + Map, Type> expressionTypes = getExpressionTypes(TEST_SESSION, getMetadata(), SQL_PARSER, SYMBOL_TYPES, expression, emptyMap(), WarningCollector.NOOP); + ExpressionInterpreter interpreter = expressionOptimizer(expression, getMetadata(), TEST_SESSION, expressionTypes); + return interpreter.optimize(variable -> { + Symbol symbol = new Symbol(variable.getName()); + Object value = symbolConstant(symbol); + if (value == null) { + return symbol.toSymbolReference(); + } + return value; + }); + } + + @Override + protected void assertDoNotOptimize(@Language("SQL") String expression, Level optimizationLevel) + { + assertRoundTrip(expression); + Expression translatedExpression = expression(expression); + RowExpression rowExpression = toRowExpression(translatedExpression); + + Object expressionResult = optimize(translatedExpression); + if (expressionResult instanceof Expression) { + expressionResult = toRowExpression((Expression) expressionResult); + } + Object rowExpressionResult = optimizeRowExpression(rowExpression, optimizationLevel); + assertRowExpressionEvaluationEquals(expressionResult, rowExpressionResult); + assertRowExpressionEvaluationEquals(rowExpressionResult, rowExpression); + } + + private void assertExpressionAndRowExpressionEquals(Object expressionResult, Object rowExpressionResult) + { + if (rowExpressionResult instanceof RowExpression) { + // Cannot be completely evaluated into a constant; compare expressions + assertTrue(expressionResult instanceof Expression); + + // It is tricky to check the equivalence of an expression and a row expression. + // We rely on the optimized translator to fill the gap. + RowExpression translated = translator.translateAndOptimize((Expression) expressionResult, SYMBOL_TYPES); + assertRowExpressionEvaluationEquals(translated, rowExpressionResult); + } + else { + // We have constants; directly compare + assertRowExpressionEvaluationEquals(expressionResult, rowExpressionResult); + } + } + @Override + protected Object evaluate(String expression, boolean deterministic) + { + assertRoundTrip(expression); + + Expression parsedExpression = FunctionAssertions.createExpression(expression, getMetadata(), SYMBOL_TYPES); + + return evaluate(parsedExpression, deterministic); + } + + private Object evaluate(Expression expression, boolean deterministic) + { + Map, Type> expressionTypes = getExpressionTypes(TEST_SESSION, getMetadata(), SQL_PARSER, SYMBOL_TYPES, expression, emptyMap(), WarningCollector.NOOP); + Object expressionResult = expressionInterpreter(expression, getMetadata(), TEST_SESSION, expressionTypes).evaluate(); + Object rowExpressionResult = rowExpressionInterpreter(translator.translateAndOptimize(expression), getMetadata().getFunctionAndTypeManager(), TEST_SESSION.toConnectorSession()).evaluate(); + + if (deterministic) { + assertExpressionAndRowExpressionEquals(expressionResult, rowExpressionResult); + } + return expressionResult; + } + + private static class FailedFunctionRewriter + extends ExpressionRewriter + { + @Override + public Expression rewriteFunctionCall(FunctionCall node, Object context, ExpressionTreeRewriter treeRewriter) + { + if (node.getName().equals(QualifiedName.of("fail"))) { + return new FunctionCall(QualifiedName.of("fail"), ImmutableList.of(node.getArguments().get(0), new StringLiteral("ignored failure message"))); + } + return node; + } + } +} diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressionOptimizers.java b/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressionOptimizers.java new file mode 100644 index 000000000000..48add5a17fd8 --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressionOptimizers.java @@ -0,0 +1,133 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests.expressions; + +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; +import com.google.common.collect.ImmutableList; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.EVALUATED; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.type.LikeFunctions.castVarcharToLikePattern; +import static com.facebook.presto.type.LikePatternType.LIKE_PATTERN; +import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.slice.Slices.wrappedBuffer; +import static org.testng.Assert.assertEquals; + +public class TestExpressionOptimizers + extends TestExpressions +{ + private final FunctionResolution resolution = new FunctionResolution(getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + private ExpressionOptimizer expressionOptimizer; + + @BeforeClass + public void setup() + { + expressionOptimizer = new RowExpressionOptimizer(getMetadata().getFunctionAndTypeManager()); + } + + @Test + public void assertLikeOptimizations() + { + assertOptimizedMatches("unbound_string LIKE bound_pattern", "unbound_string LIKE CAST('%el%' AS varchar)"); + } + + @Override + protected void assertLike(byte[] value, String pattern, boolean expected) + { + CallExpression predicate = call( + "LIKE", + resolution.likeVarcharFunction(), + BOOLEAN, + ImmutableList.of( + new ConstantExpression(wrappedBuffer(value), VARCHAR), + new ConstantExpression(castVarcharToLikePattern(utf8Slice(pattern)), LIKE_PATTERN))); + assertEquals(optimizeRowExpression(predicate, EVALUATED), expected); + } + @Override + protected Object evaluate(String expression, boolean deterministic) + { + assertRoundTrip(expression); + RowExpression rowExpression = sqlToRowExpression(expression); + return optimizeRowExpression(rowExpression, EVALUATED); + } + + @Override + protected Object optimize(@Language("SQL") String expression) + { + assertRoundTrip(expression); + RowExpression parsedExpression = sqlToRowExpression(expression); + return optimizeRowExpression(parsedExpression, OPTIMIZED); + } + + @Override + protected Object optimizeRowExpression(RowExpression expression, ExpressionOptimizer.Level level) + { + return expressionOptimizer.optimize( + expression, + level, + TEST_SESSION.toConnectorSession(), + variable -> { + Symbol symbol = new Symbol(variable.getName()); + Object value = symbolConstant(symbol); + if (value == null) { + return new VariableReferenceExpression(Optional.empty(), symbol.getName(), SYMBOL_TYPES.get(symbol.toSymbolReference())); + } + return value; + }); + } + + @Override + protected void assertOptimizedEquals(@Language("SQL") String actual, @Language("SQL") String expected) + { + Object optimizedActual = optimize(actual); + Object optimizedExpected = optimize(expected); + assertEquals(optimizedActual, optimizedExpected); + } + + @Override + protected void assertOptimizedMatches(@Language("SQL") String actual, @Language("SQL") String expected) + { + Object actualOptimized = optimize(actual); + Object expectedOptimized = optimize(expected); + assertRowExpressionEvaluationEquals( + actualOptimized, + expectedOptimized); + } + + @Override + protected void assertDoNotOptimize(@Language("SQL") String expression, Level optimizationLevel) + { + assertRoundTrip(expression); + RowExpression rowExpression = sqlToRowExpression(expression); + Object rowExpressionResult = optimizeRowExpression(rowExpression, optimizationLevel); + assertRowExpressionEvaluationEquals(rowExpressionResult, rowExpression); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java b/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressions.java similarity index 87% rename from presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java rename to presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressions.java index 37295146c429..a932a099c73e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/expressions/TestExpressions.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql; +package com.facebook.presto.tests.expressions; import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; @@ -24,14 +24,11 @@ import com.facebook.presto.common.type.SqlTimestampWithTimeZone; import com.facebook.presto.common.type.StandardTypes; import com.facebook.presto.common.type.Type; -import com.facebook.presto.common.type.VarbinaryType; import com.facebook.presto.functionNamespace.json.JsonFileBasedFunctionNamespaceManagerFactory; import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; -import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.operator.scalar.FunctionAssertions; import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.function.AggregationFunctionMetadata; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.Parameter; @@ -39,26 +36,19 @@ import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.InputReferenceExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.planner.ExpressionInterpreter; -import com.facebook.presto.sql.planner.RowExpressionInterpreter; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.tree.Expression; -import com.facebook.presto.sql.tree.ExpressionRewriter; -import com.facebook.presto.sql.tree.ExpressionTreeRewriter; -import com.facebook.presto.sql.tree.FunctionCall; -import com.facebook.presto.sql.tree.LikePredicate; -import com.facebook.presto.sql.tree.NodeRef; -import com.facebook.presto.sql.tree.QualifiedName; -import com.facebook.presto.sql.tree.StringLiteral; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -71,11 +61,9 @@ import org.joda.time.DateTimeZone; import org.joda.time.LocalDate; import org.joda.time.LocalTime; -import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.math.BigInteger; -import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -91,37 +79,33 @@ import static com.facebook.presto.common.type.TimeZoneKey.getTimeZoneKey; import static com.facebook.presto.common.type.TimestampType.TIMESTAMP; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.common.type.VarcharType.createVarcharType; +import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; import static com.facebook.presto.operator.scalar.ApplyFunction.APPLY_FUNCTION; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.function.FunctionVersion.notVersioned; import static com.facebook.presto.spi.function.RoutineCharacteristics.Determinism.DETERMINISTIC; import static com.facebook.presto.spi.function.RoutineCharacteristics.Language.CPP; -import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.SERIALIZABLE; import static com.facebook.presto.sql.ExpressionFormatter.formatExpression; -import static com.facebook.presto.sql.ExpressionUtils.rewriteIdentifiersToSymbolReferences; -import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; -import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionInterpreter; -import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionOptimizer; -import static com.facebook.presto.sql.planner.RowExpressionInterpreter.rowExpressionInterpreter; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.util.AnalyzerUtil.createParsingOptions; import static com.facebook.presto.util.DateTimeZoneIndex.getDateTimeZone; import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.slice.Slices.wrappedBuffer; import static java.lang.String.format; -import static java.util.Collections.emptyMap; import static java.util.Locale.ENGLISH; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -public class TestExpressionInterpreter +public abstract class TestExpressions { - public static final SqlInvokedFunction SQUARE_UDF_CPP = new SqlInvokedFunction( + private static final SqlInvokedFunction SQUARE_UDF_CPP = new SqlInvokedFunction( QualifiedObjectName.valueOf(new CatalogSchemaName("json", "test_schema"), "square"), ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.BIGINT))), parseTypeSignature(StandardTypes.BIGINT), @@ -130,7 +114,7 @@ public class TestExpressionInterpreter "", notVersioned()); - public static final SqlInvokedFunction AVG_UDAF_CPP = new SqlInvokedFunction( + private static final SqlInvokedFunction AVG_UDAF_CPP = new SqlInvokedFunction( QualifiedObjectName.valueOf(new CatalogSchemaName("json", "test_schema"), "avg"), ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.DOUBLE))), parseTypeSignature(StandardTypes.DOUBLE), @@ -142,11 +126,11 @@ public class TestExpressionInterpreter Optional.of(new AggregationFunctionMetadata(parseTypeSignature("ROW(double, int)"), false))); private static final int TEST_VARCHAR_TYPE_LENGTH = 17; - private static final TypeProvider SYMBOL_TYPES = TypeProvider.viewOf(ImmutableMap.builder() + protected static final TypeProvider SYMBOL_TYPES = TypeProvider.viewOf(ImmutableMap.builder() .put("bound_integer", INTEGER) .put("bound_long", BIGINT) .put("bound_string", createVarcharType(TEST_VARCHAR_TYPE_LENGTH)) - .put("bound_varbinary", VarbinaryType.VARBINARY) + .put("bound_varbinary", VARBINARY) .put("bound_double", DOUBLE) .put("bound_boolean", BOOLEAN) .put("bound_date", DATE) @@ -173,16 +157,20 @@ public class TestExpressionInterpreter .put("unbound_null_string", VARCHAR) .build()); - private static final SqlParser SQL_PARSER = new SqlParser(); - private static final Metadata METADATA = MetadataManager.createTestMetadataManager(); - private static final TestingRowExpressionTranslator TRANSLATOR = new TestingRowExpressionTranslator(METADATA); - private static final BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(); + protected static final SqlParser SQL_PARSER = new SqlParser(); + private final Metadata metadata = createTestMetadataManager(); + private final TestingRowExpressionTranslator translator = new TestingRowExpressionTranslator(metadata); + private static final BlockEncodingSerde BLOCK_ENCODING_SERDE = new BlockEncodingManager(); - @BeforeClass - public void setup() + public TestExpressions() { - METADATA.getFunctionAndTypeManager().registerBuiltInFunctions(ImmutableList.of(APPLY_FUNCTION)); - setupJsonFunctionNamespaceManager(METADATA.getFunctionAndTypeManager()); + metadata.getFunctionAndTypeManager().registerBuiltInFunctions(ImmutableList.of(APPLY_FUNCTION)); + setupJsonFunctionNamespaceManager(metadata.getFunctionAndTypeManager()); + } + + public Metadata getMetadata() + { + return metadata; } @Test @@ -415,19 +403,19 @@ public void testNonDeterministicFunctionCall() @Test public void testCppFunctionCall() { - METADATA.getFunctionAndTypeManager().createFunction(SQUARE_UDF_CPP, false); + metadata.getFunctionAndTypeManager().createFunction(SQUARE_UDF_CPP, false); assertOptimizedEquals("json.test_schema.square(-5)", "json.test_schema.square(-5)"); } @Test public void testCppAggregateFunctionCall() { - METADATA.getFunctionAndTypeManager().createFunction(AVG_UDAF_CPP, false); + metadata.getFunctionAndTypeManager().createFunction(AVG_UDAF_CPP, false); assertOptimizedEquals("json.test_schema.avg(1.0)", "json.test_schema.avg(1.0)"); } // Run this method exactly once. - private void setupJsonFunctionNamespaceManager(FunctionAndTypeManager functionAndTypeManager) + protected void setupJsonFunctionNamespaceManager(FunctionAndTypeManager functionAndTypeManager) { functionAndTypeManager.addFunctionNamespaceFactory(new JsonFileBasedFunctionNamespaceManagerFactory()); functionAndTypeManager.loadFunctionNamespaceManager( @@ -1162,14 +1150,14 @@ public void testSimpleCase() "else 3 " + "end"); - assertOptimizedEquals("case true " + + assertOptimizedMatches("case true " + "when unbound_long = 1 then 1 " + "when 0 / 0 = 0 then 2 " + "else 33 end", "" + "case true " + - "when unbound_long = 1 then 1 " + - "when 0 / 0 = 0 then 2 else 33 " + + "when unbound_long = BIGINT '1' then 1 " + + "when CAST(fail(8, 'ignored failure message') AS boolean) then 2 else 33 " + "end"); assertOptimizedEquals("case bound_long " + @@ -1199,18 +1187,6 @@ public void testSimpleCase() "when unbound_long then 4 " + "end"); - assertOptimizedMatches("case 1 " + - "when unbound_long then 1 " + - "when 0 / 0 then 2 " + - "else 1 " + - "end", - "" + - "case BIGINT '1' " + - "when unbound_long then 1 " + - "when cast(fail(8, 'ignored failure message') AS integer) then 2 " + - "else 1 " + - "end"); - assertOptimizedMatches("case 1 " + "when 0 / 0 then 1 " + "when 0 / 0 then 2 " + @@ -1394,16 +1370,15 @@ public void testLikeOptimization() assertOptimizedEquals("unbound_string LIKE 'a#_b' ESCAPE '#'", "unbound_string = CAST('a_b' AS VARCHAR)"); assertOptimizedEquals("unbound_string LIKE 'a#%b' ESCAPE '#'", "unbound_string = CAST('a%b' AS VARCHAR)"); assertOptimizedEquals("unbound_string LIKE 'a#_##b' ESCAPE '#'", "unbound_string = CAST('a_#b' AS VARCHAR)"); - assertOptimizedEquals("unbound_string LIKE 'a#__b' ESCAPE '#'", "unbound_string LIKE 'a#__b' ESCAPE '#'"); - assertOptimizedEquals("unbound_string LIKE 'a##%b' ESCAPE '#'", "unbound_string LIKE 'a##%b' ESCAPE '#'"); + assertOptimizedMatches("unbound_string LIKE 'a#__b' ESCAPE '#'", "unbound_string LIKE 'a#__b' ESCAPE '#'"); + assertOptimizedMatches("unbound_string LIKE 'a##%b' ESCAPE '#'", "unbound_string LIKE 'a##%b' ESCAPE '#'"); assertOptimizedEquals("bound_string LIKE bound_pattern", "true"); assertOptimizedEquals("'abc' LIKE bound_pattern", "false"); - assertOptimizedEquals("unbound_string LIKE bound_pattern", "unbound_string LIKE bound_pattern"); assertDoNotOptimize("unbound_string LIKE 'abc%'", SERIALIZABLE); - assertOptimizedEquals("unbound_string LIKE unbound_pattern ESCAPE unbound_string", "unbound_string LIKE unbound_pattern ESCAPE unbound_string"); + assertOptimizedMatches("unbound_string LIKE unbound_pattern ESCAPE unbound_string", "unbound_string LIKE unbound_pattern ESCAPE unbound_string"); } @Test @@ -1586,123 +1561,27 @@ public void testLiterals() optimize("interval '3' day * unbound_long"); optimize("interval '3' year * unbound_long"); - assertEquals(optimize("X'1234'"), Slices.wrappedBuffer((byte) 0x12, (byte) 0x34)); - } - - private static void assertLike(byte[] value, String pattern, boolean expected) - { - Expression predicate = new LikePredicate( - rawStringLiteral(Slices.wrappedBuffer(value)), - new StringLiteral(pattern), - Optional.empty()); - assertEquals(evaluate(predicate, true), expected); - } - - private static StringLiteral rawStringLiteral(final Slice slice) - { - return new StringLiteral(slice.toStringUtf8()) - { - @Override - public Slice getSlice() - { - return slice; - } - }; + assertEquals(optimize("X'1234'"), wrappedBuffer((byte) 0x12, (byte) 0x34)); } + protected abstract Object evaluate(String expression, boolean deterministic); - private static void assertOptimizedEquals(@Language("SQL") String actual, @Language("SQL") String expected) - { - assertEquals(optimize(actual), optimize(expected)); - } - - private static void assertRowExpressionEquals(Level level, @Language("SQL") String actual, @Language("SQL") String expected) - { - Object actualResult = optimize(toRowExpression(expression(actual)), level); - Object expectedResult = optimize(toRowExpression(expression(expected)), level); - if (actualResult instanceof Block && expectedResult instanceof Block) { - assertEquals(blockToSlice((Block) actualResult), blockToSlice((Block) expectedResult)); - return; - } - assertEquals(actualResult, expectedResult); - } - - private static void assertOptimizedMatches(@Language("SQL") String actual, @Language("SQL") String expected) - { - // replaces FunctionCalls to FailureFunction by fail() - Object actualOptimized = optimize(actual); - if (actualOptimized instanceof Expression) { - actualOptimized = ExpressionTreeRewriter.rewriteWith(new FailedFunctionRewriter(), (Expression) actualOptimized); - } - assertEquals( - actualOptimized, - rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(expected))); - } - - private static Object optimize(@Language("SQL") String expression) - { - assertRoundTrip(expression); - - Expression parsedExpression = expression(expression); - Object expressionResult = optimize(parsedExpression); - - RowExpression rowExpression = toRowExpression(parsedExpression); - Object rowExpressionResult = optimize(rowExpression, OPTIMIZED); - assertExpressionAndRowExpressionEquals(expressionResult, rowExpressionResult); - return expressionResult; - } + protected abstract Object optimize(@Language("SQL") String expression); - private static Expression expression(String expression) - { - return FunctionAssertions.createExpression(expression, METADATA, SYMBOL_TYPES); - } + protected abstract void assertOptimizedEquals(@Language("SQL") String expression, @Language("SQL") String expected); - private static RowExpression toRowExpression(Expression expression) - { - return TRANSLATOR.translate(expression, SYMBOL_TYPES); - } + protected abstract void assertLike(byte[] value, String pattern, boolean expected); - private static Object optimize(Expression expression) - { - Map, Type> expressionTypes = getExpressionTypes(TEST_SESSION, METADATA, SQL_PARSER, SYMBOL_TYPES, expression, emptyMap(), WarningCollector.NOOP); - ExpressionInterpreter interpreter = expressionOptimizer(expression, METADATA, TEST_SESSION, expressionTypes); - return interpreter.optimize(variable -> { - Symbol symbol = new Symbol(variable.getName()); - Object value = symbolConstant(symbol); - if (value == null) { - return symbol.toSymbolReference(); - } - return value; - }); - } + protected abstract void assertOptimizedMatches(@Language("SQL") String actual, @Language("SQL") String expected); - private static Object optimize(RowExpression expression, Level level) - { - return new RowExpressionInterpreter(expression, METADATA.getFunctionAndTypeManager(), TEST_SESSION.toConnectorSession(), level).optimize(variable -> { - Symbol symbol = new Symbol(variable.getName()); - Object value = symbolConstant(symbol); - if (value == null) { - return new VariableReferenceExpression(Optional.empty(), symbol.getName(), SYMBOL_TYPES.get(symbol.toSymbolReference())); - } - return value; - }); - } + protected abstract void assertDoNotOptimize(@Language("SQL") String expression, ExpressionOptimizer.Level optimizationLevel); - private static void assertDoNotOptimize(@Language("SQL") String expression, Level optimizationLevel) + protected RowExpression sqlToRowExpression(String expression) { - assertRoundTrip(expression); - Expression translatedExpression = expression(expression); - RowExpression rowExpression = toRowExpression(translatedExpression); - - Object expressionResult = optimize(translatedExpression); - if (expressionResult instanceof Expression) { - expressionResult = toRowExpression((Expression) expressionResult); - } - Object rowExpressionResult = optimize(rowExpression, optimizationLevel); - assertRowExpressionEvaluationEquals(expressionResult, rowExpressionResult); - assertRowExpressionEvaluationEquals(rowExpressionResult, rowExpression); + Expression parsedExpression = FunctionAssertions.createExpression(expression, metadata, SYMBOL_TYPES); + return translator.translate(parsedExpression, SYMBOL_TYPES); } - private static Object symbolConstant(Symbol symbol) + protected Object symbolConstant(Symbol symbol) { switch (symbol.getName().toLowerCase(ENGLISH)) { case "bound_integer": @@ -1733,32 +1612,14 @@ private static Object symbolConstant(Symbol symbol) return null; } - private static void assertExpressionAndRowExpressionEquals(Object expressionResult, Object rowExpressionResult) - { - if (rowExpressionResult instanceof RowExpression) { - // Cannot be completely evaluated into a constant; compare expressions - assertTrue(expressionResult instanceof Expression); - - // It is tricky to check the equivalence of an expression and a row expression. - // We rely on the optimized translator to fill the gap. - RowExpression translated = TRANSLATOR.translateAndOptimize((Expression) expressionResult, SYMBOL_TYPES); - assertRowExpressionEvaluationEquals(translated, rowExpressionResult); - } - else { - // We have constants; directly compare - assertRowExpressionEvaluationEquals(expressionResult, rowExpressionResult); - } - } - /** * Assert the evaluation result of two row expressions equivalent * no matter they are constants or remaining row expressions. */ - private static void assertRowExpressionEvaluationEquals(Object left, Object right) + protected void assertRowExpressionEvaluationEquals(Object left, Object right) { if (right instanceof RowExpression) { assertTrue(left instanceof RowExpression); - // assertEquals(((RowExpression) left).getType(), ((RowExpression) right).getType()); if (left instanceof ConstantExpression) { if (isRemovableCast(right)) { assertRowExpressionEvaluationEquals(left, ((CallExpression) right).getArguments().get(0)); @@ -1770,6 +1631,13 @@ private static void assertRowExpressionEvaluationEquals(Object left, Object righ else if (left instanceof InputReferenceExpression || left instanceof VariableReferenceExpression) { assertEquals(left, right); } + else if (left instanceof CallExpression && ((CallExpression) left).getFunctionHandle().getName().contains("fail")) { + assertTrue(right instanceof CallExpression && ((CallExpression) right).getFunctionHandle().getName().contains("fail")); + assertEquals(((CallExpression) left).getArguments().size(), ((CallExpression) right).getArguments().size()); + for (int i = 0; i < ((CallExpression) left).getArguments().size(); i++) { + assertRowExpressionEvaluationEquals(((CallExpression) left).getArguments().get(i), ((CallExpression) right).getArguments().get(i)); + } + } else if (left instanceof CallExpression) { assertTrue(right instanceof CallExpression); assertEquals(((CallExpression) left).getFunctionHandle(), ((CallExpression) right).getFunctionHandle()); @@ -1806,68 +1674,46 @@ else if (left instanceof SpecialFormExpression) { } } - private static boolean isRemovableCast(Object value) + private boolean isRemovableCast(Object value) { if (value instanceof CallExpression && - new FunctionResolution(METADATA.getFunctionAndTypeManager().getFunctionAndTypeResolver()).isCastFunction(((CallExpression) value).getFunctionHandle())) { + new FunctionResolution(metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver()).isCastFunction(((CallExpression) value).getFunctionHandle())) { Type targetType = ((CallExpression) value).getType(); Type sourceType = ((CallExpression) value).getArguments().get(0).getType(); - return METADATA.getFunctionAndTypeManager().canCoerce(sourceType, targetType); + return metadata.getFunctionAndTypeManager().canCoerce(sourceType, targetType); } return false; } - private static Slice blockToSlice(Block block) + protected Slice blockToSlice(Block block) { // This function is strictly for testing use only SliceOutput sliceOutput = new DynamicSliceOutput(1000); - BlockSerdeUtil.writeBlock(blockEncodingSerde, sliceOutput, block); + BlockSerdeUtil.writeBlock(BLOCK_ENCODING_SERDE, sliceOutput, block); return sliceOutput.slice(); } - private static void assertEvaluatedEquals(@Language("SQL") String actual, @Language("SQL") String expected) + protected void assertEvaluatedEquals(@Language("SQL") String actual, @Language("SQL") String expected) { assertEquals(evaluate(actual, true), evaluate(expected, true)); } - private static Object evaluate(String expression, boolean deterministic) - { - assertRoundTrip(expression); - - Expression parsedExpression = FunctionAssertions.createExpression(expression, METADATA, SYMBOL_TYPES); - - return evaluate(parsedExpression, deterministic); - } - - private static void assertRoundTrip(String expression) + protected void assertRoundTrip(String expression) { ParsingOptions parsingOptions = createParsingOptions(TEST_SESSION); assertEquals(SQL_PARSER.createExpression(expression, parsingOptions), SQL_PARSER.createExpression(formatExpression(SQL_PARSER.createExpression(expression, parsingOptions), Optional.empty()), parsingOptions)); } - - private static Object evaluate(Expression expression, boolean deterministic) + protected void assertRowExpressionEquals(ExpressionOptimizer.Level level, @Language("SQL") String actual, @Language("SQL") String expected) { - Map, Type> expressionTypes = getExpressionTypes(TEST_SESSION, METADATA, SQL_PARSER, SYMBOL_TYPES, expression, emptyMap(), WarningCollector.NOOP); - Object expressionResult = expressionInterpreter(expression, METADATA, TEST_SESSION, expressionTypes).evaluate(); - Object rowExpressionResult = rowExpressionInterpreter(TRANSLATOR.translateAndOptimize(expression), METADATA.getFunctionAndTypeManager(), TEST_SESSION.toConnectorSession()).evaluate(); - - if (deterministic) { - assertExpressionAndRowExpressionEquals(expressionResult, rowExpressionResult); + Object actualResult = optimizeRowExpression(sqlToRowExpression(actual), level); + Object expectedResult = optimizeRowExpression(sqlToRowExpression(expected), level); + if (actualResult instanceof Block && expectedResult instanceof Block) { + assertEquals(blockToSlice((Block) actualResult), blockToSlice((Block) expectedResult)); + return; } - return expressionResult; + assertEquals(actualResult, expectedResult); } - private static class FailedFunctionRewriter - extends ExpressionRewriter - { - @Override - public Expression rewriteFunctionCall(FunctionCall node, Object context, ExpressionTreeRewriter treeRewriter) - { - if (node.getName().equals(QualifiedName.of("fail"))) { - return new FunctionCall(QualifiedName.of("fail"), ImmutableList.of(node.getArguments().get(0), new StringLiteral("ignored failure message"))); - } - return node; - } - } + protected abstract Object optimizeRowExpression(RowExpression expression, ExpressionOptimizer.Level level); } diff --git a/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java b/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java index c9a4f82b7cd8..45c030cff3ac 100644 --- a/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java +++ b/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java @@ -34,6 +34,7 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; @@ -254,6 +255,12 @@ public TestingAccessControlManager getAccessControl() return source.getAccessControl(); } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + return source.getExpressionManager(); + } + @Override public MaterializedResult execute(String sql) { From abfdb755c1da75b3c2bc20544f72951b76e0c938 Mon Sep 17 00:00:00 2001 From: Tim Meehan Date: Fri, 22 Nov 2024 16:23:23 -0500 Subject: [PATCH 3/5] Use ExpressionOptimizerProvider The runtime should consolidate to the `ExpressionOptimizerProvider` factory so that it can be customized without significant refactoring. --- .../presto/plugin/jdbc/JdbcConnector.java | 2 +- .../optimization/JdbcComputePushdown.java | 10 ++--- .../JdbcPlanOptimizerProvider.java | 10 ++--- .../optimization/TestJdbcComputePushdown.java | 2 +- .../facebook/presto/hive/HiveTestUtils.java | 2 +- .../presto/connector/ConnectorManager.java | 7 +++- .../presto/cost/ScalarStatsCalculator.java | 12 ++++-- .../presto/sql/planner/PlanOptimizers.java | 4 +- .../optimizations/PredicatePushDown.java | 38 +++++++++++++------ .../optimizations/PushdownSubfields.java | 13 ++++--- .../ConnectorRowExpressionService.java | 9 +++-- .../presto/testing/LocalQueryRunner.java | 1 + .../testing/TestingConnectorContext.java | 4 +- ...AbstractTestComparisonStatsCalculator.java | 3 +- .../AbstractTestFilterStatsCalculator.java | 3 +- ...ConnectorFilterStatsCalculatorService.java | 8 +++- .../presto/cost/TestJoinStatsRule.java | 6 ++- .../cost/TestScalarStatsCalculator.java | 6 ++- .../InMemoryExpressionOptimizerProvider.java | 38 +++++++++++++++++++ .../sql/planner/TestPredicatePushdown.java | 7 ++-- .../optimizations/TestReorderWindows.java | 3 +- 21 files changed, 136 insertions(+), 52 deletions(-) create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java index 3eb3ef011237..511899a2e03f 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java @@ -109,7 +109,7 @@ public ConnectorPlanOptimizerProvider getConnectorPlanOptimizerProvider() functionManager, functionResolution, rowExpressionService.getDeterminismEvaluator(), - rowExpressionService.getExpressionOptimizer()); + rowExpressionService); } @Override diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java index e0ad9a3282f3..b785abb189f1 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java @@ -29,7 +29,7 @@ import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.relation.DeterminismEvaluator; -import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.RowExpression; import java.util.Optional; @@ -44,7 +44,7 @@ public class JdbcComputePushdown implements ConnectorPlanOptimizer { - private final ExpressionOptimizer expressionOptimizer; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final JdbcFilterToSqlTranslator jdbcFilterToSqlTranslator; private final LogicalRowExpressions logicalRowExpressions; @@ -52,7 +52,7 @@ public JdbcComputePushdown( FunctionMetadataManager functionMetadataManager, StandardFunctionResolution functionResolution, DeterminismEvaluator determinismEvaluator, - ExpressionOptimizer expressionOptimizer, + ExpressionOptimizerProvider expressionOptimizerProvider, String identifierQuote, Set> functionTranslators) { @@ -62,7 +62,7 @@ public JdbcComputePushdown( requireNonNull(determinismEvaluator, "determinismEvaluator is null"); requireNonNull(functionResolution, "functionResolution is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.jdbcFilterToSqlTranslator = new JdbcFilterToSqlTranslator( functionMetadataManager, buildFunctionTranslator(functionTranslators), @@ -106,7 +106,7 @@ public PlanNode visitFilter(FilterNode node, RewriteContext context) TableHandle oldTableHandle = oldTableScanNode.getTable(); JdbcTableHandle oldConnectorTable = (JdbcTableHandle) oldTableHandle.getConnectorHandle(); - RowExpression predicate = expressionOptimizer.optimize(node.getPredicate(), OPTIMIZED, session); + RowExpression predicate = expressionOptimizerProvider.getExpressionOptimizer().optimize(node.getPredicate(), OPTIMIZED, session); predicate = logicalRowExpressions.convertToConjunctiveNormalForm(predicate); TranslatedExpression jdbcExpression = translateWith( predicate, diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java index 9dcd399b6647..5edbfd21eb7f 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java @@ -20,7 +20,7 @@ import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.relation.DeterminismEvaluator; -import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; @@ -34,7 +34,7 @@ public class JdbcPlanOptimizerProvider private final FunctionMetadataManager functionManager; private final StandardFunctionResolution functionResolution; private final DeterminismEvaluator determinismEvaluator; - private final ExpressionOptimizer expressionOptimizer; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final String identifierQuote; @Inject @@ -43,12 +43,12 @@ public JdbcPlanOptimizerProvider( FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution, DeterminismEvaluator determinismEvaluator, - ExpressionOptimizer expressionOptimizer) + ExpressionOptimizerProvider expressionOptimizerProvider) { this.functionManager = requireNonNull(functionManager, "functionManager is null"); this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.identifierQuote = jdbcClient.getIdentifierQuote(); } @@ -65,7 +65,7 @@ public Set getPhysicalPlanOptimizers() functionManager, functionResolution, determinismEvaluator, - expressionOptimizer, + expressionOptimizerProvider, identifierQuote, getFunctionTranslators())); } diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java index 726f89f6631b..caf77ae21d1d 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java @@ -105,7 +105,7 @@ public TestJdbcComputePushdown() functionAndTypeManager, functionResolution, determinismEvaluator, - new RowExpressionOptimizer(METADATA), + () -> new RowExpressionOptimizer(METADATA), "'", getFunctionTranslators()); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java b/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java index a6fdaa9f7367..a9b1f5c0fb92 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java @@ -151,7 +151,7 @@ public String formatRowExpression(ConnectorSession session, RowExpression expres }; public static final FilterStatsCalculatorService FILTER_STATS_CALCULATOR_SERVICE = new ConnectorFilterStatsCalculatorService( - new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA), new StatsNormalizer())); + new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA, ROW_EXPRESSION_SERVICE), new StatsNormalizer())); public static final HiveClientConfig HIVE_CLIENT_CONFIG = new HiveClientConfig(); public static final MetastoreClientConfig METASTORE_CLIENT_CONFIG = new MetastoreClientConfig(); diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java index 6ed7888b33d5..56f3e15d72aa 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -62,12 +62,12 @@ import com.facebook.presto.split.RecordPageSourceProvider; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.planPrinter.RowExpressionFormatter; import com.facebook.presto.sql.relational.ConnectorRowExpressionService; import com.facebook.presto.sql.relational.FunctionResolution; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -116,6 +116,7 @@ public class ConnectorManager private final PageIndexerFactory pageIndexerFactory; private final NodeInfo nodeInfo; private final TransactionManager transactionManager; + private final ExpressionOptimizerManager expressionOptimizerManager; private final DomainTranslator domainTranslator; private final PredicateCompiler predicateCompiler; private final DeterminismEvaluator determinismEvaluator; @@ -151,6 +152,7 @@ public ConnectorManager( PageSorter pageSorter, PageIndexerFactory pageIndexerFactory, TransactionManager transactionManager, + ExpressionOptimizerManager expressionOptimizerManager, DomainTranslator domainTranslator, PredicateCompiler predicateCompiler, DeterminismEvaluator determinismEvaluator, @@ -176,6 +178,7 @@ public ConnectorManager( this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); this.nodeInfo = requireNonNull(nodeInfo, "nodeInfo is null"); this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); this.predicateCompiler = requireNonNull(predicateCompiler, "predicateCompiler is null"); this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); @@ -382,7 +385,7 @@ private Connector createConnector(ConnectorId connectorId, ConnectorFactory fact pageIndexerFactory, new ConnectorRowExpressionService( domainTranslator, - new RowExpressionOptimizer(metadataManager), + expressionOptimizerManager, predicateCompiler, determinismEvaluator, new RowExpressionFormatter(metadataManager.getFunctionAndTypeManager())), diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java index cb5d8ec8c591..d9bc1a4a8a6e 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.InputReferenceExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -78,13 +79,18 @@ public class ScalarStatsCalculator { private final Metadata metadata; - private final ExpressionOptimizerManager expressionOptimizerManager; + private final ExpressionOptimizerProvider expressionOptimizerProvider; @Inject public ScalarStatsCalculator(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager) + { + this(metadata, (ExpressionOptimizerProvider) expressionOptimizerManager); + } + + public ScalarStatsCalculator(Metadata metadata, ExpressionOptimizerProvider expressionOptimizerProvider) { this.metadata = requireNonNull(metadata, "metadata can not be null"); - this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager can not be null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerManager can not be null"); } @Deprecated @@ -128,7 +134,7 @@ public VariableStatsEstimate visitCall(CallExpression call, Void context) return computeArithmeticBinaryStatistics(call, context); } - RowExpression value = expressionOptimizerManager.getExpressionOptimizer().optimize(call, OPTIMIZED, session); + RowExpression value = expressionOptimizerProvider.getExpressionOptimizer().optimize(call, OPTIMIZED, session); if (isNull(value)) { return nullStatsEstimate(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index ab4b87ed9ab0..859a2e1d2aa6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -350,7 +350,7 @@ public PlanOptimizers( estimatedExchangesCostCalculator, new RewriteConstantArrayContainsToInExpression(metadata.getFunctionAndTypeManager()).rules()); - PlanOptimizer predicatePushDown = new StatsRecordingPlanOptimizer(optimizerStats, new PredicatePushDown(metadata, sqlParser, featuresConfig.isNativeExecutionEnabled())); + PlanOptimizer predicatePushDown = new StatsRecordingPlanOptimizer(optimizerStats, new PredicatePushDown(metadata, sqlParser, expressionOptimizerManager, featuresConfig.isNativeExecutionEnabled())); PlanOptimizer prefilterForLimitingAggregation = new StatsRecordingPlanOptimizer(optimizerStats, new PrefilterForLimitingAggregation(metadata, statsCalculator)); builder.add( @@ -731,7 +731,7 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of(new RemoveRedundantIdentityProjections(), new PruneRedundantProjectionAssignments())), - new PushdownSubfields(metadata)); + new PushdownSubfields(metadata, expressionOptimizerManager)); builder.add(predicatePushDown); // Run predicate push down one more time in case we can leverage new information from layouts' effective predicate builder.add(simplifyRowExpressionOptimizer); // Should be always run after PredicatePushDown diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java index 087cab8bcc77..4973eb71d1aa 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java @@ -45,6 +45,7 @@ import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.parser.SqlParser; @@ -65,7 +66,6 @@ import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableList; @@ -132,13 +132,15 @@ public class PredicatePushDown private final SqlParser sqlParser; private final RowExpressionDomainTranslator rowExpressionDomainTranslator; private final boolean nativeExecution; + private final ExpressionOptimizerProvider expressionOptimizerProvider; - public PredicatePushDown(Metadata metadata, SqlParser sqlParser, boolean nativeExecution) + public PredicatePushDown(Metadata metadata, SqlParser sqlParser, ExpressionOptimizerProvider expressionOptimizerProvider, boolean nativeExecution) { this.metadata = requireNonNull(metadata, "metadata is null"); rowExpressionDomainTranslator = new RowExpressionDomainTranslator(metadata); this.effectivePredicateExtractor = new EffectivePredicateExtractor(rowExpressionDomainTranslator, metadata.getFunctionAndTypeManager()); this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.nativeExecution = nativeExecution; } @@ -150,7 +152,7 @@ public PlanOptimizerResult optimize(PlanNode plan, Session session, TypeProvider requireNonNull(types, "types is null"); requireNonNull(idAllocator, "idAllocator is null"); - Rewriter rewriter = new Rewriter(variableAllocator, idAllocator, metadata, effectivePredicateExtractor, rowExpressionDomainTranslator, sqlParser, session, nativeExecution); + Rewriter rewriter = new Rewriter(variableAllocator, idAllocator, metadata, effectivePredicateExtractor, rowExpressionDomainTranslator, expressionOptimizerProvider, sqlParser, session, nativeExecution); PlanNode rewrittenPlan = SimplePlanRewriter.rewriteWith(rewriter, plan, TRUE_CONSTANT); return PlanOptimizerResult.optimizerResult(rewrittenPlan, rewriter.isPlanChanged()); } @@ -184,6 +186,7 @@ private static class Rewriter private final Metadata metadata; private final EffectivePredicateExtractor effectivePredicateExtractor; private final RowExpressionDomainTranslator rowExpressionDomainTranslator; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final Session session; private final boolean nativeExecution; private final ExpressionEquivalence expressionEquivalence; @@ -199,6 +202,7 @@ private Rewriter( Metadata metadata, EffectivePredicateExtractor effectivePredicateExtractor, RowExpressionDomainTranslator rowExpressionDomainTranslator, + ExpressionOptimizerProvider expressionOptimizerProvider, SqlParser sqlParser, Session session, boolean nativeExecution) @@ -208,6 +212,7 @@ private Rewriter( this.metadata = requireNonNull(metadata, "metadata is null"); this.effectivePredicateExtractor = requireNonNull(effectivePredicateExtractor, "effectivePredicateExtractor is null"); this.rowExpressionDomainTranslator = rowExpressionDomainTranslator; + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.session = requireNonNull(session, "session is null"); this.nativeExecution = nativeExecution; this.expressionEquivalence = new ExpressionEquivalence(metadata, sqlParser); @@ -1159,7 +1164,8 @@ private RowExpression getPostJoinPredicate() } } - private InnerJoinPushDownResult processInnerJoin(RowExpression inheritedPredicate, + private InnerJoinPushDownResult processInnerJoin( + RowExpression inheritedPredicate, RowExpression leftEffectivePredicate, RowExpression rightEffectivePredicate, RowExpression joinPredicate, @@ -1283,6 +1289,7 @@ private InnerJoinPushDownResult processInnerJoin(RowExpression inheritedPredicat joinConjuncts.addAll(allInference.generateEqualitiesPartitionedBy(in(leftVariables)::apply).getScopeStraddlingEqualities()); // scope straddling equalities get dropped in as part of the join predicate return new Rewriter.InnerJoinPushDownResult( + expressionOptimizerProvider, logicalRowExpressions.combineConjuncts(leftPushDownConjuncts.build()), logicalRowExpressions.combineConjuncts(rightPushDownConjuncts.build()), logicalRowExpressions.combineConjuncts(joinConjuncts.build()), TRUE_CONSTANT); @@ -1294,13 +1301,20 @@ private static class InnerJoinPushDownResult private final RowExpression rightPredicate; private final RowExpression joinPredicate; private final RowExpression postJoinPredicate; - - private InnerJoinPushDownResult(RowExpression leftPredicate, RowExpression rightPredicate, RowExpression joinPredicate, RowExpression postJoinPredicate) + private final ExpressionOptimizerProvider expressionOptimizerProvider; + + private InnerJoinPushDownResult( + ExpressionOptimizerProvider expressionOptimizerProvider, + RowExpression leftPredicate, + RowExpression rightPredicate, + RowExpression joinPredicate, + RowExpression postJoinPredicate) { - this.leftPredicate = leftPredicate; - this.rightPredicate = rightPredicate; - this.joinPredicate = joinPredicate; - this.postJoinPredicate = postJoinPredicate; + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); + this.leftPredicate = requireNonNull(leftPredicate, "leftPredicate is null"); + this.rightPredicate = requireNonNull(rightPredicate, "rightPredicate is null"); + this.joinPredicate = requireNonNull(joinPredicate, "joinPredicate is null"); + this.postJoinPredicate = requireNonNull(postJoinPredicate, "postJoinPredicate is null"); } private RowExpression getLeftPredicate() @@ -1425,7 +1439,7 @@ private boolean canConvertOuterToInner(List innerVa // Temporary implementation for joins because the SimplifyExpressions optimizers can not run properly on join clauses private RowExpression simplifyExpression(RowExpression expression) { - return new RowExpressionOptimizer(metadata).optimize(expression, ExpressionOptimizer.Level.SERIALIZABLE, session.toConnectorSession()); + return expressionOptimizerProvider.getExpressionOptimizer().optimize(expression, ExpressionOptimizer.Level.SERIALIZABLE, session.toConnectorSession()); } private boolean areExpressionsEquivalent(RowExpression leftExpression, RowExpression rightExpression) @@ -1440,7 +1454,7 @@ private RowExpression nullInputEvaluator(final Collection constantNull(variable.getSourceLocation(), variable.getType())))); - return new RowExpressionOptimizer(metadata).optimize(expression, ExpressionOptimizer.Level.OPTIMIZED, session.toConnectorSession()); + return expressionOptimizerProvider.getExpressionOptimizer().optimize(expression, ExpressionOptimizer.Level.OPTIMIZED, session.toConnectorSession()); } private Predicate joinEqualityExpression(final Collection leftVariables) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java index 31f15d9a694e..e57f87ec3358 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java @@ -57,6 +57,7 @@ import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; @@ -71,7 +72,6 @@ import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.relational.FunctionResolution; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -111,11 +111,13 @@ public class PushdownSubfields public static final QualifiedObjectName ELEMENT_AT = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "element_at"); public static final QualifiedObjectName CAST = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "$operator$cast"); private final Metadata metadata; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private boolean isEnabledForTesting; - public PushdownSubfields(Metadata metadata) + public PushdownSubfields(Metadata metadata, ExpressionOptimizerProvider expressionOptimizerProvider) { this.metadata = requireNonNull(metadata, "metadata is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); } @Override @@ -141,7 +143,7 @@ public PlanOptimizerResult optimize(PlanNode plan, Session session, TypeProvider return PlanOptimizerResult.optimizerResult(plan, false); } - Rewriter rewriter = new Rewriter(session, metadata); + Rewriter rewriter = new Rewriter(session, metadata, expressionOptimizerProvider); PlanNode rewrittenPlan = SimplePlanRewriter.rewriteWith(rewriter, plan, new Rewriter.Context()); return PlanOptimizerResult.optimizerResult(rewrittenPlan, rewriter.isPlanChanged()); } @@ -157,12 +159,13 @@ private static class Rewriter private static final QualifiedObjectName ARBITRARY_AGGREGATE_FUNCTION = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "arbitrary"); private boolean planChanged; - public Rewriter(Session session, Metadata metadata) + public Rewriter(Session session, Metadata metadata, ExpressionOptimizerProvider expressionOptimizerProvider) { this.session = requireNonNull(session, "session is null"); this.metadata = requireNonNull(metadata, "metadata is null"); + requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.functionResolution = new FunctionResolution(metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver()); - this.expressionOptimizer = new RowExpressionOptimizer(metadata); + this.expressionOptimizer = expressionOptimizerProvider.getExpressionOptimizer(); this.subfieldExtractor = new SubfieldExtractor( functionResolution, expressionOptimizer, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java index a8ab25fefc51..c8d52e0e309c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.PredicateCompiler; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; @@ -28,20 +29,20 @@ public final class ConnectorRowExpressionService implements RowExpressionService { private final DomainTranslator domainTranslator; - private final ExpressionOptimizer expressionOptimizer; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final PredicateCompiler predicateCompiler; private final DeterminismEvaluator determinismEvaluator; private final RowExpressionFormatter rowExpressionFormatter; public ConnectorRowExpressionService( DomainTranslator domainTranslator, - ExpressionOptimizer expressionOptimizer, + ExpressionOptimizerProvider expressionOptimizerProvider, PredicateCompiler predicateCompiler, DeterminismEvaluator determinismEvaluator, RowExpressionFormatter rowExpressionFormatter) { this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.predicateCompiler = requireNonNull(predicateCompiler, "predicateCompiler is null"); this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); this.rowExpressionFormatter = requireNonNull(rowExpressionFormatter, "rowExpressionFormatter is null"); @@ -56,7 +57,7 @@ public DomainTranslator getDomainTranslator() @Override public ExpressionOptimizer getExpressionOptimizer() { - return expressionOptimizer; + return expressionOptimizerProvider.getExpressionOptimizer(); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 248bd6a11ec0..0cedd5e66b99 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -493,6 +493,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, pageSorter, pageIndexerFactory, transactionManager, + expressionOptimizerManager, new RowExpressionDomainTranslator(metadata), new RowExpressionPredicateCompiler(metadata), new RowExpressionDeterminismEvaluator(metadata.getFunctionAndTypeManager()), diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java index 7cef5e6c1cd2..957bb96a15ec 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java @@ -41,6 +41,7 @@ import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.PredicateCompiler; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; @@ -66,7 +67,8 @@ public class TestingConnectorContext private final DomainTranslator domainTranslator = new RowExpressionDomainTranslator(metadata); private final PredicateCompiler predicateCompiler = new RowExpressionPredicateCompiler(metadata); private final DeterminismEvaluator determinismEvaluator = new RowExpressionDeterminismEvaluator(functionAndTypeManager); - private final FilterStatsCalculatorService filterStatsCalculatorService = new ConnectorFilterStatsCalculatorService(new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer())); + private final ExpressionOptimizerProvider expressionOptimizerProvider = () -> new RowExpressionOptimizer(metadata); + private final FilterStatsCalculatorService filterStatsCalculatorService = new ConnectorFilterStatsCalculatorService(new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata, expressionOptimizerProvider), new StatsNormalizer())); private final BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(); @Override diff --git a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java index 66c2ceef405f..865cf0d018e8 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java @@ -18,6 +18,7 @@ import com.facebook.presto.common.type.VarcharType; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.ComparisonExpression; @@ -82,7 +83,7 @@ public void setUp() throws Exception { MetadataManager metadata = MetadataManager.createTestMetadataManager(); - filterStatsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer()); + filterStatsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata, new InMemoryExpressionOptimizerProvider(metadata)), new StatsNormalizer()); uStats = VariableStatsEstimate.builder() .setAverageRowSize(8.0) diff --git a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java index c8f919326f92..aef0e39f5629 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Expression; @@ -146,7 +147,7 @@ public void setUp() .build()); MetadataManager metadata = MetadataManager.createTestMetadataManager(); - statsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer()); + statsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata, new InMemoryExpressionOptimizerProvider(metadata)), new StatsNormalizer()); translator = new TestingRowExpressionTranslator(MetadataManager.createTestMetadataManager()); } diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java b/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java index 584f55140e3a..8c8a9fe1acc0 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.statistics.DoubleRange; import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.TableStatistics; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Expression; @@ -56,7 +57,12 @@ public void setUp() { session = testSessionBuilder().build(); MetadataManager metadata = MetadataManager.createTestMetadataManager(); - FilterStatsCalculator statsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer()); + FilterStatsCalculator statsCalculator = new FilterStatsCalculator( + metadata, + new ScalarStatsCalculator( + metadata, + new InMemoryExpressionOptimizerProvider(metadata)), + new StatsNormalizer()); statsCalculatorService = new ConnectorFilterStatsCalculatorService(statsCalculator); xStats = ColumnStatistics.builder() .setDistinctValuesCount(Estimate.of(40)) diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java b/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java index 0f642ae51756..5d757b68d1e5 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java @@ -18,8 +18,10 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.JoinType; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -90,8 +92,10 @@ public class TestJoinStatsRule private static final MetadataManager METADATA = createTestMetadataManager(); private static final StatsNormalizer NORMALIZER = new StatsNormalizer(); + + private static final ExpressionOptimizerProvider EXPRESSION_OPTIMIZER_PROVIDER = new InMemoryExpressionOptimizerProvider(METADATA); private static final JoinStatsRule JOIN_STATS_RULE = new JoinStatsRule( - new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA), NORMALIZER), + new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA, EXPRESSION_OPTIMIZER_PROVIDER), NORMALIZER), NORMALIZER, 1.0); diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java index c7951ffbeebb..afe69f74e8fe 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.LiteralEncoder; @@ -68,9 +69,10 @@ public class TestScalarStatsCalculator @BeforeClass public void setUp() { - calculator = new ScalarStatsCalculator(MetadataManager.createTestMetadataManager()); + MetadataManager metadata = createTestMetadataManager(); + calculator = new ScalarStatsCalculator(metadata, new InMemoryExpressionOptimizerProvider(metadata)); session = testSessionBuilder().build(); - translator = new TestingRowExpressionTranslator(MetadataManager.createTestMetadataManager()); + translator = new TestingRowExpressionTranslator(metadata); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java b/presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java new file mode 100644 index 000000000000..14fe4185ca80 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; + +import static java.util.Objects.requireNonNull; + +public class InMemoryExpressionOptimizerProvider + implements ExpressionOptimizerProvider +{ + private final Metadata metadata; + + public InMemoryExpressionOptimizerProvider(Metadata metadata) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + } + + @Override + public ExpressionOptimizer getExpressionOptimizer() + { + return new RowExpressionOptimizer(metadata); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java index df925d425bef..f3aaca8d2d19 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.WindowNode; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; @@ -476,7 +477,7 @@ public void testNoPushdownWithTry() public void testPredicatePushDownCanReduceInnerToCrossJoin() { RuleTester tester = new RuleTester(); - tester.assertThat(new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), false)) + tester.assertThat(new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), new InMemoryExpressionOptimizerProvider(tester.getMetadata()), false)) .on(p -> p.join(INNER, p.filter(p.comparison(OperatorType.EQUAL, p.variable("a1"), constant(1L, INTEGER)), @@ -509,7 +510,7 @@ public void testPredicatePushDownCanReduceInnerToCrossJoin() public void testPredicatePushdownDoesNotAddProjectsBetweenJoinNodes() { RuleTester tester = new RuleTester(); - PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), false); + PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), new InMemoryExpressionOptimizerProvider(tester.getMetadata()), false); tester.assertThat(predicatePushDownOptimizer) .on("SELECT 1 " + @@ -589,7 +590,7 @@ public void testPredicatePushdownDoesNotAddProjectsBetweenJoinNodes() public void testDomainFiltersCanBeInferredForLargeDisjunctiveFilters() { RuleTester tester = new RuleTester(emptyList(), ImmutableMap.of(GENERATE_DOMAIN_FILTERS, "true")); - PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), false); + PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), new InMemoryExpressionOptimizerProvider(tester.getMetadata()), false); // For Inner Join tester.assertThat(predicatePushDownOptimizer) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java index e35960c601fd..1ea58b5627d5 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java @@ -15,6 +15,7 @@ import com.facebook.presto.common.block.SortOrder; import com.facebook.presto.spi.plan.WindowNode; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.ExpectedValueProvider; @@ -322,7 +323,7 @@ private void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern patter { List optimizers = ImmutableList.of( new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), - new PredicatePushDown(getMetadata(), getQueryRunner().getSqlParser(), false), + new PredicatePushDown(getMetadata(), getQueryRunner().getSqlParser(), new InMemoryExpressionOptimizerProvider(getMetadata()), false), new IterativeOptimizer( getMetadata(), new RuleStatsRecorder(), From 4cfbaa25cc71668c969809b32936425aeb2f04f1 Mon Sep 17 00:00:00 2001 From: Tim Meehan Date: Tue, 30 Jul 2024 16:21:46 -0400 Subject: [PATCH 4/5] Add native row expression optimizer --- pom.xml | 20 ++ .../presto/metadata/HandleJsonModule.java | 19 +- .../presto/server/ServerMainModule.java | 5 + .../ExpressionOptimizerManager.java | 9 +- .../JsonCodecRowExpressionSerde.java | 48 +++ .../presto/testing/LocalQueryRunner.java | 4 +- .../planner/assertions/OptimizerAssert.java | 6 +- .../rule/TestSimplifyRowExpressions.java | 4 +- .../PrestoNativeQueryRunnerUtils.java | 56 ++- presto-native-sidecar-plugin/pom.xml | 11 + .../sql/expressions/ForSidecarInfo.java | 26 ++ .../NativeExpressionOptimizer.java | 332 ++++++++++++++++++ .../NativeExpressionOptimizerFactory.java | 64 ++++ .../NativeExpressionOptimizerProvider.java | 42 +++ .../NativeExpressionsCommunicationModule.java | 29 ++ .../expressions/NativeExpressionsModule.java | 66 ++++ .../NativeSidecarExpressionInterpreter.java | 112 ++++++ .../RowExpressionDeserializer.java | 52 +++ .../expressions/RowExpressionSerializer.java | 52 +++ .../presto/sidecar/NativeSidecarPlugin.java | 7 + .../TestNativeExpressionOptimizer.java | 319 +++++++++++++++++ .../expressions/TestNativeExpressions.java | 224 ++++++++++++ .../NativeSidecarPluginQueryRunnerUtils.java | 4 + .../sidecar/TestNativeSidecarPlugin.java | 40 +++ .../UnimplementedFunctionMetadataManager.java | 28 ++ .../UnimplementedFunctionResolution.java | 229 ++++++++++++ .../sidecar/UnimplementedNodeManager.java | 117 ++++++ .../UnimplementedRowExpressionSerde.java | 33 ++ .../src/main/resources/expressions.yaml | 173 +++++++++ .../presto/spark/PrestoSparkModule.java | 5 + .../presto/spi/RowExpressionSerde.java | 23 ++ .../planner/ExpressionOptimizerContext.java | 10 +- .../tests/AbstractTestQueryFramework.java | 6 +- 33 files changed, 2165 insertions(+), 10 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/expressions/JsonCodecRowExpressionSerde.java create mode 100644 presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/ForSidecarInfo.java create mode 100644 presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizer.java create mode 100644 presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizerFactory.java create mode 100644 presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizerProvider.java create mode 100644 presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionsCommunicationModule.java create mode 100644 presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionsModule.java create mode 100644 presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeSidecarExpressionInterpreter.java create mode 100644 presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/RowExpressionDeserializer.java create mode 100644 presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/RowExpressionSerializer.java create mode 100644 presto-native-sidecar-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressionOptimizer.java create mode 100644 presto-native-sidecar-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressions.java create mode 100644 presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java create mode 100644 presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedFunctionMetadataManager.java create mode 100644 presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedFunctionResolution.java create mode 100644 presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedNodeManager.java create mode 100644 presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedRowExpressionSerde.java create mode 100644 presto-openapi/src/main/resources/expressions.yaml create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/RowExpressionSerde.java diff --git a/pom.xml b/pom.xml index ee4920c08462..c78fd45fb54f 100644 --- a/pom.xml +++ b/pom.xml @@ -728,6 +728,13 @@ ${project.version} + + com.facebook.presto + presto-tests + ${project.version} + test-jar + + com.facebook.presto presto-benchmark @@ -953,6 +960,19 @@ ${project.version} + + com.facebook.presto + presto-native-plugin + ${project.version} + + + + com.facebook.presto + presto-native-execution + ${project.version} + test-jar + + com.facebook.hive hive-dwrf diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java b/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java index 0c07b99aaab4..cc2dee1bd61d 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/HandleJsonModule.java @@ -23,6 +23,18 @@ public class HandleJsonModule implements Module { + private final HandleResolver handleResolver; + + public HandleJsonModule() + { + this(null); + } + + public HandleJsonModule(HandleResolver handleResolver) + { + this.handleResolver = handleResolver; + } + @Override public void configure(Binder binder) { @@ -38,6 +50,11 @@ public void configure(Binder binder) jsonBinder(binder).addModuleBinding().to(FunctionHandleJacksonModule.class); jsonBinder(binder).addModuleBinding().to(MetadataUpdateJacksonModule.class); - binder.bind(HandleResolver.class).in(Scopes.SINGLETON); + if (handleResolver == null) { + binder.bind(HandleResolver.class).in(Scopes.SINGLETON); + } + else { + binder.bind(HandleResolver.class).toInstance(handleResolver); + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index 881588658003..364dbe279c95 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -148,6 +148,7 @@ import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; +import com.facebook.presto.spi.RowExpressionSerde; import com.facebook.presto.spi.analyzer.ViewDefinition; import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.plan.SimplePlanFragment; @@ -155,6 +156,7 @@ import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.PredicateCompiler; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.session.WorkerSessionPropertyProvider; import com.facebook.presto.spiller.FileSingleStreamSpillerFactory; @@ -195,6 +197,7 @@ import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; +import com.facebook.presto.sql.expressions.JsonCodecRowExpressionSerde; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -362,6 +365,7 @@ else if (serverConfig.isCoordinator()) { // expression manager binder.bind(ExpressionOptimizerManager.class).in(Scopes.SINGLETON); + binder.bind(RowExpressionSerde.class).to(JsonCodecRowExpressionSerde.class).in(Scopes.SINGLETON); // schema properties binder.bind(SchemaPropertyManager.class).in(Scopes.SINGLETON); @@ -555,6 +559,7 @@ public ListeningExecutorService createResourceManagerExecutor(ResourceManagerCon jsonCodecBinder(binder).bindJsonCodec(SqlInvokedFunction.class); jsonCodecBinder(binder).bindJsonCodec(TaskSource.class); jsonCodecBinder(binder).bindJsonCodec(TableWriteInfo.class); + jsonCodecBinder(binder).bindJsonCodec(RowExpression.class); smileCodecBinder(binder).bindSmileCodec(TaskStatus.class); smileCodecBinder(binder).bindSmileCodec(TaskInfo.class); thriftCodecBinder(binder).bindThriftCodec(TaskStatus.class); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java b/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java index 303a5e2abb29..af9f5baf3eef 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java @@ -16,6 +16,7 @@ import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.RowExpressionSerde; import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.sql.planner.ExpressionOptimizerContext; @@ -48,15 +49,17 @@ public class ExpressionOptimizerManager private final AtomicReference rowExpressionInterpreter = new AtomicReference<>(); private final NodeManager nodeManager; private final FunctionAndTypeManager functionAndTypeManager; + private final RowExpressionSerde rowExpressionSerde; private final FunctionResolution functionResolution; private final ExpressionOptimizer defaultExpressionOptimizer; @Inject - public ExpressionOptimizerManager(PluginNodeManager nodeManager, FunctionAndTypeManager functionAndTypeManager) + public ExpressionOptimizerManager(PluginNodeManager nodeManager, FunctionAndTypeManager functionAndTypeManager, RowExpressionSerde rowExpressionSerde) { requireNonNull(nodeManager, "nodeManager is null"); this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); + this.rowExpressionSerde = requireNonNull(rowExpressionSerde, "rowExpressionSerde is null"); this.functionResolution = new FunctionResolution(functionAndTypeManager.getFunctionAndTypeResolver()); this.defaultExpressionOptimizer = new RowExpressionOptimizer(functionAndTypeManager); rowExpressionInterpreter.set(defaultExpressionOptimizer); @@ -75,7 +78,7 @@ public void loadExpressionOptimizerFactory() } } - private void loadExpressionOptimizerFactory(Map properties) + public void loadExpressionOptimizerFactory(Map properties) { properties = new HashMap<>(properties); String factoryName = properties.remove(EXPRESSION_MANAGER_FACTORY_NAME); @@ -83,7 +86,7 @@ private void loadExpressionOptimizerFactory(Map properties) checkArgument( rowExpressionInterpreter.compareAndSet( defaultExpressionOptimizer, - expressionOptimizerFactories.get(factoryName).createOptimizer(properties, new ExpressionOptimizerContext(nodeManager, functionAndTypeManager, functionResolution))), + expressionOptimizerFactories.get(factoryName).createOptimizer(properties, new ExpressionOptimizerContext(nodeManager, rowExpressionSerde, functionAndTypeManager, functionResolution))), "ExpressionManager is already loaded"); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/expressions/JsonCodecRowExpressionSerde.java b/presto-main/src/main/java/com/facebook/presto/sql/expressions/JsonCodecRowExpressionSerde.java new file mode 100644 index 000000000000..20cfcf151df6 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/expressions/JsonCodecRowExpressionSerde.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.expressions; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.spi.RowExpressionSerde; +import com.facebook.presto.spi.relation.RowExpression; + +import javax.inject.Inject; + +import java.nio.charset.StandardCharsets; + +import static java.util.Objects.requireNonNull; + +public class JsonCodecRowExpressionSerde + implements RowExpressionSerde +{ + private final JsonCodec codec; + + @Inject + public JsonCodecRowExpressionSerde(JsonCodec codec) + { + this.codec = requireNonNull(codec, "codec is null"); + } + + @Override + public String serialize(RowExpression expression) + { + return new String(codec.toBytes(expression), StandardCharsets.UTF_8); + } + + @Override + public RowExpression deserialize(String data) + { + return codec.fromBytes(data.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 0cedd5e66b99..a6726b53acb3 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -144,6 +144,7 @@ import com.facebook.presto.spi.plan.SimplePlanFragment; import com.facebook.presto.spi.plan.StageExecutionDescriptor; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spiller.FileSingleStreamSpillerFactory; import com.facebook.presto.spiller.GenericPartitioningSpillerFactory; import com.facebook.presto.spiller.GenericSpillerFactory; @@ -170,6 +171,7 @@ import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; +import com.facebook.presto.sql.expressions.JsonCodecRowExpressionSerde; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -455,7 +457,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, this.pageIndexerFactory = new GroupByHashPageIndexerFactory(joinCompiler); NodeInfo nodeInfo = new NodeInfo("test"); - expressionOptimizerManager = new ExpressionOptimizerManager(new PluginNodeManager(nodeManager, nodeInfo.getEnvironment()), getFunctionAndTypeManager()); + expressionOptimizerManager = new ExpressionOptimizerManager(new PluginNodeManager(nodeManager, nodeInfo.getEnvironment()), getFunctionAndTypeManager(), new JsonCodecRowExpressionSerde(jsonCodec(RowExpression.class))); this.statsNormalizer = new StatsNormalizer(); this.scalarStatsCalculator = new ScalarStatsCalculator(metadata, expressionOptimizerManager); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java index ddb6b0d64767..7bd78483a2bc 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java @@ -23,10 +23,12 @@ import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.Optimizer; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; +import com.facebook.presto.sql.expressions.JsonCodecRowExpressionSerde; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.TypeProvider; @@ -50,6 +52,7 @@ import java.util.function.Consumer; import java.util.function.Function; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.sql.planner.assertions.PlanAssert.assertPlan; import static com.facebook.presto.sql.planner.assertions.PlanAssert.assertPlanDoesNotMatch; import static com.facebook.presto.transaction.TransactionBuilder.transaction; @@ -178,7 +181,8 @@ private List getMinimalOptimizers() metadata, new ExpressionOptimizerManager( new PluginNodeManager(new InMemoryNodeManager()), - queryRunner.getFunctionAndTypeManager()), + queryRunner.getFunctionAndTypeManager(), + new JsonCodecRowExpressionSerde(jsonCodec(RowExpression.class))), new FeaturesConfig()).rules())); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java index d81aa041bc24..5086fe5b6418 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java @@ -27,6 +27,7 @@ import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; +import com.facebook.presto.sql.expressions.JsonCodecRowExpressionSerde; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Expression; @@ -40,6 +41,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.SystemSessionProperties.DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; @@ -190,7 +192,7 @@ private static void assertSimplifies(String expression, String rowExpressionExpe Expression actualExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(expression)); InMemoryNodeManager nodeManager = new InMemoryNodeManager(); - ExpressionOptimizerManager expressionOptimizerManager = new ExpressionOptimizerManager(new PluginNodeManager(nodeManager), METADATA.getFunctionAndTypeManager()); + ExpressionOptimizerManager expressionOptimizerManager = new ExpressionOptimizerManager(new PluginNodeManager(nodeManager), METADATA.getFunctionAndTypeManager(), new JsonCodecRowExpressionSerde(jsonCodec(RowExpression.class))); expressionOptimizerManager.loadExpressionOptimizerFactory(); TestingRowExpressionTranslator translator = new TestingRowExpressionTranslator(METADATA); diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java index 58431a64f7ca..38e48f12ff96 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java @@ -39,6 +39,7 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.ServerSocket; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -496,6 +497,11 @@ public static NativeQueryRunnerParameters getNativeQueryRunnerParameters() } public static Optional> getExternalWorkerLauncher(String catalogName, String prestoServerPath, int cacheMaxSize, Optional remoteFunctionServerUds, Boolean failOnNestedLoopJoin, boolean isCoordinatorSidecarEnabled) + { + return getExternalWorkerLauncher(catalogName, prestoServerPath, OptionalInt.empty(), cacheMaxSize, remoteFunctionServerUds, failOnNestedLoopJoin, isCoordinatorSidecarEnabled); + } + + public static Optional> getExternalWorkerLauncher(String catalogName, String prestoServerPath, OptionalInt port, int cacheMaxSize, Optional remoteFunctionServerUds, Boolean failOnNestedLoopJoin, boolean isCoordinatorSidecarEnabled) { return Optional.of((workerIndex, discoveryUri) -> { @@ -509,7 +515,8 @@ public static Optional> getExternalWorkerLaunc String configProperties = format("discovery.uri=%s%n" + "presto.version=testversion%n" + "system-memory-gb=4%n" + - "http-server.http.port=0%n", discoveryUri); + "native-sidecar=true%n" + + "http-server.http.port=%d", discoveryUri, port.orElse(0)); if (isCoordinatorSidecarEnabled) { configProperties = format("%s%n" + @@ -572,6 +579,45 @@ public static Optional> getExternalWorkerLaunc }); } + public static Process getNativeSidecarProcess(URI discoveryUri, int port) + throws IOException + { + NativeQueryRunnerParameters nativeQueryRunnerParameters = getNativeQueryRunnerParameters(); + return getNativeSidecarProcess(nativeQueryRunnerParameters.serverBinary.toString(), discoveryUri, port); + } + + public static Process getNativeSidecarProcess(String prestoServerPath, URI discoveryUri, int port) + throws IOException + { + Path tempDirectoryPath = Files.createTempDirectory(PrestoNativeQueryRunnerUtils.class.getSimpleName()); + log.info("Temp directory for Sidecar: %s", tempDirectoryPath.toString()); + + // Write config file + String configProperties = format("discovery.uri=%s%n" + + "presto.version=testversion%n" + + "system-memory-gb=4%n" + + "native-sidecar=true%n" + + "http-server.http.port=%d", discoveryUri, port); + + Files.write(tempDirectoryPath.resolve("config.properties"), configProperties.getBytes()); + Files.write(tempDirectoryPath.resolve("node.properties"), + format("node.id=%s%n" + + "node.internal-address=127.0.0.1%n" + + "node.environment=testing%n" + + "node.location=test-location", UUID.randomUUID()).getBytes()); + + // TODO: sidecars require that a catalog directory exist + Path catalogDirectoryPath = tempDirectoryPath.resolve("catalog"); + Files.createDirectory(catalogDirectoryPath); + + return new ProcessBuilder(prestoServerPath) + .directory(tempDirectoryPath.toFile()) + .redirectErrorStream(true) + .redirectOutput(ProcessBuilder.Redirect.to(tempDirectoryPath.resolve("sidecar.out").toFile())) + .redirectError(ProcessBuilder.Redirect.to(tempDirectoryPath.resolve("sidecar.out").toFile())) + .start(); + } + public static class NativeQueryRunnerParameters { public final Path serverBinary; @@ -618,4 +664,12 @@ private static Table createHiveSymlinkTable(String databaseName, String tableNam Optional.empty(), Optional.empty()); } + + public static int findRandomPortForWorker() + throws IOException + { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } } diff --git a/presto-native-sidecar-plugin/pom.xml b/presto-native-sidecar-plugin/pom.xml index f3e82500967e..be75867363f3 100644 --- a/presto-native-sidecar-plugin/pom.xml +++ b/presto-native-sidecar-plugin/pom.xml @@ -42,6 +42,11 @@ guice + + javax.inject + javax.inject + + com.facebook.airlift configuration @@ -187,6 +192,12 @@ test + + javax.ws.rs + javax.ws.rs-api + test + + com.facebook.presto presto-client diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/ForSidecarInfo.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/ForSidecarInfo.java new file mode 100644 index 000000000000..8bf10b20223d --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/ForSidecarInfo.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; + +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@BindingAnnotation +public @interface ForSidecarInfo +{ +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizer.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizer.java new file mode 100644 index 000000000000..08ab15cbdd40 --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizer.java @@ -0,0 +1,332 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.facebook.presto.common.type.Type; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.SourceLocation; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.EVALUATED; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.COALESCE; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Collections.newSetFromMap; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toMap; + +public class NativeExpressionOptimizer + implements ExpressionOptimizer +{ + private static final Function NO_RESOLUTION = i -> i; + private final FunctionMetadataManager functionMetadataManager; + private final StandardFunctionResolution resolution; + private final NativeSidecarExpressionInterpreter rowExpressionInterpreterService; + + public NativeExpressionOptimizer( + NativeSidecarExpressionInterpreter rowExpressionInterpreterService, + FunctionMetadataManager functionMetadataManager, + StandardFunctionResolution resolution) + { + this.rowExpressionInterpreterService = requireNonNull(rowExpressionInterpreterService, "rowExpressionInterpreterService is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.resolution = requireNonNull(resolution, "resolution is null"); + } + + @Override + public RowExpression optimize(RowExpression expression, Level level, ConnectorSession session) + { + return toRowExpression( + expression.getSourceLocation(), + optimize(expression, level, session, NO_RESOLUTION), + expression.getType()); + } + + @Override + public Object optimize(RowExpression expression, Level level, ConnectorSession session, Function variableResolver) + { + // Collect expressions to optimize + CollectingVisitor collectingVisitor = new CollectingVisitor(functionMetadataManager, level, resolution); + expression.accept(collectingVisitor, variableResolver); + List expressionsToOptimize = collectingVisitor.getExpressionsToOptimize(); + Map expressions = expressionsToOptimize.stream() + .collect(toMap( + Function.identity(), + rowExpression -> toRowExpression( + rowExpression.getSourceLocation(), + rowExpression.accept(new ReplacingVisitor(variableResolver), null), + rowExpression.getType()), + (a, b) -> a)); + if (expressions.isEmpty()) { + return expression; + } + + // Constants can be trivially replaced without invoking the interpreter. Move them into a separate map. + Map constants = new HashMap<>(); + Iterator> entries = expressions.entrySet().iterator(); + while (entries.hasNext()) { + Map.Entry entry = entries.next(); + if (entry.getValue() instanceof ConstantExpression) { + constants.put(entry.getKey(), entry.getValue()); + entries.remove(); + } + } + + // Optimize the expressions using the sidecar interpreter + Map replacements = new HashMap<>(); + if (!expressions.isEmpty()) { + replacements.putAll(rowExpressionInterpreterService.optimizeBatch(session, expressions, level)); + } + + // Add back in the constants + replacements.putAll(constants); + + // Replace all the expressions in the original expression with the optimized expressions + return expression.accept(new ReplacingVisitor(replacements), null); + } + + private static class CollectingVisitor + implements RowExpressionVisitor + { + private final FunctionMetadataManager functionMetadataManager; + private final Level optimizationLevel; + private final StandardFunctionResolution resolution; + private final Set expressionsToOptimize = newSetFromMap(new IdentityHashMap<>()); + + public CollectingVisitor(FunctionMetadataManager functionMetadataManager, Level optimizationLevel, StandardFunctionResolution resolution) + { + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.optimizationLevel = requireNonNull(optimizationLevel, "optimizationLevel is null"); + this.resolution = requireNonNull(resolution, "resolution is null"); + } + + @Override + public Void visitExpression(RowExpression node, Object context) + { + return null; + } + + @Override + public Void visitConstant(ConstantExpression node, Object context) + { + addRowExpressionToOptimize(node); + return null; + } + + @Override + public Void visitVariableReference(VariableReferenceExpression node, Object context) + { + Object value = ((Function) context).apply(node); + if (value == null || value instanceof RowExpression) { + return null; + } + addRowExpressionToOptimize(node); + return null; + } + + @Override + public Void visitCall(CallExpression node, Object context) + { + // If the optimization level is not EVALUATED, then we cannot optimize non-deterministic functions + boolean isDeterministic = functionMetadataManager.getFunctionMetadata(node.getFunctionHandle()).isDeterministic(); + boolean canBeEvaluated = (optimizationLevel.ordinal() < EVALUATED.ordinal() && isDeterministic) || + optimizationLevel.ordinal() == EVALUATED.ordinal(); + + // All arguments must be optimizable in order to evaluate the function + boolean allConstantFoldable = node.getArguments().stream() + .peek(argument -> argument.accept(this, context)) + .reduce(true, (a, b) -> canBeOptimized(b) && a, (a, b) -> a && b); + if (canBeEvaluated && allConstantFoldable) { + addRowExpressionToOptimize(node); + return null; + } + + // If it's a cast and the type is already the same, then it's constant foldable + if (resolution.isCastFunction(node.getFunctionHandle()) + && node.getArguments().size() == 1 + && node.getType().equals(node.getArguments().get(0).getType())) { + addRowExpressionToOptimize(node); + return null; + } + return null; + } + + @Override + public Void visitSpecialForm(SpecialFormExpression node, Object context) + { + // Most special form expressions short circuit, meaning that they potentially don't evaluate all arguments. For example, the AND expression + // will stop evaluating arguments as soon as it finds a false argument. Because a sub-expression could be simplified into a constant, and this + // constant could cause the expression to short circuit, if there is at least one argument which is optimizable, then the entire expression should + // be sent to the sidecar to be optimized. + boolean anyArgumentsOptimizable = node.getArguments().stream() + .peek(argument -> argument.accept(this, context)) + .anyMatch(this::canBeOptimized); + + // If all arguments are constant foldable, then the whole expression is constant foldable + if (anyArgumentsOptimizable) { + addRowExpressionToOptimize(node); + return null; + } + + // If the special form is COALESCE, then we can optimize it if there are any duplicate arguments + if (node.getForm() == COALESCE) { + ImmutableSet.Builder builder = ImmutableSet.builder(); + // Check if there's any duplicate arguments, these can be de-duplicated + for (RowExpression argument : node.getArguments()) { + // The duplicate argument must either be a leaf (variable reference) or constant foldable + if (canBeOptimized(argument)) { + builder.add(argument); + } + } + // If there were any duplicates, or if there's no arguments (cancel out), or if there's only one argument (just return it), + // then it's also constant foldable + boolean canBeOptimized = builder.build().size() <= node.getArguments().size() || node.getArguments().size() <= 1; + if (canBeOptimized) { + addRowExpressionToOptimize(node); + } + } + return null; + } + + @Override + public Void visitLambda(LambdaDefinitionExpression node, Object context) + { + node.getBody().accept(this, (Function) variable -> variable); + if (canBeOptimized(node.getBody())) { + addRowExpressionToOptimize(node.getBody()); + } + return null; + } + + public boolean canBeOptimized(RowExpression rowExpression) + { + return expressionsToOptimize.contains(rowExpression); + } + + private void removeChildren(RowExpression resolvedRowExpression) + { + for (RowExpression rowExpression : resolvedRowExpression.getChildren()) { + expressionsToOptimize.remove(rowExpression); + } + } + + private void addRowExpressionToOptimize(RowExpression original) + { + requireNonNull(original, "original is null"); + removeChildren(original); + expressionsToOptimize.add(original); + } + + public List getExpressionsToOptimize() + { + return ImmutableList.copyOf(expressionsToOptimize); + } + } + + private static class ReplacingVisitor + implements RowExpressionVisitor + { + private final Function resolver; + + public ReplacingVisitor(Map replacements) + { + requireNonNull(replacements, "replacements is null"); + this.resolver = i -> replacements.getOrDefault(i, i); + } + + public ReplacingVisitor(Function variableResolver) + { + requireNonNull(variableResolver, "variableResolver is null"); + this.resolver = i -> i instanceof VariableReferenceExpression ? variableResolver.apply((VariableReferenceExpression) i) : i; + } + + private boolean canBeReplaced(RowExpression rowExpression) + { + return resolver.apply(rowExpression) != rowExpression; + } + + @Override + public Object visitExpression(RowExpression originalExpression, Void context) + { + return resolver.apply(originalExpression); + } + + @Override + public Object visitLambda(LambdaDefinitionExpression lambda, Void context) + { + if (canBeReplaced(lambda.getBody())) { + return new LambdaDefinitionExpression( + lambda.getSourceLocation(), + lambda.getArgumentTypes(), + lambda.getArguments(), + toRowExpression(lambda.getSourceLocation(), resolver.apply(lambda.getBody()), lambda.getBody().getType())); + } + return lambda; + } + + @Override + public Object visitCall(CallExpression call, Void context) + { + if (canBeReplaced(call)) { + return resolver.apply(call); + } + List updatedArguments = call.getArguments().stream() + .map(argument -> toRowExpression(argument.getSourceLocation(), argument.accept(this, context), argument.getType())) + .collect(toImmutableList()); + return new CallExpression(call.getSourceLocation(), call.getDisplayName(), call.getFunctionHandle(), call.getType(), updatedArguments); + } + + @Override + public Object visitSpecialForm(SpecialFormExpression specialForm, Void context) + { + if (canBeReplaced(specialForm)) { + return resolver.apply(specialForm); + } + List updatedArguments = specialForm.getArguments().stream() + .map(argument -> toRowExpression(argument.getSourceLocation(), argument.accept(this, context), argument.getType())) + .collect(toImmutableList()); + return new SpecialFormExpression(specialForm.getSourceLocation(), specialForm.getForm(), specialForm.getType(), updatedArguments); + } + } + + private static RowExpression toRowExpression(Optional sourceLocation, Object object, Type type) + { + requireNonNull(type, "type is null"); + + if (object instanceof RowExpression) { + return (RowExpression) object; + } + + return new ConstantExpression(sourceLocation, object, type); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizerFactory.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizerFactory.java new file mode 100644 index 000000000000..4157cffe2c83 --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizerFactory.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; +import com.facebook.presto.spi.classloader.ThreadContextClassLoader; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerContext; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; +import com.google.inject.Injector; + +import java.util.Map; + +import static java.util.Objects.requireNonNull; + +public class NativeExpressionOptimizerFactory + implements ExpressionOptimizerFactory +{ + private final ClassLoader classLoader; + + public NativeExpressionOptimizerFactory(ClassLoader classLoader) + { + this.classLoader = requireNonNull(classLoader, "classLoader is null"); + } + + @Override + public ExpressionOptimizer createOptimizer(Map config, ExpressionOptimizerContext context) + { + requireNonNull(context, "context is null"); + + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + Bootstrap app = new Bootstrap( + new JsonModule(), + new NativeExpressionsCommunicationModule(), + new NativeExpressionsModule(context.getNodeManager(), context.getRowExpressionSerde(), context.getFunctionMetadataManager(), context.getFunctionResolution())); + + Injector injector = app + .noStrictConfig() + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .quiet() + .initialize(); + return injector.getInstance(NativeExpressionOptimizerProvider.class).createOptimizer(); + } + } + + @Override + public String getName() + { + return "native"; + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizerProvider.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizerProvider.java new file mode 100644 index 000000000000..8efe3b8a77eb --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionOptimizerProvider.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.ExpressionOptimizer; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; + +public class NativeExpressionOptimizerProvider +{ + private final NativeSidecarExpressionInterpreter expressionInterpreterService; + private final FunctionMetadataManager functionMetadataManager; + private final StandardFunctionResolution resolution; + + @Inject + public NativeExpressionOptimizerProvider(NativeSidecarExpressionInterpreter expressionInterpreterService, FunctionMetadataManager functionMetadataManager, StandardFunctionResolution resolution) + { + this.expressionInterpreterService = requireNonNull(expressionInterpreterService, "expressionInterpreterService is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.resolution = requireNonNull(resolution, "resolution is null"); + } + + public ExpressionOptimizer createOptimizer() + { + return new NativeExpressionOptimizer(expressionInterpreterService, functionMetadataManager, resolution); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionsCommunicationModule.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionsCommunicationModule.java new file mode 100644 index 000000000000..fdc6da53f91b --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionsCommunicationModule.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.google.inject.Binder; +import com.google.inject.Module; + +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; + +public class NativeExpressionsCommunicationModule + implements Module +{ + @Override + public void configure(Binder binder) + { + httpClientBinder(binder).bindHttpClient("sidecar", ForSidecarInfo.class); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionsModule.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionsModule.java new file mode 100644 index 000000000000..046b4e60eac7 --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeExpressionsModule.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.facebook.airlift.json.JsonModule; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.RowExpressionSerde; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.RowExpression; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static java.util.Objects.requireNonNull; + +public class NativeExpressionsModule + implements Module +{ + private final NodeManager nodeManager; + private final RowExpressionSerde rowExpressionSerde; + private final FunctionMetadataManager functionMetadataManager; + private final StandardFunctionResolution functionResolution; + + public NativeExpressionsModule(NodeManager nodeManager, RowExpressionSerde rowExpressionSerde, FunctionMetadataManager functionMetadataManager, StandardFunctionResolution functionResolution) + { + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.rowExpressionSerde = requireNonNull(rowExpressionSerde, "rowExpressionSerde is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + } + + @Override + public void configure(Binder binder) + { + // Core dependencies + binder.bind(NodeManager.class).toInstance(nodeManager); + binder.bind(RowExpressionSerde.class).toInstance(rowExpressionSerde); + binder.bind(FunctionMetadataManager.class).toInstance(functionMetadataManager); + binder.bind(StandardFunctionResolution.class).toInstance(functionResolution); + + // JSON dependencies and setup + binder.install(new JsonModule()); + jsonBinder(binder).addDeserializerBinding(RowExpression.class).to(RowExpressionDeserializer.class).in(Scopes.SINGLETON); + jsonBinder(binder).addSerializerBinding(RowExpression.class).to(RowExpressionSerializer.class).in(Scopes.SINGLETON); + jsonCodecBinder(binder).bindListJsonCodec(RowExpression.class); + + binder.bind(NativeSidecarExpressionInterpreter.class).in(Scopes.SINGLETON); + + // The main service provider + binder.bind(NativeExpressionOptimizerProvider.class).in(Scopes.SINGLETON); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeSidecarExpressionInterpreter.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeSidecarExpressionInterpreter.java new file mode 100644 index 000000000000..6374bbce911f --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/NativeSidecarExpressionInterpreter.java @@ -0,0 +1,112 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpUriBuilder; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.Node; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import javax.inject.Inject; + +import java.net.URI; +import java.util.List; +import java.util.Map; + +import static com.facebook.airlift.http.client.JsonBodyGenerator.jsonBodyGenerator; +import static com.facebook.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static com.facebook.airlift.http.client.Request.Builder.preparePost; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.net.HttpHeaders.ACCEPT; +import static com.google.common.net.HttpHeaders.CONTENT_TYPE; +import static com.google.common.net.MediaType.JSON_UTF_8; +import static java.util.Objects.requireNonNull; + +public class NativeSidecarExpressionInterpreter +{ + private static final String PRESTO_TIME_ZONE_HEADER = "X-Presto-Time-Zone"; + private static final String PRESTO_USER_HEADER = "X-Presto-User"; + private static final String PRESTO_EXPRESSION_OPTIMIZER_LEVEL_HEADER = "X-Presto-Expression-Optimizer-Level"; + private static final String SIDECAR_ENDPOINT = "/v1/expressions"; + private final NodeManager nodeManager; + private final HttpClient httpClient; + private final JsonCodec> rowExpressionSerde; + + @Inject + public NativeSidecarExpressionInterpreter(NodeManager nodeManager, @ForSidecarInfo HttpClient httpClient, JsonCodec> rowExpressionSerde) + { + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + this.rowExpressionSerde = requireNonNull(rowExpressionSerde, "rowExpressionSerde is null"); + } + + public Map optimizeBatch(ConnectorSession session, Map expressions, ExpressionOptimizer.Level level) + { + ImmutableList.Builder originalExpressionsBuilder = ImmutableList.builder(); + ImmutableList.Builder resolvedExpressionsBuilder = ImmutableList.builder(); + for (Map.Entry entry : expressions.entrySet()) { + originalExpressionsBuilder.add(entry.getKey()); + resolvedExpressionsBuilder.add(entry.getValue()); + } + List originalExpressions = originalExpressionsBuilder.build(); + List resolvedExpressions = resolvedExpressionsBuilder.build(); + + List optimizedExpressions = httpClient.execute( + getSidecarRequest(session, level, resolvedExpressions), + createJsonResponseHandler(rowExpressionSerde)); + checkArgument( + optimizedExpressions.size() == resolvedExpressions.size(), + "Expected %s optimized expressions, but got %s", + resolvedExpressions.size(), + optimizedExpressions.size()); + + ImmutableMap.Builder result = ImmutableMap.builder(); + for (int i = 0; i < optimizedExpressions.size(); i++) { + result.put(originalExpressions.get(i), optimizedExpressions.get(i)); + } + return result.build(); + } + + private Request getSidecarRequest(ConnectorSession session, Level level, List resolvedExpressions) + { + return preparePost() + .setUri(getLocation()) + .setBodyGenerator(jsonBodyGenerator(rowExpressionSerde, resolvedExpressions)) + .setHeader(CONTENT_TYPE, JSON_UTF_8.toString()) + .setHeader(ACCEPT, JSON_UTF_8.toString()) + .setHeader(PRESTO_TIME_ZONE_HEADER, session.getSqlFunctionProperties().getTimeZoneKey().getId()) + .setHeader(PRESTO_USER_HEADER, session.getUser()) + .setHeader(PRESTO_EXPRESSION_OPTIMIZER_LEVEL_HEADER, level.name()) + .build(); + } + + private URI getLocation() + { + Node sidecarNode = nodeManager.getSidecarNode(); + return HttpUriBuilder.uriBuilder() + .scheme("http") + .host(sidecarNode.getHost()) + .port(sidecarNode.getHostAndPort().getPort()) + .appendPath(SIDECAR_ENDPOINT) + .build(); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/RowExpressionDeserializer.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/RowExpressionDeserializer.java new file mode 100644 index 000000000000..ffc7039d4568 --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/RowExpressionDeserializer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.facebook.presto.spi.RowExpressionSerde; +import com.facebook.presto.spi.relation.RowExpression; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.jsontype.TypeDeserializer; +import com.google.inject.Inject; + +import java.io.IOException; + +import static java.util.Objects.requireNonNull; + +public final class RowExpressionDeserializer + extends JsonDeserializer +{ + private final RowExpressionSerde rowExpressionSerde; + + @Inject + public RowExpressionDeserializer(RowExpressionSerde rowExpressionSerde) + { + this.rowExpressionSerde = requireNonNull(rowExpressionSerde, "rowExpressionSerde is null"); + } + + @Override + public RowExpression deserialize(JsonParser jsonParser, DeserializationContext context) + throws IOException + { + return rowExpressionSerde.deserialize(jsonParser.readValueAsTree().toString()); + } + + @Override + public RowExpression deserializeWithType(JsonParser jsonParser, DeserializationContext context, TypeDeserializer typeDeserializer) + throws IOException + { + return deserialize(jsonParser, context); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/RowExpressionSerializer.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/RowExpressionSerializer.java new file mode 100644 index 000000000000..6ca5dcc4354f --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/session/sql/expressions/RowExpressionSerializer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.facebook.presto.spi.RowExpressionSerde; +import com.facebook.presto.spi.relation.RowExpression; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.google.inject.Inject; + +import java.io.IOException; + +import static java.util.Objects.requireNonNull; + +public final class RowExpressionSerializer + extends JsonSerializer +{ + private final RowExpressionSerde rowExpressionSerde; + + @Inject + public RowExpressionSerializer(RowExpressionSerde rowExpressionSerde) + { + this.rowExpressionSerde = requireNonNull(rowExpressionSerde, "rowExpressionSerde is null"); + } + + @Override + public void serialize(RowExpression rowExpression, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException + { + jsonGenerator.writeRawValue(rowExpressionSerde.serialize(rowExpression)); + } + + @Override + public void serializeWithType(RowExpression rowExpression, JsonGenerator jsonGenerator, SerializerProvider serializerProvider, TypeSerializer typeSerializer) + throws IOException + { + serialize(rowExpression, jsonGenerator, serializerProvider); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/NativeSidecarPlugin.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/NativeSidecarPlugin.java index c719f5a2db80..252e6d52846c 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/NativeSidecarPlugin.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/NativeSidecarPlugin.java @@ -13,11 +13,13 @@ */ package com.facebook.presto.sidecar; +import com.facebook.presto.session.sql.expressions.NativeExpressionOptimizerFactory; import com.facebook.presto.sidecar.nativechecker.NativePlanCheckerProviderFactory; import com.facebook.presto.sidecar.sessionpropertyproviders.NativeSystemSessionPropertyProviderFactory; import com.facebook.presto.spi.CoordinatorPlugin; import com.facebook.presto.spi.plan.PlanCheckerProviderFactory; import com.facebook.presto.spi.session.WorkerSessionPropertyProviderFactory; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; import com.google.common.collect.ImmutableList; public class NativeSidecarPlugin @@ -43,4 +45,9 @@ private static ClassLoader getClassLoader() } return classLoader; } + @Override + public Iterable getExpressionOptimizerFactories() + { + return ImmutableList.of(new NativeExpressionOptimizerFactory(getClassLoader())); + } } diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressionOptimizer.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressionOptimizer.java new file mode 100644 index 000000000000..8adc29b05232 --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressionOptimizer.java @@ -0,0 +1,319 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.facebook.presto.metadata.HandleResolver; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.relational.DelegatingRowExpressionOptimizer; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.tests.expressions.TestExpressions; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.net.URI; +import java.util.Optional; + +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.findRandomPortForWorker; +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.getNativeSidecarProcess; +import static com.facebook.presto.session.sql.expressions.TestNativeExpressions.getExpressionOptimizer; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.EVALUATED; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.type.LikeFunctions.castVarcharToLikePattern; +import static com.facebook.presto.type.LikePatternType.LIKE_PATTERN; +import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.slice.Slices.wrappedBuffer; +import static org.testng.Assert.assertEquals; + +public class TestNativeExpressionOptimizer + extends TestExpressions +{ + private ExpressionOptimizer expressionOptimizer; + private Process sidecar; + private FunctionResolution functionResolution; + + @BeforeClass + public void setup() + throws Exception + { + int port = findRandomPortForWorker(); + URI sidecarUri = URI.create("http://127.0.0.1:" + port); + sidecar = getNativeSidecarProcess(sidecarUri, port); + + this.functionResolution = new FunctionResolution(getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + ExpressionOptimizer optimizer = getExpressionOptimizer(getMetadata(), new HandleResolver()); + expressionOptimizer = new DelegatingRowExpressionOptimizer(getMetadata(), () -> optimizer); + } + + @AfterClass + public void tearDown() + { + sidecar.destroyForcibly(); + } + + // TODO: Pending on native function namespace manager. + @Test(enabled = false) + public void assertLikeOptimizations() + { + assertOptimizedMatches("unbound_string LIKE bound_pattern", "unbound_string LIKE CAST('%el%' AS varchar)"); + } + + // TODO: this test is invalid as it manually constructs an expression which can't be serialized + @Test(enabled = false) + @Override + public void testLikeInvalidUtf8() + { + } + + // TODO: Pending on native function namespace manager. + @Test(enabled = false) + @Override + public void testLike() + { + } + + // TODO: Pending on native function namespace manager. + @Test(enabled = false) + @Override + public void testLikeOptimization() + { + } + + // TODO: Pending on native function namespace manager. + @Test(enabled = false) + @Override + public void testInvalidLike() + { + } + + @Test(enabled = false) + @Override + public void testLambda() + { + assertDoNotOptimize("transform(unbound_array, x -> x + x)", OPTIMIZED); + assertOptimizedEquals("transform(ARRAY[1, 5], x -> x + x)", "transform(ARRAY[1, 5], x -> x + x)"); + assertOptimizedEquals("transform(sequence(1, 5), x -> x + x)", "transform(sequence(1, 5), x -> x + x)"); + assertRowExpressionEquals( + OPTIMIZED, + "transform(sequence(1, unbound_long), x -> cast(json_parse('[1, 2]') AS ARRAY)[1] + x)", + "transform(sequence(1, unbound_long), x -> 1 + x)"); + assertRowExpressionEquals( + OPTIMIZED, + "transform(sequence(1, unbound_long), x -> cast(json_parse('[1, 2]') AS ARRAY)[1] + 1)", + "transform(sequence(1, unbound_long), x -> 2)"); + // TODO: lambdas are currently unsupported by this test +// assertEquals(evaluate("reduce(ARRAY[1, 5], 0, (x, y) -> x + y, x -> x)", true), 6L); + } + + // TODO: current timestamp returns the session timestamp, which is not supported by this test + @Test(enabled = false) + @Override + public void testCurrentTimestamp() + { + } + + // TODO: this function is not supported by this test because its contents are not serializable + @Test(enabled = false) + @Override + public void testMassiveArray() + { + } + + // TODO: apply function is not supported in Presto native. + @Test(enabled = false) + @Override + public void testBind() + { + } + + @Test + @Override + public void testLiterals() + { + optimize("date '2013-04-03' + unbound_interval"); + // TODO: TIME type is unsupported in Presto native. +// optimize("time '03:04:05.321' + unbound_interval"); +// optimize("time '03:04:05.321 UTC' + unbound_interval"); + optimize("timestamp '2013-04-03 03:04:05.321' + unbound_interval"); + optimize("timestamp '2013-04-03 03:04:05.321 UTC' + unbound_interval"); + + optimize("interval '3' day * unbound_long"); + // TODO: Pending on velox PR: https://github.com/facebookincubator/velox/pull/11612. +// optimize("interval '3' year * unbound_integer"); + } + + // TODO: NULL_IF special form is unsupported in Presto native. + @Test(enabled = false) + @Override + public void testNullIf() + { + } + + @Test + @Override + public void testCastBigintToBoundedVarchar() + { + assertEvaluatedEquals("CAST(12300000000 AS varchar(11))", "'12300000000'"); + assertEvaluatedEquals("CAST(12300000000 AS varchar(50))", "'12300000000'"); + + // TODO: Velox permits this cast, but Presto does not +// try { +// evaluate("CAST(12300000000 AS varchar(3))", true); +// fail("Expected to throw an INVALID_CAST_ARGUMENT exception"); +// } +// catch (PrestoException e) { +// try { +// assertEquals(e.getErrorCode(), INVALID_CAST_ARGUMENT.toErrorCode()); +// assertEquals(e.getMessage(), "Value 12300000000 cannot be represented as varchar(3)"); +// } +// catch (Throwable failure) { +// failure.addSuppressed(e); +// throw failure; +// } +// } +// +// try { +// evaluate("CAST(-12300000000 AS varchar(3))", true); +// } +// catch (PrestoException e) { +// try { +// assertEquals(e.getErrorCode(), INVALID_CAST_ARGUMENT.toErrorCode()); +// assertEquals(e.getMessage(), "Value -12300000000 cannot be represented as varchar(3)"); +// } +// catch (Throwable failure) { +// failure.addSuppressed(e); +// throw failure; +// } +// } + } + + // TODO: Non-legacy map subscript is not supported in Presto native. + @Test(enabled = false) + @Override + public void testMapSubscriptMissingKey() + { + } + + // TODO: This test is not applicable in the sidecar so disabled for now. + // To be enabled after changes in sidecar query runner to support Json + // based UDF registration. + @Test(enabled = false) + @Override + public void testCppAggregateFunctionCall() + { + } + + // TODO: This test is not applicable in the sidecar so disabled for now. + // To be enabled after changes in sidecar query runner to support Json + // based UDF registration. + @Test(enabled = false) + @Override + public void testCppFunctionCall() + { + } + + @Override + protected void assertLike(byte[] value, String pattern, boolean expected) + { + CallExpression predicate = call( + "LIKE", + functionResolution.likeVarcharFunction(), + BOOLEAN, + ImmutableList.of( + new ConstantExpression(wrappedBuffer(value), VARCHAR), + new ConstantExpression(castVarcharToLikePattern(utf8Slice(pattern)), LIKE_PATTERN))); + assertEquals(optimizeRowExpression(predicate, EVALUATED), expected); + } + @Override + protected Object evaluate(String expression, boolean deterministic) + { + assertRoundTrip(expression); + RowExpression rowExpression = sqlToRowExpression(expression); + return optimizeRowExpression(rowExpression, EVALUATED); + } + + @Override + protected Object optimize(String expression) + { + assertRoundTrip(expression); + RowExpression parsedExpression = sqlToRowExpression(expression); + return optimizeRowExpression(parsedExpression, OPTIMIZED); + } + + @Override + protected Object optimizeRowExpression(RowExpression expression, Level level) + { + Object optimized = expressionOptimizer.optimize( + expression, + level, + TEST_SESSION.toConnectorSession(), + variable -> { + Symbol symbol = new Symbol(variable.getName()); + Object value = symbolConstant(symbol); + if (value == null) { + return new VariableReferenceExpression(Optional.empty(), symbol.getName(), SYMBOL_TYPES.get(symbol.toSymbolReference())); + } + return value; + }); + return unwrap(optimized); + } + + public Object unwrap(Object result) + { + if (result instanceof ConstantExpression) { + return ((ConstantExpression) result).getValue(); + } + else { + return result; + } + } + + @Override + protected void assertOptimizedEquals(String actual, String expected) + { + Object optimizedActual = optimize(actual); + Object optimizedExpected = optimize(expected); + assertRowExpressionEvaluationEquals(optimizedActual, optimizedExpected); + } + + @Override + protected void assertOptimizedMatches(String actual, String expected) + { + Object actualOptimized = optimize(actual); + Object expectedOptimized = optimize(expected); + assertRowExpressionEvaluationEquals( + actualOptimized, + expectedOptimized); + } + + @Override + protected void assertDoNotOptimize(String expression, Level optimizationLevel) + { + assertRoundTrip(expression); + RowExpression rowExpression = sqlToRowExpression(expression); + Object rowExpressionResult = optimizeRowExpression(rowExpression, optimizationLevel); + assertRowExpressionEvaluationEquals(rowExpressionResult, rowExpression); + } +} diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressions.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressions.java new file mode 100644 index 000000000000..c3310b4f38e2 --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/session/sql/expressions/TestNativeExpressions.java @@ -0,0 +1,224 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.sql.expressions; + +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.facebook.airlift.jaxrs.JsonMapper; +import com.facebook.airlift.jaxrs.testing.JaxrsTestingHttpProcessor; +import com.facebook.airlift.json.JsonModule; +import com.facebook.presto.block.BlockJsonSerde; +import com.facebook.presto.client.NodeVersion; +import com.facebook.presto.common.block.Block; +import com.facebook.presto.common.block.BlockEncoding; +import com.facebook.presto.common.block.BlockEncodingManager; +import com.facebook.presto.common.block.BlockEncodingSerde; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.metadata.HandleJsonModule; +import com.facebook.presto.metadata.HandleResolver; +import com.facebook.presto.metadata.InMemoryNodeManager; +import com.facebook.presto.metadata.InternalNode; +import com.facebook.presto.metadata.InternalNodeManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.nodeManager.PluginNodeManager; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.RowExpressionSerde; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.facebook.presto.sql.expressions.JsonCodecRowExpressionSerde; +import com.facebook.presto.sql.planner.RowExpressionInterpreter; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.type.TypeDeserializer; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.Scopes; +import org.testng.annotations.Test; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.UriBuilder; + +import java.net.URI; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.SERIALIZABLE; +import static com.facebook.presto.sql.planner.LiteralEncoder.toRowExpression; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.inject.multibindings.Multibinder.newSetBinder; +import static java.util.Objects.requireNonNull; +import static org.testng.Assert.assertEquals; + +public class TestNativeExpressions +{ + public static final URI SIDECAR_URI = URI.create("http://127.0.0.1:1122"); + private static final Metadata METADATA = MetadataManager.createTestMetadataManager(); + private static final TestingRowExpressionTranslator TRANSLATOR = new TestingRowExpressionTranslator(METADATA); + @Test + public void testLoadPlugin() + { + ExpressionOptimizer interpreterService = getExpressionOptimizer(METADATA, null); + + // Test the native row expression interpreter service with some simple expressions + RowExpression simpleAddition = compileExpression("1+1"); + RowExpression unnecessaryCoalesce = compileExpression("coalesce(1, 2)"); + + // Assert simple optimizations are performed + assertEquals(interpreterService.optimize(simpleAddition, OPTIMIZED, TEST_SESSION.toConnectorSession()), toRowExpression(2L, simpleAddition.getType())); + assertEquals(interpreterService.optimize(unnecessaryCoalesce, OPTIMIZED, TEST_SESSION.toConnectorSession()), toRowExpression(1L, unnecessaryCoalesce.getType())); + } + + private static RowExpression compileExpression(String expression) + { + return TRANSLATOR.translate(expression, ImmutableMap.of()); + } + + protected static ExpressionOptimizer getExpressionOptimizer(Metadata metadata, HandleResolver handleResolver) + { + // Set up dependencies in main for this module + InMemoryNodeManager nodeManager = getNodeManagerWithSidecar(SIDECAR_URI); + Injector prestoMainInjector = getPrestoMainInjector(metadata, handleResolver); + JsonMapper jsonMapper = prestoMainInjector.getInstance(JsonMapper.class); + RowExpressionSerde rowExpressionSerde = prestoMainInjector.getInstance(RowExpressionSerde.class); + FunctionAndTypeManager functionMetadataManager = prestoMainInjector.getInstance(FunctionAndTypeManager.class); + + // Set up the mock HTTP endpoint that delegates to the Java based row expression interpreter + TestingExpressionOptimizerResource resource = new TestingExpressionOptimizerResource( + metadata.getFunctionAndTypeManager(), + testSessionBuilder().build().toConnectorSession(), + SERIALIZABLE); + JaxrsTestingHttpProcessor jaxrsTestingHttpProcessor = new JaxrsTestingHttpProcessor( + UriBuilder.fromUri(SIDECAR_URI).path("/").build(), + resource, + jsonMapper); + TestingHttpClient testingHttpClient = new TestingHttpClient(jaxrsTestingHttpProcessor); + + // Create the native row expression interpreter service + return createExpressionOptimizer(nodeManager, rowExpressionSerde, testingHttpClient, functionMetadataManager); + } + + private static InMemoryNodeManager getNodeManagerWithSidecar(URI sidecarUri) + { + InMemoryNodeManager nodeManager = new InMemoryNodeManager(); + nodeManager.addNode(new ConnectorId("test"), new InternalNode("test", sidecarUri, NodeVersion.UNKNOWN, false, false, false, true)); + return nodeManager; + } + + private static ExpressionOptimizer createExpressionOptimizer(InternalNodeManager internalNodeManager, RowExpressionSerde rowExpressionSerde, HttpClient httpClient, FunctionAndTypeManager functionMetadataManager) + { + requireNonNull(internalNodeManager, "inMemoryNodeManager is null"); + NodeManager nodeManager = new PluginNodeManager(internalNodeManager); + FunctionResolution functionResolution = new FunctionResolution(functionMetadataManager.getFunctionAndTypeResolver()); + + Bootstrap app = new Bootstrap( + // Specially use a testing HTTP client instead of a real one + binder -> binder.bind(HttpClient.class).annotatedWith(ForSidecarInfo.class).toInstance(httpClient), + // Otherwise use the exact same module as the native row expression interpreter service + new NativeExpressionsModule(nodeManager, rowExpressionSerde, functionMetadataManager, functionResolution)); + + Injector injector = app + .noStrictConfig() + .doNotInitializeLogging() + .setRequiredConfigurationProperties(ImmutableMap.of()) + .quiet() + .initialize(); + return injector.getInstance(NativeExpressionOptimizerProvider.class).createOptimizer(); + } + + private static Injector getPrestoMainInjector(Metadata metadata, HandleResolver handleResolver) + { + Module module = binder -> { + // Installs the JSON codec + binder.install(new JsonModule()); + // Required to deserialize function handles + binder.install(new HandleJsonModule(handleResolver)); + // Required for this test in the JaxrsTestingHttpProcessor because the underlying object mapper + // must be the same as all other object mappers + binder.bind(JsonMapper.class); + + // These dependencies are needed to serialize and deserialize types (found in expressions) + FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager(); + binder.bind(FunctionAndTypeManager.class).toInstance(functionAndTypeManager); + binder.bind(TypeManager.class).toInstance(functionAndTypeManager); + jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); + newSetBinder(binder, Type.class); + + // These dependencies are needed to serialize and deserialize blocks (found in constant values of expressions) + binder.bind(BlockEncodingSerde.class).to(BlockEncodingManager.class).in(Scopes.SINGLETON); + newSetBinder(binder, BlockEncoding.class); + jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class); + jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class); + + // Create the serde which is used by the plugin to serialize and deserialize expressions + jsonCodecBinder(binder).bindJsonCodec(RowExpression.class); + binder.bind(RowExpressionSerde.class).to(JsonCodecRowExpressionSerde.class).in(Scopes.SINGLETON); + }; + Bootstrap app = new Bootstrap(ImmutableList.of(module)); + Injector injector = app + .doNotInitializeLogging() + .quiet() + .initialize(); + return injector; + } + + @Path("/v1/expressions") + public static class TestingExpressionOptimizerResource + { + private final FunctionAndTypeManager functionAndTypeManager; + private final ConnectorSession connectorSession; + private final ExpressionOptimizer.Level level; + + public TestingExpressionOptimizerResource(FunctionAndTypeManager functionAndTypeManager, ConnectorSession connectorSession, ExpressionOptimizer.Level level) + { + this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); + this.connectorSession = requireNonNull(connectorSession, "connectorSession is null"); + this.level = requireNonNull(level, "level is null"); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public List post(List rowExpressions) + { + Map input = rowExpressions.stream().collect(toImmutableMap(i -> i, i -> i)); + Map optimizedExpressions = new HashMap<>(); + input.forEach((key, value) -> optimizedExpressions.put( + key, + new RowExpressionInterpreter(key, functionAndTypeManager, connectorSession, level).optimize())); + ImmutableList.Builder builder = ImmutableList.builder(); + for (RowExpression inputExpression : rowExpressions) { + builder.add(toRowExpression(optimizedExpressions.get(inputExpression), inputExpression.getType())); + } + return builder.build(); + } + } +} diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java index 4ac96f6f837f..b5aea38f73e0 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java @@ -12,8 +12,10 @@ * limitations under the License. */ package com.facebook.presto.sidecar; +import com.facebook.presto.session.sql.expressions.NativeExpressionOptimizerFactory; import com.facebook.presto.sidecar.sessionpropertyproviders.NativeSystemSessionPropertyProviderFactory; import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; public class NativeSidecarPluginQueryRunnerUtils { @@ -23,5 +25,7 @@ public static void setupNativeSidecarPlugin(QueryRunner queryRunner) { queryRunner.installCoordinatorPlugin(new NativeSidecarPlugin()); queryRunner.loadSessionPropertyProvider(NativeSystemSessionPropertyProviderFactory.NAME); + queryRunner.getExpressionManager().addExpressionOptimizerFactory(new NativeExpressionOptimizerFactory(ClassLoader.getSystemClassLoader())); + queryRunner.getExpressionManager().loadExpressionOptimizerFactory(ImmutableMap.builder().put("expression-manager-factory.name", "native").build()); } } diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java new file mode 100644 index 000000000000..eacec8962bba --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar; + +import com.facebook.presto.spi.CoordinatorPlugin; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerContext; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.google.common.collect.Iterables.getOnlyElement; + +public class TestNativeSidecarPlugin +{ + @Test + public void testLoadPlugin() + { + CoordinatorPlugin plugin = new NativeSidecarPlugin(); + Iterable serviceFactories = plugin.getExpressionOptimizerFactories(); + ExpressionOptimizerFactory factory = getOnlyElement(serviceFactories); + factory.createOptimizer( + ImmutableMap.of(), + new ExpressionOptimizerContext( + new UnimplementedNodeManager(), + new UnimplementedRowExpressionSerde(), + new UnimplementedFunctionMetadataManager(), + new UnimplementedFunctionResolution())); + } +} diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedFunctionMetadataManager.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedFunctionMetadataManager.java new file mode 100644 index 000000000000..b321e69b423a --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedFunctionMetadataManager.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar; + +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionMetadataManager; + +class UnimplementedFunctionMetadataManager + implements FunctionMetadataManager +{ + @Override + public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } +} diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedFunctionResolution.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedFunctionResolution.java new file mode 100644 index 000000000000..a747d1d4b1ff --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedFunctionResolution.java @@ -0,0 +1,229 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar; + +import com.facebook.presto.common.function.OperatorType; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.StandardFunctionResolution; + +import java.util.List; + +class UnimplementedFunctionResolution + implements StandardFunctionResolution +{ + @Override + public FunctionHandle notFunction() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isNotFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle negateFunction(Type type) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isNegateFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle likeVarcharFunction() + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle likeCharFunction(Type valueType) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isLikeFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle likePatternFunction() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isLikePatternFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle arrayConstructor(List argumentTypes) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle arithmeticFunction(OperatorType operator, Type leftType, Type rightType) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isArithmeticFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle comparisonFunction(OperatorType operator, Type leftType, Type rightType) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isComparisonFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isEqualsFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle betweenFunction(Type valueType, Type lowerBoundType, Type upperBoundType) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isBetweenFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle subscriptFunction(Type baseType, Type indexType) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isSubscriptFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCastFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCountFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle countFunction() + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle countFunction(Type valueType) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isMaxFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle maxFunction(Type valueType) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle greatestFunction(List valueTypes) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isMinFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle minFunction(Type valueType) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle leastFunction(List valueTypes) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isApproximateCountDistinctFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle approximateCountDistinctFunction(Type valueType) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isApproximateSetFunction(FunctionHandle functionHandle) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle approximateSetFunction(Type valueType) + { + throw new UnsupportedOperationException(); + } + + @Override + public FunctionHandle lookupBuiltInFunction(String functionName, List inputTypes) + { + throw new UnsupportedOperationException(); + } +} diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedNodeManager.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedNodeManager.java new file mode 100644 index 000000000000..38f16b9d8c95 --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedNodeManager.java @@ -0,0 +1,117 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar; + +import com.facebook.presto.spi.HostAddress; +import com.facebook.presto.spi.Node; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.NodePoolType; + +import java.net.URI; +import java.util.Set; + +class UnimplementedNodeManager + implements NodeManager +{ + @Override + public Set getAllNodes() + { + throw new UnsupportedOperationException(); + } + + @Override + public Set getWorkerNodes() + { + throw new UnsupportedOperationException(); + } + + @Override + public Node getCurrentNode() + { + return new Node() + { + @Override + public String getHost() + { + throw new UnsupportedOperationException(); + } + + @Override + public HostAddress getHostAndPort() + { + throw new UnsupportedOperationException(); + } + + @Override + public URI getHttpUri() + { + throw new UnsupportedOperationException(); + } + + @Override + public String getNodeIdentifier() + { + throw new UnsupportedOperationException(); + } + + @Override + public String getVersion() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCoordinator() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isResourceManager() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCatalogServer() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCoordinatorSidecar() + { + throw new UnsupportedOperationException(); + } + + @Override + public NodePoolType getPoolType() + { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public Node getSidecarNode() + { + throw new UnsupportedOperationException(); + } + + @Override + public String getEnvironment() + { + throw new UnsupportedOperationException(); + } +} diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedRowExpressionSerde.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedRowExpressionSerde.java new file mode 100644 index 000000000000..2a71b6908f74 --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/UnimplementedRowExpressionSerde.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar; + +import com.facebook.presto.spi.RowExpressionSerde; +import com.facebook.presto.spi.relation.RowExpression; + +class UnimplementedRowExpressionSerde + implements RowExpressionSerde +{ + @Override + public String serialize(RowExpression expression) + { + throw new UnsupportedOperationException(); + } + + @Override + public RowExpression deserialize(String value) + { + throw new UnsupportedOperationException(); + } +} diff --git a/presto-openapi/src/main/resources/expressions.yaml b/presto-openapi/src/main/resources/expressions.yaml new file mode 100644 index 000000000000..8d1e3ef5d937 --- /dev/null +++ b/presto-openapi/src/main/resources/expressions.yaml @@ -0,0 +1,173 @@ +openapi: 3.0.0 +info: + title: Presto Expression API + description: API for evaluating and simplifying row expressions in Presto + version: "1" +servers: + - url: http://localhost:8080 + description: Presto endpoint when running locally +paths: + /v1/expressions: + post: + summary: Simplify the list of row expressions + description: This endpoint takes in a list of row expressions and attempts to simplify them to their simplest logical equivalent expression. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/RowExpressions' + required: true + responses: + '200': + description: Results + content: + application/json: + schema: + $ref: '#/components/schemas/RowExpressions' +components: + schemas: + RowExpressions: + type: array + maxItems: 100 + items: + $ref: "#/components/schemas/RowExpression" + RowExpression: + oneOf: + - $ref: "#/components/schemas/ConstantExpression" + - $ref: "#/components/schemas/VariableReferenceExpression" + - $ref: "#/components/schemas/InputReferenceExpression" + - $ref: "#/components/schemas/LambdaDefinitionExpression" + - $ref: "#/components/schemas/SpecialFormExpression" + - $ref: "#/components/schemas/CallExpression" + RowExpressionParent: + type: object + properties: + sourceLocation: + $ref: "#/components/schemas/SourceLocation" + SourceLocation: + description: The source location of the row expression in the original query, referencing the line and the column of the query. + type: object + properties: + line: + type: integer + column: + type: integer + ConstantExpression: + description: A constant expression is a row expression that represents a constant value. The value attribute is the constant value. + allOf: + - $ref: "#/components/schemas/RowExpressionParent" + - type: object + properties: + "@type": + type: string + enum : ["constant"] + typeSignature: + type: string + valueBlock: + type: string + VariableReferenceExpression: + description: A variable reference expression is a row expression that represents a reference to a variable. The name attribute indicates the name of the variable. + allOf: + - $ref: "#/components/schemas/RowExpressionParent" + - type: object + properties: + "@type": + type: string + enum : ["variable"] + typeSignature: + type: string + name: + type: string + InputReferenceExpression: + description: > + An input reference expression is a row expression that represents a reference to a column in the input schema. The field attribute indicates the index of the column in the + input schema. + allOf: + - $ref: "#/components/schemas/RowExpressionParent" + - type: object + properties: + "@type": + type: string + enum : ["input"] + typeSignature: + type: string + field: + type: integer + LambdaDefinitionExpression: + description: > + A lambda definition expression is a row expression that represents a lambda function. The lambda function is defined by a list of argument types, a list of argument names, + and a body expression. + allOf: + - $ref: "#/components/schemas/RowExpressionParent" + - type: object + properties: + "@type": + type: string + enum : ["lambda"] + argumentTypeSignatures: + type: array + items: + type: string + arguments: + type: array + items: + type: string + body: + $ref: "#/components/schemas/RowExpression" + SpecialFormExpression: + description: > + A special form expression is a row expression that represents a special language construct. The form attribute indicates the specific form of the special form, + which is a well known list, and with each having special semantics. The arguments attribute is a list of row expressions that are the arguments to the special form, with + each form taking in a specific number of arguments. + allOf: + - $ref: "#/components/schemas/RowExpressionParent" + - type: object + properties: + "@type": + type: string + enum : ["special"] + form: + type: string + enum: ["IF","NULL_IF","SWITCH","WHEN","IS_NULL","COALESCE","IN","AND","OR","DEREFERENCE","ROW_CONSTRUCTOR","BIND"] + returnTypeSignature: + type: string + arguments: + type: array + items: + $ref: "#/components/schemas/RowExpression" + CallExpression: + description: > + A call expression is a row expression that represents a call to a function. The functionHandle attribute is an opaque handle to the function that is being called. + The arguments attribute is a list of row expressions that are the arguments to the function. + allOf: + - $ref: "#/components/schemas/RowExpressionParent" + - type: object + properties: + "@type": + type: string + enum : ["call"] + displayName: + type: string + functionHandle: + $ref: "#/components/schemas/FunctionHandle" + returnTypeSignature: + type: string + arguments: + type: array + items: + $ref: "#/components/schemas/RowExpression" + FunctionHandle: + description: An opaque handle to a function that may be invoked. This is interpreted by the registered function namespace manager. + anyOf: + - $ref: "#/components/schemas/OpaqueFunctionHandle" + - $ref: "#/components/schemas/SqlFunctionHandle" + OpaqueFunctionHandle: + type: object + properties: {} # any opaque object may be passed and interpreted by a function namespace manager + SqlFunctionHandle: + type: object + properties: + functionId: + type: string + version: + type: string diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java index b84557391a8b..a6e3705e9866 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java @@ -135,6 +135,7 @@ import com.facebook.presto.spi.ConnectorTypeSerde; import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; +import com.facebook.presto.spi.RowExpressionSerde; import com.facebook.presto.spi.analyzer.ViewDefinition; import com.facebook.presto.spi.memory.ClusterMemoryPoolManager; import com.facebook.presto.spi.plan.SimplePlanFragment; @@ -142,6 +143,7 @@ import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.PredicateCompiler; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spiller.GenericPartitioningSpillerFactory; import com.facebook.presto.spiller.GenericSpillerFactory; @@ -175,6 +177,7 @@ import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; +import com.facebook.presto.sql.expressions.JsonCodecRowExpressionSerde; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -304,6 +307,7 @@ protected void setup(Binder binder) jsonCodecBinder(binder).bindJsonCodec(BroadcastFileInfo.class); jsonCodecBinder(binder).bindJsonCodec(SimplePlanFragment.class); binder.bind(SimplePlanFragmentSerde.class).to(JsonCodecSimplePlanFragmentSerde.class).in(Scopes.SINGLETON); + jsonCodecBinder(binder).bindJsonCodec(RowExpression.class); // smile codecs smileCodecBinder(binder).bindSmileCodec(TaskSource.class); @@ -350,6 +354,7 @@ protected void setup(Binder binder) // expression manager binder.bind(ExpressionOptimizerManager.class).in(Scopes.SINGLETON); + binder.bind(RowExpressionSerde.class).to(JsonCodecRowExpressionSerde.class).in(Scopes.SINGLETON); // tracer provider managers binder.bind(TracerProviderManager.class).in(Scopes.SINGLETON); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/RowExpressionSerde.java b/presto-spi/src/main/java/com/facebook/presto/spi/RowExpressionSerde.java new file mode 100644 index 000000000000..ab5381aa2556 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/RowExpressionSerde.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi; + +import com.facebook.presto.spi.relation.RowExpression; + +public interface RowExpressionSerde +{ + String serialize(RowExpression expression); + + RowExpression deserialize(String value); +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java index c9a6f84aaaa1..51b7cce149d8 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java @@ -14,6 +14,7 @@ package com.facebook.presto.spi.sql.planner; import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.RowExpressionSerde; import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; @@ -22,12 +23,14 @@ public class ExpressionOptimizerContext { private final NodeManager nodeManager; + private final RowExpressionSerde rowExpressionSerde; private final FunctionMetadataManager functionMetadataManager; private final StandardFunctionResolution functionResolution; - public ExpressionOptimizerContext(NodeManager nodeManager, FunctionMetadataManager functionMetadataManager, StandardFunctionResolution functionResolution) + public ExpressionOptimizerContext(NodeManager nodeManager, RowExpressionSerde rowExpressionSerde, FunctionMetadataManager functionMetadataManager, StandardFunctionResolution functionResolution) { this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.rowExpressionSerde = requireNonNull(rowExpressionSerde, "rowExpressionSerde is null"); this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); } @@ -37,6 +40,11 @@ public NodeManager getNodeManager() return nodeManager; } + public RowExpressionSerde getRowExpressionSerde() + { + return rowExpressionSerde; + } + public FunctionMetadataManager getFunctionMetadataManager() { return functionMetadataManager; diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java index bedd15a5d9e1..09b03cfb0409 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java @@ -26,10 +26,12 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.security.AccessDeniedException; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; +import com.facebook.presto.sql.expressions.JsonCodecRowExpressionSerde; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.Plan; @@ -61,6 +63,7 @@ import java.util.OptionalLong; import java.util.function.Consumer; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.sql.SqlFormatter.formatSql; import static com.facebook.presto.transaction.TransactionBuilder.transaction; @@ -576,7 +579,8 @@ private QueryExplainer getQueryExplainer() featuresConfig, new ExpressionOptimizerManager( new PluginNodeManager(new InMemoryNodeManager()), - queryRunner.getMetadata().getFunctionAndTypeManager())) + queryRunner.getMetadata().getFunctionAndTypeManager(), + new JsonCodecRowExpressionSerde(jsonCodec(RowExpression.class)))) .getPlanningTimeOptimizers(); return new QueryExplainer( optimizers, From 1e8db6112841a286161506771f7bc7f56757515e Mon Sep 17 00:00:00 2001 From: Pramod Date: Tue, 10 Sep 2024 21:43:47 +0530 Subject: [PATCH 5/5] [native] Add expression optimization support in sidecar --- .../presto_cpp/main/CMakeLists.txt | 2 + .../presto_cpp/main/PrestoServer.cpp | 10 + .../presto_cpp/main/PrestoServer.h | 2 + .../presto_cpp/main/expression/CMakeLists.txt | 32 + .../expression/RowExpressionEvaluator.cpp | 637 ++++++++++++++++++ .../main/expression/RowExpressionEvaluator.h | 126 ++++ .../main/expression/tests/CMakeLists.txt | 26 + .../tests/RowExpressionEvaluatorTest.cpp | 187 +++++ .../tests/data/SimpleExpressionsExpected.json | 17 + .../tests/data/SimpleExpressionsInput.json | 278 ++++++++ .../tests/data/SpecialFormExpected.json | 17 + .../tests/data/SpecialFormInput.json | 193 ++++++ .../main/types/PrestoToVeloxExpr.cpp | 35 +- .../presto_cpp/main/types/PrestoToVeloxExpr.h | 24 + .../PrestoNativeQueryRunnerUtils.java | 2 +- 15 files changed, 1563 insertions(+), 25 deletions(-) create mode 100644 presto-native-execution/presto_cpp/main/expression/CMakeLists.txt create mode 100644 presto-native-execution/presto_cpp/main/expression/RowExpressionEvaluator.cpp create mode 100644 presto-native-execution/presto_cpp/main/expression/RowExpressionEvaluator.h create mode 100644 presto-native-execution/presto_cpp/main/expression/tests/CMakeLists.txt create mode 100644 presto-native-execution/presto_cpp/main/expression/tests/RowExpressionEvaluatorTest.cpp create mode 100644 presto-native-execution/presto_cpp/main/expression/tests/data/SimpleExpressionsExpected.json create mode 100644 presto-native-execution/presto_cpp/main/expression/tests/data/SimpleExpressionsInput.json create mode 100644 presto-native-execution/presto_cpp/main/expression/tests/data/SpecialFormExpected.json create mode 100644 presto-native-execution/presto_cpp/main/expression/tests/data/SpecialFormInput.json diff --git a/presto-native-execution/presto_cpp/main/CMakeLists.txt b/presto-native-execution/presto_cpp/main/CMakeLists.txt index abe71dd6f3fa..e27854d292cc 100644 --- a/presto-native-execution/presto_cpp/main/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/CMakeLists.txt @@ -13,6 +13,7 @@ add_subdirectory(operators) add_subdirectory(types) add_subdirectory(http) add_subdirectory(common) +add_subdirectory(expression) add_subdirectory(thrift) add_library( @@ -51,6 +52,7 @@ target_link_libraries( presto_http presto_operators presto_velox_conversion + presto_expr_eval velox_aggregates velox_caching velox_common_base diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp index a01eb58bdd17..5be83d3754fd 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp @@ -1539,6 +1539,16 @@ void PrestoServer::registerSidecarEndpoints() { downstream, json(response), http::kHttpUnprocessableContent); } }); + + rowExpressionEvaluator_ = + std::make_unique(); + httpServer_->registerPost( + "/v1/expressions", + [&](proxygen::HTTPMessage* message, + const std::vector>& body, + proxygen::ResponseHandler* downstream) { + return rowExpressionEvaluator_->evaluate(message, body, downstream); + }); } protocol::NodeStatus PrestoServer::fetchNodeStatus() { diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.h b/presto-native-execution/presto_cpp/main/PrestoServer.h index d364befaa1b6..e8cf348d1c9e 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.h +++ b/presto-native-execution/presto_cpp/main/PrestoServer.h @@ -25,6 +25,7 @@ #include "presto_cpp/main/PeriodicHeartbeatManager.h" #include "presto_cpp/main/PrestoExchangeSource.h" #include "presto_cpp/main/PrestoServerOperations.h" +#include "presto_cpp/main/expression/RowExpressionEvaluator.h" #include "presto_cpp/main/types/VeloxPlanValidator.h" #include "velox/common/caching/AsyncDataCache.h" #include "velox/common/memory/MemoryAllocator.h" @@ -288,6 +289,7 @@ class PrestoServer { std::string address_; std::string nodeLocation_; folly::SSLContextPtr sslContext_; + std::unique_ptr rowExpressionEvaluator_; }; } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/expression/CMakeLists.txt b/presto-native-execution/presto_cpp/main/expression/CMakeLists.txt new file mode 100644 index 000000000000..1e1c970547fc --- /dev/null +++ b/presto-native-execution/presto_cpp/main/expression/CMakeLists.txt @@ -0,0 +1,32 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +add_library(presto_expr_eval RowExpressionEvaluator.cpp) + +target_link_libraries( + presto_expr_eval + presto_type_converter + presto_types + presto_protocol + presto_http + velox_coverage_util + velox_parse_expression + velox_parse_parser + velox_presto_serializer + velox_serialization + velox_type_parser + ${FOLLY_WITH_DEPENDENCIES}) + +if(PRESTO_ENABLE_TESTING) + add_subdirectory(tests) +endif() diff --git a/presto-native-execution/presto_cpp/main/expression/RowExpressionEvaluator.cpp b/presto-native-execution/presto_cpp/main/expression/RowExpressionEvaluator.cpp new file mode 100644 index 000000000000..3972bbc9934b --- /dev/null +++ b/presto-native-execution/presto_cpp/main/expression/RowExpressionEvaluator.cpp @@ -0,0 +1,637 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "presto_cpp/main/expression/RowExpressionEvaluator.h" +#include +#include "presto_cpp/main/common/Utils.h" +#include "velox/common/encode/Base64.h" +#include "velox/exec/ExchangeQueue.h" +#include "velox/expression/EvalCtx.h" +#include "velox/expression/Expr.h" +#include "velox/expression/ExprCompiler.h" +#include "velox/expression/FieldReference.h" + +using namespace facebook::presto; +using namespace facebook::velox; + +namespace facebook::presto::expression { + +namespace { + +protocol::TypeSignature getTypeSignature(const TypePtr& type) { + std::string typeSignature; + if (type->parameters().empty()) { + typeSignature = type->toString(); + boost::algorithm::to_lower(typeSignature); + } else if (type->isDecimal()) { + typeSignature = type->toString(); + } else { + std::vector childTypes; + if (type->isRow()) { + typeSignature = "row("; + childTypes = asRowType(type)->children(); + } else if (type->isArray()) { + typeSignature = "array("; + childTypes = type->asArray().children(); + } else if (type->isMap()) { + typeSignature = "map("; + const auto mapType = type->asMap(); + childTypes = {mapType.keyType(), mapType.valueType()}; + } else { + VELOX_USER_FAIL("Invalid type {}", type->toString()); + } + + if (!childTypes.empty()) { + auto numChildren = childTypes.size(); + for (auto i = 0; i < numChildren - 1; i++) { + typeSignature += fmt::format("{},", getTypeSignature(childTypes[i])); + } + typeSignature += getTypeSignature(childTypes[numChildren - 1]); + } + typeSignature += ")"; + } + + return typeSignature; +} + +json toVariableReferenceExpression( + const std::shared_ptr& fieldReference) { + protocol::VariableReferenceExpression vexpr; + vexpr.name = fieldReference->name(); + vexpr._type = "variable"; + vexpr.type = getTypeSignature(fieldReference->type()); + json res; + protocol::to_json(res, vexpr); + + return res; +} + +bool isPrestoSpecialForm(const std::string& name) { + static const std::unordered_set kPrestoSpecialForms = { + "and", + "coalesce", + "if", + "in", + "is_null", + "or", + "switch", + "when", + "null_if"}; + return kPrestoSpecialForms.count(name) != 0; +} + +json::array_t getInputExpressions( + const std::vector>& body) { + std::ostringstream oss; + for (auto& buf : body) { + oss << std::string((const char*)buf->data(), buf->length()); + } + return json::parse(oss.str()); +} +} // namespace + +// ValueBlock in ConstantExpression requires only the column from the serialized +// PrestoPage without the page header. +std::string RowExpressionConverter::getValueBlock(const VectorPtr& vector) { + std::ostringstream output; + serde_->serializeSingleColumn(vector, nullptr, pool_.get(), &output); + const auto serialized = output.str(); + const auto serializedSize = serialized.size(); + return velox::encoding::Base64::encode(serialized.c_str(), serializedSize); +} + +std::shared_ptr +RowExpressionConverter::getConstantRowExpression( + const std::shared_ptr& constantExpr) { + protocol::ConstantExpression cexpr; + cexpr.type = getTypeSignature(constantExpr->type()); + cexpr.valueBlock.data = getValueBlock(constantExpr->value()); + return std::make_shared(cexpr); +} + +// TODO: Remove this once native plugin supports evaluation of current_user. +std::shared_ptr +RowExpressionConverter::getCurrentUser(const std::string& currentUser) { + protocol::ConstantExpression cexpr; + cexpr.type = getTypeSignature(VARCHAR()); + cexpr.valueBlock.data = getValueBlock( + BaseVector::createConstant(VARCHAR(), currentUser, 1, pool_.get())); + return std::make_shared(cexpr); +} + +json RowExpressionConverter::getRowConstructorSpecialForm( + const exec::ExprPtr& expr, + const json& input) { + json res; + res["@type"] = "special"; + res["form"] = "ROW_CONSTRUCTOR"; + res["returnType"] = getTypeSignature(expr->type()); + + res["arguments"] = json::array(); + auto exprInputs = expr->inputs(); + if (!exprInputs.empty()) { + for (const auto& exprInput : exprInputs) { + res["arguments"].push_back(veloxExprToRowExpression(exprInput, input)); + } + } else if ( + auto constantExpr = + std::dynamic_pointer_cast(expr)) { + auto value = constantExpr->value(); + auto* constVector = value->as>(); + auto* rowVector = constVector->valueVector()->as(); + auto type = asRowType(constantExpr->type()); + auto children = rowVector->children(); + auto size = children.size(); + + json j; + protocol::ConstantExpression cexpr; + for (auto i = 0; i < size; i++) { + cexpr.type = getTypeSignature(type->childAt(i)); + cexpr.valueBlock.data = getValueBlock(rowVector->childAt(i)); + protocol::to_json(j, cexpr); + res["arguments"].push_back(j); + } + } + + return res; +} + +// When the second value in the returned pair is true, the arguments for switch +// special form are returned. Otherwise, the switch expression has been +// simplified and the first value corresponding to the switch case that always +// evaluates to true is returned. +std::pair RowExpressionConverter::getSwitchSpecialFormArgs( + const exec::ExprPtr& expr, + const json& input) { + json::array_t inputArgs = input["arguments"]; + auto numArgs = inputArgs.size(); + json::array_t result = json::array(); + const std::vector exprInputs = expr->inputs(); + const auto numInputs = exprInputs.size(); + + auto getWhenSpecialForm = [&](const json::array_t& whenArgs, + const vector_size_t idx) -> json { + json when; + when["@type"] = "special"; + when["form"] = "WHEN"; + when["arguments"] = whenArgs; + when["returnType"] = getTypeSignature(exprInputs[idx + 1]->type()); + return when; + }; + + // The searched form of the conditional expression needs to be handled + // differently from the simple form. The searched form can be detected by the + // presence of a boolean value in the first argument. This default boolean + // argument is not present in the Velox switch expression, so it is added to + // the arguments of output switch expression unchanged. + if (inputArgs[0].at("@type") == "constant" && + inputArgs[0].at("type") == "boolean") { + result.emplace_back(inputArgs[0]); + for (auto i = 0; i < numInputs - 1; i += 2) { + const vector_size_t argsIdx = i / 2 + 1; + json::array_t inputWhenArgs = inputArgs[argsIdx].at("arguments"); + json::array_t whenArgs; + whenArgs.emplace_back( + veloxExprToRowExpression(exprInputs[i], inputWhenArgs[0])); + whenArgs.emplace_back( + veloxExprToRowExpression(exprInputs[i + 1], inputWhenArgs[1])); + + result.emplace_back(getWhenSpecialForm(whenArgs, i)); + } + } else { + // The case 'expression' in simple form of conditional cannot be inferred + // from Velox since it could evaluate all when clauses to true or false, so + // we get it from the input json. + result.emplace_back(inputArgs[0]); + for (auto i = 0; i < numInputs - 1; i += 2) { + json::array_t whenArgs; + const vector_size_t argsIdx = i / 2 + 1; + const auto& caseValue = exprInputs[i + 1]; + json::array_t inputWhenArgs = inputArgs[argsIdx].at("arguments"); + + if (exprInputs[i]->isConstant()) { + auto constantExpr = + std::dynamic_pointer_cast(exprInputs[i]); + if (auto constVector = + constantExpr->value()->as>()) { + if (constVector->valueAt(0)) { + if (result.size() == 1) { + // This is the first case statement that evaluates to true, so + // return the expression corresponding to this case. + return { + json::array( + {veloxExprToRowExpression(caseValue, inputWhenArgs[1])}), + false}; + } else { + // If the case has been constant folded to false in the Velox + // switch expression, we do not have access to the expression + // inputs in Velox anymore. So we return the corresponding + // argument from the input switch expression. + result.emplace_back(inputArgs[argsIdx]); + } + } else { + // Skip cases that evaluate to false from the output switch + // expression's arguments. + continue; + } + } else { + whenArgs.emplace_back(getConstantRowExpression(constantExpr)); + } + } else { + VELOX_USER_CHECK(!exprInputs[i]->inputs().empty()); + const auto& matchExpr = exprInputs[i]->inputs().back(); + whenArgs.emplace_back( + veloxExprToRowExpression(matchExpr, inputWhenArgs[0])); + } + + whenArgs.emplace_back( + veloxExprToRowExpression(caseValue, inputWhenArgs[1])); + result.emplace_back(getWhenSpecialForm(whenArgs, i)); + } + } + + // Else clause. + if (numInputs % 2 != 0) { + result.push_back(veloxExprToRowExpression( + exprInputs[numInputs - 1], inputArgs[numArgs - 1])); + } + return {result, true}; +} + +json RowExpressionConverter::getSpecialForm( + const exec::ExprPtr& expr, + const json& input) { + json res; + res["@type"] = "special"; + std::string form; + if (input.contains("form")) { + form = input["form"]; + } else { + // If input json is a call expression instead of a special form, for cases + // like 'is_null', the key 'form' will not be present in the input json. + form = expr->name(); + } + // Presto requires the field form to be in upper case. + std::transform(form.begin(), form.end(), form.begin(), ::toupper); + res["form"] = form; + auto exprInputs = expr->inputs(); + res["arguments"] = json::array(); + + // Arguments for switch expression include special form expression 'when' + // so it is constructed separately. If the switch expression evaluation found + // a case that always evaluates to true, the second value in pair switchResult + // will be false and the first value in pair will contain the value + // corresponding to the simplified case. + if (form == "SWITCH") { + auto switchResult = getSwitchSpecialFormArgs(expr, input); + if (switchResult.second) { + res["arguments"] = switchResult.first; + } else { + return switchResult.first.front(); + } + } else { + json::array_t inputArguments = input["arguments"]; + const auto numInputs = exprInputs.size(); + VELOX_USER_CHECK_LE(numInputs, inputArguments.size()); + for (auto i = 0; i < numInputs; i++) { + res["arguments"].push_back( + veloxExprToRowExpression(exprInputs[i], inputArguments[i])); + } + } + res["returnType"] = getTypeSignature(expr->type()); + + return res; +} + +json RowExpressionConverter::toConstantRowExpression( + const velox::exec::ExprPtr& expr) { + json res; + auto constantExpr = std::dynamic_pointer_cast(expr); + VELOX_USER_CHECK_NOT_NULL(constantExpr); + auto constantRowExpr = getConstantRowExpression(constantExpr); + protocol::to_json(res, constantRowExpr); + return res; +} + +json RowExpressionConverter::toCallRowExpression( + const exec::ExprPtr& expr, + const json& input) { + json res; + res["@type"] = "call"; + protocol::Signature signature; + std::string exprName = expr->name(); + if (veloxToPrestoOperatorMap_.find(expr->name()) != + veloxToPrestoOperatorMap_.end()) { + exprName = veloxToPrestoOperatorMap_.at(expr->name()); + } + signature.name = exprName; + res["displayName"] = exprName; + signature.kind = protocol::FunctionKind::SCALAR; + signature.typeVariableConstraints = {}; + signature.longVariableConstraints = {}; + signature.returnType = getTypeSignature(expr->type()); + + std::vector argumentTypes; + auto exprInputs = expr->inputs(); + auto numArgs = exprInputs.size(); + argumentTypes.reserve(numArgs); + for (auto i = 0; i < numArgs; i++) { + argumentTypes.emplace_back(getTypeSignature(exprInputs[i]->type())); + } + signature.argumentTypes = argumentTypes; + signature.variableArity = false; + + protocol::BuiltInFunctionHandle builtInFunctionHandle; + builtInFunctionHandle._type = "$static"; + builtInFunctionHandle.signature = signature; + res["functionHandle"] = builtInFunctionHandle; + res["returnType"] = getTypeSignature(expr->type()); + res["arguments"] = json::array(); + for (const auto& exprInput : exprInputs) { + res["arguments"].push_back(veloxExprToRowExpression(exprInput, input)); + } + + return res; +} + +json RowExpressionConverter::veloxExprToRowExpression( + const exec::ExprPtr& expr, + const json& input) { + if (expr->type()->isRow()) { + // Velox constant expressions of ROW type map to special form expression + // row_constructor in Presto. + return getRowConstructorSpecialForm(expr, input); + } else if (expr->isConstant()) { + if (expr->inputs().empty()) { + return toConstantRowExpression(expr); + } else { + // Inputs to constant expressions are constant, eg: divide(1, 2). + return input; + } + } else if ( + auto field = + std::dynamic_pointer_cast(expr)) { + return toVariableReferenceExpression(field); + } else if (expr->isSpecialForm() || expr->vectorFunction()) { + // Check if special form expression or call expression. + auto exprName = expr->name(); + boost::algorithm::to_lower(exprName); + if (isPrestoSpecialForm(exprName)) { + return getSpecialForm(expr, input); + } else { + return toCallRowExpression(expr, input); + } + } + + VELOX_NYI( + "Conversion of Velox Expr {} to Presto RowExpression is not supported", + expr->toString()); +} + +RowExpressionPtr RowExpressionEvaluator::optimizeAndSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr) { + auto left = specialFormExpr->arguments[0]; + auto right = specialFormExpr->arguments[1]; + auto leftExpr = compileExpression(left); + bool isLeftNull; + + if (auto constantExpr = + std::dynamic_pointer_cast(leftExpr)) { + isLeftNull = constantExpr->value()->isNullAt(0); + if (!isLeftNull) { + if (auto constVector = + constantExpr->value()->as>()) { + if (!constVector->valueAt(0)) { + return rowExpressionConverter_.getConstantRowExpression(constantExpr); + } else { + return right; + } + } + } + } + + auto rightExpr = compileExpression(right); + if (auto constantExpr = + std::dynamic_pointer_cast(rightExpr)) { + if (isLeftNull && constantExpr->value()->isNullAt(0)) { + return rowExpressionConverter_.getConstantRowExpression(constantExpr); + } + if (auto constVector = constantExpr->value()->as>()) { + if (constVector->valueAt(0)) { + return left; + } + return right; + } + } + + return specialFormExpr; +} + +RowExpressionPtr RowExpressionEvaluator::optimizeIfSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr) { + auto condition = specialFormExpr->arguments[0]; + auto expr = compileExpression(condition); + + if (auto constantExpr = + std::dynamic_pointer_cast(expr)) { + if (auto constVector = constantExpr->value()->as>()) { + if (constVector->valueAt(0)) { + return specialFormExpr->arguments[1]; + } + return specialFormExpr->arguments[2]; + } + } + + return specialFormExpr; +} + +RowExpressionPtr RowExpressionEvaluator::optimizeIsNullSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr) { + auto expr = compileExpression(specialFormExpr); + if (auto constantExpr = + std::dynamic_pointer_cast(expr)) { + if (constantExpr->value()->isNullAt(0)) { + return rowExpressionConverter_.getConstantRowExpression(constantExpr); + } + } + + return specialFormExpr; +} + +RowExpressionPtr RowExpressionEvaluator::optimizeOrSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr) { + auto left = specialFormExpr->arguments[0]; + auto right = specialFormExpr->arguments[1]; + auto leftExpr = compileExpression(left); + bool isLeftNull; + + if (auto constantExpr = + std::dynamic_pointer_cast(leftExpr)) { + isLeftNull = constantExpr->value()->isNullAt(0); + if (!isLeftNull) { + if (auto constVector = + constantExpr->value()->as>()) { + if (constVector->valueAt(0)) { + return rowExpressionConverter_.getConstantRowExpression(constantExpr); + } + return right; + } + } + } + + auto rightExpr = compileExpression(right); + if (auto constantExpr = + std::dynamic_pointer_cast(rightExpr)) { + if (isLeftNull && constantExpr->value()->isNullAt(0)) { + return rowExpressionConverter_.getConstantRowExpression(constantExpr); + } + if (auto constVector = constantExpr->value()->as>()) { + if (!constVector->valueAt(0)) { + return left; + } + return right; + } + } + + return specialFormExpr; +} + +RowExpressionPtr RowExpressionEvaluator::optimizeCoalesceSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr) { + auto argsNoNulls = specialFormExpr->arguments; + argsNoNulls.erase( + std::remove_if( + argsNoNulls.begin(), + argsNoNulls.end(), + [&](const auto& arg) { + auto compiledExpr = compileExpression(arg); + if (auto constantExpr = + std::dynamic_pointer_cast( + compiledExpr)) { + return constantExpr->value()->isNullAt(0); + } + return false; + }), + argsNoNulls.end()); + + if (argsNoNulls.empty()) { + return specialFormExpr->arguments[0]; + } + specialFormExpr->arguments = argsNoNulls; + return specialFormExpr; +} + +RowExpressionPtr RowExpressionEvaluator::optimizeSpecialForm( + const std::shared_ptr& specialFormExpr) { + switch (specialFormExpr->form) { + case protocol::Form::IF: + return optimizeIfSpecialForm(specialFormExpr); + case protocol::Form::NULL_IF: + VELOX_USER_FAIL("NULL_IF specialForm not supported"); + break; + case protocol::Form::IS_NULL: + return optimizeIsNullSpecialForm(specialFormExpr); + case protocol::Form::AND: + return optimizeAndSpecialForm(specialFormExpr); + case protocol::Form::OR: + return optimizeOrSpecialForm(specialFormExpr); + case protocol::Form::COALESCE: + return optimizeCoalesceSpecialForm(specialFormExpr); + case protocol::Form::IN: + case protocol::Form::DEREFERENCE: + case protocol::Form::SWITCH: + case protocol::Form::WHEN: + case protocol::Form::ROW_CONSTRUCTOR: + case protocol::Form::BIND: + default: + break; + } + + return specialFormExpr; +} + +exec::ExprPtr RowExpressionEvaluator::compileExpression( + const std::shared_ptr& inputRowExpr) { + auto typedExpr = veloxExprConverter_.toVeloxExpr(inputRowExpr); + exec::ExprSet exprSet{{typedExpr}, execCtx_.get()}; + auto compiledExprs = + exec::compileExpressions({typedExpr}, execCtx_.get(), &exprSet, true); + return compiledExprs[0]; +} + +json::array_t RowExpressionEvaluator::evaluateExpressions( + const json::array_t& input, + const std::string& currentUser) { + const auto numExpr = input.size(); + json::array_t output = json::array(); + for (auto i = 0; i < numExpr; i++) { + // TODO: current_user to be evaluated in the native plugin and will not be + // sent to the sidecar. + if (input[i].contains("displayName") && + input[i].at("displayName") == "$current_user") { + output.emplace_back(rowExpressionConverter_.getCurrentUser(currentUser)); + continue; + } + + std::shared_ptr inputRowExpr = input[i]; + if (const auto special = + std::dynamic_pointer_cast( + inputRowExpr)) { + inputRowExpr = optimizeSpecialForm(special); + } + const auto compiledExpr = compileExpression(inputRowExpr); + json resultJson = rowExpressionConverter_.veloxExprToRowExpression( + compiledExpr, input[i]); + output.push_back(resultJson); + } + + return output; +} + +void RowExpressionEvaluator::evaluate( + proxygen::HTTPMessage* message, + const std::vector>& body, + proxygen::ResponseHandler* downstream) { + try { + auto timezone = + message->getHeaders().getSingleOrEmpty("X-Presto-Time-Zone"); + auto currentUser = message->getHeaders().getSingleOrEmpty("X-Presto-User"); + std::unordered_map config( + {{core::QueryConfig::kSessionTimezone, timezone}, + {core::QueryConfig::kAdjustTimestampToTimezone, "true"}}); + auto queryCtx = + core::QueryCtx::create(nullptr, core::QueryConfig{std::move(config)}); + execCtx_ = + std::make_unique(pool_.get(), queryCtx.get()); + + json::array_t inputList = getInputExpressions(body); + json output = evaluateExpressions(inputList, currentUser); + proxygen::ResponseBuilder(downstream) + .status(http::kHttpOk, "OK") + .header( + proxygen::HTTP_HEADER_CONTENT_TYPE, http::kMimeTypeApplicationJson) + .body(output.dump()) + .sendWithEOM(); + } catch (const velox::VeloxUserError& e) { + VLOG(1) << "VeloxUserError during expression evaluation: " << e.what(); + http::sendErrorResponse(downstream, e.what()); + } catch (const velox::VeloxException& e) { + VLOG(1) << "VeloxException during expression evaluation: " << e.what(); + http::sendErrorResponse(downstream, e.what()); + } catch (const std::exception& e) { + VLOG(1) << "std::exception during expression evaluation: " << e.what(); + http::sendErrorResponse(downstream, e.what()); + } +} + +} // namespace facebook::presto::expression diff --git a/presto-native-execution/presto_cpp/main/expression/RowExpressionEvaluator.h b/presto-native-execution/presto_cpp/main/expression/RowExpressionEvaluator.h new file mode 100644 index 000000000000..07375387ee43 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/expression/RowExpressionEvaluator.h @@ -0,0 +1,126 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "presto_cpp/external/json/nlohmann/json.hpp" +#include "presto_cpp/main/http/HttpServer.h" +#include "presto_cpp/main/types/PrestoToVeloxExpr.h" +#include "velox/core/QueryCtx.h" +#include "velox/expression/ConstantExpr.h" +#include "velox/expression/Expr.h" +#include "velox/serializers/PrestoSerializer.h" + +namespace facebook::presto::expression { + +using RowExpressionPtr = std::shared_ptr; +using SpecialFormExpressionPtr = + std::shared_ptr; + +// Helper class to convert Velox Expr of different types to the respective kind +// of Presto RowExpression. +class RowExpressionConverter { + public: + explicit RowExpressionConverter( + const std::shared_ptr& pool) + : pool_(pool), veloxToPrestoOperatorMap_(veloxToPrestoOperatorMap()) {} + + std::shared_ptr getConstantRowExpression( + const std::shared_ptr& constantExpr); + + std::shared_ptr getCurrentUser( + const std::string& currentUser); + + json veloxExprToRowExpression( + const velox::exec::ExprPtr& expr, + const json& inputRowExpr); + + protected: + std::string getValueBlock(const velox::VectorPtr& vector); + + json getRowConstructorSpecialForm( + const velox::exec::ExprPtr& expr, + const json& inputRowExpr); + + std::pair getSwitchSpecialFormArgs( + const velox::exec::ExprPtr& expr, + const json& input); + + json getSpecialForm( + const velox::exec::ExprPtr& expr, + const json& inputRowExpr); + + json toConstantRowExpression(const velox::exec::ExprPtr& expr); + + json toCallRowExpression(const velox::exec::ExprPtr& expr, const json& input); + + const std::shared_ptr pool_; + const std::unordered_map veloxToPrestoOperatorMap_; + const std::unique_ptr serde_ = + std::make_unique(); +}; + +class RowExpressionEvaluator { + public: + explicit RowExpressionEvaluator() + : pool_(velox::memory::MemoryManager::getInstance()->addLeafPool( + "RowExpressionEvaluator")), + veloxExprConverter_(pool_.get(), &typeParser_), + rowExpressionConverter_(RowExpressionConverter(pool_)) {} + + /// Evaluate expressions sent along endpoint '/v1/expressions'. + void evaluate( + proxygen::HTTPMessage* message, + const std::vector>& body, + proxygen::ResponseHandler* downstream); + + protected: + RowExpressionPtr optimizeAndSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr); + + RowExpressionPtr optimizeIfSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr); + + RowExpressionPtr optimizeIsNullSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr); + + RowExpressionPtr optimizeOrSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr); + + RowExpressionPtr optimizeCoalesceSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr); + + /// Optimizes special form expressions. Optimization rules borrowed from + /// Presto function visitSpecialForm() in RowExpressionInterpreter.java. + RowExpressionPtr optimizeSpecialForm( + const SpecialFormExpressionPtr& specialFormExpr); + + /// Converts protocol::RowExpression into a velox expression with constant + /// folding enabled during velox expression compilation. + velox::exec::ExprPtr compileExpression(const RowExpressionPtr& inputRowExpr); + + /// Optimizes and constant folds each expression from input json array and + /// returns an array of expressions that are optimized and constant folded. + /// Uses RowExpressionConverter to convert Velox expression(s) to their + /// corresponding Presto RowExpression(s). + json::array_t evaluateExpressions( + const json::array_t& input, + const std::string& currentUser); + + const std::shared_ptr pool_; + std::unique_ptr execCtx_; + TypeParser typeParser_; + VeloxExprConverter veloxExprConverter_; + RowExpressionConverter rowExpressionConverter_; +}; +} // namespace facebook::presto::expression diff --git a/presto-native-execution/presto_cpp/main/expression/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/expression/tests/CMakeLists.txt new file mode 100644 index 000000000000..0e95be687a2c --- /dev/null +++ b/presto-native-execution/presto_cpp/main/expression/tests/CMakeLists.txt @@ -0,0 +1,26 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +add_executable(presto_expr_eval_test RowExpressionEvaluatorTest.cpp) + +add_test(presto_expr_eval_test presto_expr_eval_test) + +target_link_libraries( + presto_expr_eval_test + presto_expr_eval + presto_http + velox_exec_test_lib + velox_presto_serializer + GTest::gtest + GTest::gtest_main + ${PROXYGEN_LIBRARIES}) diff --git a/presto-native-execution/presto_cpp/main/expression/tests/RowExpressionEvaluatorTest.cpp b/presto-native-execution/presto_cpp/main/expression/tests/RowExpressionEvaluatorTest.cpp new file mode 100644 index 000000000000..d457497f32f3 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/expression/tests/RowExpressionEvaluatorTest.cpp @@ -0,0 +1,187 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "presto_cpp/main/expression/RowExpressionEvaluator.h" +#include +#include +#include +#include +#include "presto_cpp/main/common/tests/test_json.h" +#include "presto_cpp/main/http/tests/HttpTestBase.h" +#include "velox/exec/OutputBufferManager.h" +#include "velox/expression/RegisterSpecialForm.h" +#include "velox/functions/prestosql/registration/RegistrationFunctions.h" +#include "velox/parse/TypeResolver.h" +#include "velox/vector/VectorStream.h" +#include "velox/vector/tests/utils/VectorTestBase.h" + +using namespace facebook::presto; +using namespace facebook::velox; + +namespace { +std::string getDataPath(const std::string& fileName) { + std::string currentPath = fs::current_path().c_str(); + + if (boost::algorithm::ends_with(currentPath, "fbcode")) { + return currentPath + + "/github/presto-trunk/presto-native-execution/presto_cpp/main/expression/tests/data/" + + fileName; + } + + if (boost::algorithm::ends_with(currentPath, "fbsource")) { + return currentPath + "/third-party/presto_cpp/main/expression/tests/data/" + + fileName; + } + + // CLion runs the tests from cmake-build-release/ or cmake-build-debug/ + // directory. Hard-coded json files are not copied there and test fails with + // file not found. Fixing the path so that we can trigger these tests from + // CLion. + boost::algorithm::replace_all(currentPath, "cmake-build-release/", ""); + boost::algorithm::replace_all(currentPath, "cmake-build-debug/", ""); + + return currentPath + "/data/" + fileName; +} +} // namespace + +// RowExpressionEvaluatorTest only tests basic expression evaluation via the +// 'v1/expressions' endpoint. End to end tests for different expression types +// can be found in TestDelegatingExpressionOptimizer.java, in the module +// presto-native-sidecar-plugin. +class RowExpressionEvaluatorTest + : public ::testing::Test, + public facebook::velox::test::VectorTestBase { + protected: + static void SetUpTestCase() { + memory::MemoryManager::testingSetInstance({}); + } + + void SetUp() override { + parse::registerTypeResolver(); + functions::prestosql::registerAllScalarFunctions("presto.default."); + exec::registerFunctionCallToSpecialForms(); + + auto httpServer = std::make_unique( + httpSrvIOExecutor_, + std::make_unique( + folly::SocketAddress("127.0.0.1", 0))); + driverExecutor_ = std::make_unique(4); + rowExpressionEvaluator_ = + std::make_unique(); + httpServer->registerPost( + "/v1/expressions", + [&](proxygen::HTTPMessage* message, + const std::vector>& body, + proxygen::ResponseHandler* downstream) { + return rowExpressionEvaluator_->evaluate(message, body, downstream); + }); + httpServerWrapper_ = + std::make_unique(std::move(httpServer)); + auto address = httpServerWrapper_->start().get(); + client_ = clientFactory_.newClient( + address, + std::chrono::milliseconds(100'000), + std::chrono::milliseconds(0), + false, + pool_); + } + + void TearDown() override { + if (httpServerWrapper_) { + httpServerWrapper_->stop(); + } + } + + static std::string getHttpBody( + const std::unique_ptr& response) { + std::ostringstream oss; + auto iobufs = response->consumeBody(); + for (auto& body : iobufs) { + oss << std::string((const char*)body->data(), body->length()); + } + return oss.str(); + } + + void validateHttpResponse( + const std::string& inputStr, + const std::string& expectedStr) { + http::RequestBuilder() + .method(proxygen::HTTPMethod::POST) + .url("/v1/expressions") + .send(client_.get(), inputStr) + .via(driverExecutor_.get()) + .thenValue([expectedStr](std::unique_ptr response) { + VELOX_USER_CHECK_EQ( + response->headers()->getStatusCode(), http::kHttpOk); + if (response->hasError()) { + VELOX_USER_FAIL( + "Expression evaluation failed: {}", response->error()); + } + + auto resStr = getHttpBody(response); + auto resJson = json::parse(resStr); + ASSERT_TRUE(resJson.is_array()); + auto expectedJson = json::parse(expectedStr); + ASSERT_TRUE(expectedJson.is_array()); + EXPECT_EQ(expectedJson.size(), resJson.size()); + auto size = resJson.size(); + for (auto i = 0; i < size; i++) { + EXPECT_EQ(resJson[i], expectedJson[i]); + } + }) + .thenError( + folly::tag_t{}, [&](const std::exception& e) { + VLOG(1) << "Expression evaluation failed: " << e.what(); + }); + } + + void testFile(const std::string& prefix) { + std::string input = slurp(getDataPath(fmt::format("{}Input.json", prefix))); + auto inputExpressions = json::parse(input); + std::string output = + slurp(getDataPath(fmt::format("{}Expected.json", prefix))); + auto expectedExpressions = json::parse(output); + + validateHttpResponse(inputExpressions.dump(), expectedExpressions.dump()); + } + + std::unique_ptr rowExpressionEvaluator_; + std::unique_ptr httpServerWrapper_; + HttpClientFactory clientFactory_; + std::shared_ptr client_; + std::shared_ptr httpSrvIOExecutor_{ + std::make_shared(8)}; + std::unique_ptr driverExecutor_; + std::shared_ptr pool_{ + memory::memoryManager()->addLeafPool("RowExpressionEvaluatorTest")}; +}; + +TEST_F(RowExpressionEvaluatorTest, simple) { + // File SimpleExpressions{Input|Expected}.json contain the input and expected + // JSON representing the RowExpressions resulting from the following queries: + // select 1 + 2; + // select abs(-11) + ceil(cast(3.4 as double)) + floor(cast(5.6 as double)); + // select 2 between 1 and 3; + // Simple expression evaluation with constant folding is verified here. + testFile("SimpleExpressions"); +} + +TEST_F(RowExpressionEvaluatorTest, specialFormRewrites) { + // File SpecialExpressions{Input|Expected}.json contain the input and expected + // JSON representing the RowExpressions resulting from the following queries: + // select if(1 < 2, 2, 3); + // select (1 < 2) and (2 < 3); + // select (1 < 2) or (2 < 3); + // Special form expression rewrites are verified here. + testFile("SpecialForm"); +} diff --git a/presto-native-execution/presto_cpp/main/expression/tests/data/SimpleExpressionsExpected.json b/presto-native-execution/presto_cpp/main/expression/tests/data/SimpleExpressionsExpected.json new file mode 100644 index 000000000000..e9b014fecd3b --- /dev/null +++ b/presto-native-execution/presto_cpp/main/expression/tests/data/SimpleExpressionsExpected.json @@ -0,0 +1,17 @@ +[ + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAwAAAA==" + }, + { + "@type": "constant", + "type": "double", + "valueBlock": "CgAAAExPTkdfQVJSQVkBAAAAAAAAAAAAADRA" + }, + { + "@type": "constant", + "type": "boolean", + "valueBlock": "CgAAAEJZVEVfQVJSQVkBAAAAAAE=" + } +] diff --git a/presto-native-execution/presto_cpp/main/expression/tests/data/SimpleExpressionsInput.json b/presto-native-execution/presto_cpp/main/expression/tests/data/SimpleExpressionsInput.json new file mode 100644 index 000000000000..dd2a0497703e --- /dev/null +++ b/presto-native-execution/presto_cpp/main/expression/tests/data/SimpleExpressionsInput.json @@ -0,0 +1,278 @@ +[ + { + "@type": "call", + "arguments": [ + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAQAAAA==" + }, + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAgAAAA==" + } + ], + "displayName": "ADD", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "integer", + "integer" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$add", + "returnType": "integer", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "integer" + }, + { + "@type": "call", + "arguments": [ + { + "@type": "call", + "arguments": [ + { + "@type": "call", + "arguments": [ + { + "@type": "call", + "arguments": [ + { + "@type": "call", + "arguments": [ + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAACwAAAA==" + } + ], + "displayName": "NEGATION", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "integer" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$negation", + "returnType": "integer", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "integer" + } + ], + "displayName": "abs", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "integer" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.abs", + "returnType": "integer", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "integer" + } + ], + "displayName": "CAST", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "integer" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$cast", + "returnType": "double", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "double" + }, + { + "@type": "call", + "arguments": [ + { + "@type": "call", + "arguments": [ + { + "@type": "constant", + "type": "decimal(2,1)", + "valueBlock": "CgAAAExPTkdfQVJSQVkBAAAAACIAAAAAAAAA" + } + ], + "displayName": "CAST", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "decimal(2,1)" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$cast", + "returnType": "double", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "double" + } + ], + "displayName": "ceil", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "double" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.ceil", + "returnType": "double", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "double" + } + ], + "displayName": "ADD", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "double", + "double" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$add", + "returnType": "double", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "double" + }, + { + "@type": "call", + "arguments": [ + { + "@type": "call", + "arguments": [ + { + "@type": "constant", + "type": "decimal(2,1)", + "valueBlock": "CgAAAExPTkdfQVJSQVkBAAAAADgAAAAAAAAA" + } + ], + "displayName": "CAST", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "decimal(2,1)" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$cast", + "returnType": "double", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "double" + } + ], + "displayName": "floor", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "double" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.floor", + "returnType": "double", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "double" + } + ], + "displayName": "ADD", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "double", + "double" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$add", + "returnType": "double", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "double" + }, + { + "@type": "call", + "arguments": [ + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAgAAAA==" + }, + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAQAAAA==" + }, + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAwAAAA==" + } + ], + "displayName": "BETWEEN", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "integer", + "integer", + "integer" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$between", + "returnType": "boolean", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "boolean" + } +] diff --git a/presto-native-execution/presto_cpp/main/expression/tests/data/SpecialFormExpected.json b/presto-native-execution/presto_cpp/main/expression/tests/data/SpecialFormExpected.json new file mode 100644 index 000000000000..2ce6acb1ab46 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/expression/tests/data/SpecialFormExpected.json @@ -0,0 +1,17 @@ +[ + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAgAAAA==" + }, + { + "@type": "constant", + "type": "boolean", + "valueBlock": "CgAAAEJZVEVfQVJSQVkBAAAAAAE=" + }, + { + "@type": "constant", + "type": "boolean", + "valueBlock": "CgAAAEJZVEVfQVJSQVkBAAAAAAE=" + } +] diff --git a/presto-native-execution/presto_cpp/main/expression/tests/data/SpecialFormInput.json b/presto-native-execution/presto_cpp/main/expression/tests/data/SpecialFormInput.json new file mode 100644 index 000000000000..77802722541b --- /dev/null +++ b/presto-native-execution/presto_cpp/main/expression/tests/data/SpecialFormInput.json @@ -0,0 +1,193 @@ +[ + { + "@type": "special", + "arguments": [ + { + "@type": "call", + "arguments": [ + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAQAAAA==" + }, + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAgAAAA==" + } + ], + "displayName": "LESS_THAN", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "integer", + "integer" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$less_than", + "returnType": "boolean", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "boolean" + }, + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAgAAAA==" + }, + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAwAAAA==" + } + ], + "form": "IF", + "returnType": "integer" + }, + { + "@type": "special", + "arguments": [ + { + "@type": "call", + "arguments": [ + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAQAAAA==" + }, + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAgAAAA==" + } + ], + "displayName": "LESS_THAN", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "integer", + "integer" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$less_than", + "returnType": "boolean", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "boolean" + }, + { + "@type": "call", + "arguments": [ + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAgAAAA==" + }, + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAwAAAA==" + } + ], + "displayName": "LESS_THAN", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "integer", + "integer" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$less_than", + "returnType": "boolean", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "boolean" + } + ], + "form": "AND", + "returnType": "boolean" + }, + { + "@type": "special", + "arguments": [ + { + "@type": "call", + "arguments": [ + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAQAAAA==" + }, + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAgAAAA==" + } + ], + "displayName": "LESS_THAN", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "integer", + "integer" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$less_than", + "returnType": "boolean", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "boolean" + }, + { + "@type": "call", + "arguments": [ + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAgAAAA==" + }, + { + "@type": "constant", + "type": "integer", + "valueBlock": "CQAAAElOVF9BUlJBWQEAAAAAAwAAAA==" + } + ], + "displayName": "LESS_THAN", + "functionHandle": { + "@type": "$static", + "signature": { + "argumentTypes": [ + "integer", + "integer" + ], + "kind": "SCALAR", + "longVariableConstraints": [], + "name": "presto.default.$operator$less_than", + "returnType": "boolean", + "typeVariableConstraints": [], + "variableArity": false + } + }, + "returnType": "boolean" + } + ], + "form": "OR", + "returnType": "boolean" + } +] diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp index 2c2b2a3c5ea0..791e215fb0f3 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp @@ -33,32 +33,10 @@ std::string toJsonString(const T& value) { } std::string mapScalarFunction(const std::string& name) { - static const std::unordered_map kFunctionNames = { - // Operator overrides: com.facebook.presto.common.function.OperatorType - {"presto.default.$operator$add", "presto.default.plus"}, - {"presto.default.$operator$between", "presto.default.between"}, - {"presto.default.$operator$divide", "presto.default.divide"}, - {"presto.default.$operator$equal", "presto.default.eq"}, - {"presto.default.$operator$greater_than", "presto.default.gt"}, - {"presto.default.$operator$greater_than_or_equal", "presto.default.gte"}, - {"presto.default.$operator$is_distinct_from", - "presto.default.distinct_from"}, - {"presto.default.$operator$less_than", "presto.default.lt"}, - {"presto.default.$operator$less_than_or_equal", "presto.default.lte"}, - {"presto.default.$operator$modulus", "presto.default.mod"}, - {"presto.default.$operator$multiply", "presto.default.multiply"}, - {"presto.default.$operator$negation", "presto.default.negate"}, - {"presto.default.$operator$not_equal", "presto.default.neq"}, - {"presto.default.$operator$subtract", "presto.default.minus"}, - {"presto.default.$operator$subscript", "presto.default.subscript"}, - // Special form function overrides. - {"presto.default.in", "in"}, - }; - std::string lowerCaseName = boost::to_lower_copy(name); - auto it = kFunctionNames.find(lowerCaseName); - if (it != kFunctionNames.end()) { + auto it = kPrestoOperatorMap.find(lowerCaseName); + if (it != kPrestoOperatorMap.end()) { return it->second; } @@ -102,6 +80,15 @@ std::string getFunctionName(const protocol::SqlFunctionId& functionId) { } // namespace +const std::unordered_map veloxToPrestoOperatorMap() { + std::unordered_map veloxToPrestoOperatorMap; + for (const auto& entry : kPrestoOperatorMap) { + veloxToPrestoOperatorMap[entry.second] = entry.first; + } + veloxToPrestoOperatorMap.insert({"cast", "presto.default.$operator$cast"}); + return veloxToPrestoOperatorMap; +} + velox::variant VeloxExprConverter::getConstantValue( const velox::TypePtr& type, const protocol::Block& block) const { diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h index f63a84ec35ad..06c326dfddc1 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h @@ -20,6 +20,30 @@ namespace facebook::presto { +static const std::unordered_map kPrestoOperatorMap = { + // Operator overrides: com.facebook.presto.common.function.OperatorType + {"presto.default.$operator$add", "presto.default.plus"}, + {"presto.default.$operator$between", "presto.default.between"}, + {"presto.default.$operator$divide", "presto.default.divide"}, + {"presto.default.$operator$equal", "presto.default.eq"}, + {"presto.default.$operator$greater_than", "presto.default.gt"}, + {"presto.default.$operator$greater_than_or_equal", "presto.default.gte"}, + {"presto.default.$operator$is_distinct_from", + "presto.default.distinct_from"}, + {"presto.default.$operator$less_than", "presto.default.lt"}, + {"presto.default.$operator$less_than_or_equal", "presto.default.lte"}, + {"presto.default.$operator$modulus", "presto.default.mod"}, + {"presto.default.$operator$multiply", "presto.default.multiply"}, + {"presto.default.$operator$negation", "presto.default.negate"}, + {"presto.default.$operator$not_equal", "presto.default.neq"}, + {"presto.default.$operator$subtract", "presto.default.minus"}, + {"presto.default.$operator$subscript", "presto.default.subscript"}, + // Special form function overrides. + {"presto.default.in", "in"}, +}; + +const std::unordered_map veloxToPrestoOperatorMap(); + class VeloxExprConverter { public: VeloxExprConverter(velox::memory::MemoryPool* pool, TypeParser* typeParser) diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java index 38e48f12ff96..02803b0d358e 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java @@ -610,7 +610,7 @@ public static Process getNativeSidecarProcess(String prestoServerPath, URI disco Path catalogDirectoryPath = tempDirectoryPath.resolve("catalog"); Files.createDirectory(catalogDirectoryPath); - return new ProcessBuilder(prestoServerPath) + return new ProcessBuilder(prestoServerPath, "--logtostderr=1", "--v=1") .directory(tempDirectoryPath.toFile()) .redirectErrorStream(true) .redirectOutput(ProcessBuilder.Redirect.to(tempDirectoryPath.resolve("sidecar.out").toFile()))