From 5e844fa5f53ccaddd4c501d79254016445aacea4 Mon Sep 17 00:00:00 2001 From: Tim Meehan Date: Tue, 30 Jul 2024 10:11:46 -0400 Subject: [PATCH 1/3] Add SPI to customize expression optimization --- .../presto/spi/CoordinatorPlugin.java | 6 +++ .../planner/ExpressionOptimizerContext.java | 49 +++++++++++++++++++ .../planner/ExpressionOptimizerFactory.java | 25 ++++++++++ 3 files changed, 80 insertions(+) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerFactory.java diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java b/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java index d161738b67bc9..ca8ea46d58aa6 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/CoordinatorPlugin.java @@ -16,6 +16,7 @@ import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; import com.facebook.presto.spi.plan.PlanCheckerProviderFactory; import com.facebook.presto.spi.session.WorkerSessionPropertyProviderFactory; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; import static java.util.Collections.emptyList; @@ -40,4 +41,9 @@ default Iterable getPlanCheckerProviderFactories() { return emptyList(); } + + default Iterable getExpressionOptimizerFactories() + { + return emptyList(); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java new file mode 100644 index 0000000000000..c9a6f84aaaa1d --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerContext.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.sql.planner; + +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; + +import static java.util.Objects.requireNonNull; + +public class ExpressionOptimizerContext +{ + private final NodeManager nodeManager; + private final FunctionMetadataManager functionMetadataManager; + private final StandardFunctionResolution functionResolution; + + public ExpressionOptimizerContext(NodeManager nodeManager, FunctionMetadataManager functionMetadataManager, StandardFunctionResolution functionResolution) + { + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + } + + public NodeManager getNodeManager() + { + return nodeManager; + } + + public FunctionMetadataManager getFunctionMetadataManager() + { + return functionMetadataManager; + } + + public StandardFunctionResolution getFunctionResolution() + { + return functionResolution; + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerFactory.java b/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerFactory.java new file mode 100644 index 0000000000000..ad645e0c04a8d --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/sql/planner/ExpressionOptimizerFactory.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.sql.planner; + +import com.facebook.presto.spi.relation.ExpressionOptimizer; + +import java.util.Map; + +public interface ExpressionOptimizerFactory +{ + ExpressionOptimizer createOptimizer(Map config, ExpressionOptimizerContext context); + + String getName(); +} From 9ede3efbe5b568c960f470fd08d29179681563f9 Mon Sep 17 00:00:00 2001 From: Tim Meehan Date: Tue, 30 Jul 2024 10:37:54 -0400 Subject: [PATCH 2/3] Add ExpressionOptimizerManager --- .../presto/SystemSessionProperties.java | 13 +- .../facebook/presto/server/PluginManager.java | 12 +- .../facebook/presto/server/PrestoServer.java | 3 + .../presto/server/ServerMainModule.java | 4 + .../server/testing/TestingPrestoServer.java | 9 + .../ExpressionOptimizerManager.java | 152 +++++++++++++++ .../presto/sql/planner/PlanOptimizers.java | 16 +- .../rule/SimplifyRowExpressions.java | 26 +-- .../CteProjectionAndPredicatePushDown.java | 12 +- .../relational/RowExpressionOptimizer.java | 17 +- .../presto/testing/LocalQueryRunner.java | 20 +- .../facebook/presto/testing/QueryRunner.java | 3 + .../presto/sql/TestExpressionInterpreter.java | 2 +- .../TestExpressionOptimizerManager.java | 184 ++++++++++++++++++ .../sql/planner/TestLogicalPlanner.java | 34 +++- .../planner/assertions/OptimizerAssert.java | 9 +- .../iterative/rule/TestRemoveMapCastRule.java | 4 +- ...teConstantArrayContainsToInExpression.java | 8 +- .../rule/TestSimplifyRowExpressions.java | 8 +- .../iterative/rule/test/BaseRuleTest.java | 6 + .../iterative/rule/test/RuleTester.java | 8 + ...TestCteProjectionAndPredicatePushdown.java | 3 +- .../nativeworker/ContainerQueryRunner.java | 7 + .../presto/spark/PrestoSparkModule.java | 6 + .../presto/spark/PrestoSparkQueryRunner.java | 7 + .../tests/AbstractTestQueryFramework.java | 10 +- .../presto/tests/DistributedQueryRunner.java | 8 + .../presto/tests/StandaloneQueryRunner.java | 7 + .../thrift/integration/ThriftQueryRunner.java | 7 + 29 files changed, 561 insertions(+), 44 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/expressions/TestExpressionOptimizerManager.java diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 7f23e8172e800..daa61ead85128 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -326,6 +326,7 @@ public final class SystemSessionProperties public static final String INLINE_PROJECTIONS_ON_VALUES = "inline_projections_on_values"; public static final String INCLUDE_VALUES_NODE_IN_CONNECTOR_OPTIMIZER = "include_values_node_in_connector_optimizer"; public static final String SINGLE_NODE_EXECUTION_ENABLED = "single_node_execution_enabled"; + public static final String EXPRESSION_OPTIMIZER_INSTANCE = "expression_optimizer_instance"; // TODO: Native execution related session properties that are temporarily put here. They will be relocated in the future. public static final String NATIVE_AGGREGATION_SPILL_ALL = "native_aggregation_spill_all"; @@ -1846,7 +1847,12 @@ public SystemSessionProperties( booleanProperty(NATIVE_EXECUTION_SCALE_WRITER_THREADS_ENABLED, "Enable automatic scaling of writer threads", featuresConfig.isNativeExecutionScaleWritersThreadsEnabled(), - !featuresConfig.isNativeExecutionEnabled())); + !featuresConfig.isNativeExecutionEnabled()), + stringProperty( + EXPRESSION_OPTIMIZER_INSTANCE, + "Configure which expression optimizer instance to use", + null, + false)); } public static boolean isSpoolingOutputBufferEnabled(Session session) @@ -3142,4 +3148,9 @@ public static boolean isNativeExecutionScaleWritersThreadsEnabled(Session sessio { return session.getSystemProperty(NATIVE_EXECUTION_SCALE_WRITER_THREADS_ENABLED, Boolean.class); } + + public static String getExpressionOptimizerInstance(Session session) + { + return session.getSystemProperty(EXPRESSION_OPTIMIZER_INSTANCE, String.class); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java index f1cef40a1a3f6..5a3a06b3b97e0 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java @@ -43,6 +43,7 @@ import com.facebook.presto.spi.security.SystemAccessControlFactory; import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory; import com.facebook.presto.spi.session.WorkerSessionPropertyProviderFactory; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; import com.facebook.presto.spi.storage.TempStorageFactory; import com.facebook.presto.spi.tracing.TracerProvider; @@ -50,6 +51,7 @@ import com.facebook.presto.spi.ttl.NodeTtlFetcherFactory; import com.facebook.presto.sql.analyzer.AnalyzerProviderManager; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; import com.facebook.presto.storage.TempStorageManager; import com.facebook.presto.tracing.TracerProviderManager; @@ -135,6 +137,7 @@ public class PluginManager private final QueryPreparerProviderManager queryPreparerProviderManager; private final NodeStatusNotificationManager nodeStatusNotificationManager; private final PlanCheckerProviderManager planCheckerProviderManager; + private final ExpressionOptimizerManager expressionOptimizerManager; @Inject public PluginManager( @@ -157,7 +160,8 @@ public PluginManager( HistoryBasedPlanStatisticsManager historyBasedPlanStatisticsManager, TracerProviderManager tracerProviderManager, NodeStatusNotificationManager nodeStatusNotificationManager, - PlanCheckerProviderManager planCheckerProviderManager) + PlanCheckerProviderManager planCheckerProviderManager, + ExpressionOptimizerManager expressionOptimizerManager) { requireNonNull(nodeInfo, "nodeInfo is null"); requireNonNull(config, "config is null"); @@ -190,6 +194,7 @@ public PluginManager( this.queryPreparerProviderManager = requireNonNull(queryPreparerProviderManager, "queryPreparerProviderManager is null"); this.nodeStatusNotificationManager = requireNonNull(nodeStatusNotificationManager, "nodeStatusNotificationManager is null"); this.planCheckerProviderManager = requireNonNull(planCheckerProviderManager, "planCheckerProviderManager is null"); + this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionManager is null"); } public void loadPlugins() @@ -372,6 +377,11 @@ public void installCoordinatorPlugin(CoordinatorPlugin plugin) log.info("Registering plan checker provider factory %s", planCheckerProviderFactory.getName()); planCheckerProviderManager.addPlanCheckerProviderFactory(planCheckerProviderFactory); } + + for (ExpressionOptimizerFactory expressionOptimizerFactory : plugin.getExpressionOptimizerFactories()) { + log.info("Registering expression optimizer factory %s", expressionOptimizerFactory.getName()); + expressionOptimizerManager.addExpressionOptimizerFactory(expressionOptimizerFactory); + } } private URLClassLoader buildClassLoader(String plugin) diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index d8a0fe050e7ad..7864423728414 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -51,6 +51,7 @@ import com.facebook.presto.server.security.PasswordAuthenticatorManager; import com.facebook.presto.server.security.ServerSecurityModule; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; import com.facebook.presto.storage.TempStorageManager; @@ -190,6 +191,8 @@ public void run() PluginNodeManager pluginNodeManager = new PluginNodeManager(nodeManager, nodeInfo.getEnvironment()); planCheckerProviderManager.loadPlanCheckerProviders(pluginNodeManager); + injector.getInstance(ExpressionOptimizerManager.class).loadExpressionOptimizerFactories(); + startAssociatedProcesses(injector); injector.getInstance(Announcer.class).start(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index e2ab959828527..881588658003e 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -194,6 +194,7 @@ import com.facebook.presto.sql.analyzer.MetadataExtractorMBean; import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -359,6 +360,9 @@ else if (serverConfig.isCoordinator()) { binder.bind(SystemSessionProperties.class).in(Scopes.SINGLETON); binder.bind(SessionPropertyDefaults.class).in(Scopes.SINGLETON); + // expression manager + binder.bind(ExpressionOptimizerManager.class).in(Scopes.SINGLETON); + // schema properties binder.bind(SchemaPropertyManager.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java index 034763c47212c..3a7c34e1e5b2f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java @@ -71,6 +71,7 @@ import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; @@ -168,6 +169,7 @@ public class TestingPrestoServer private final TaskManager taskManager; private final GracefulShutdownHandler gracefulShutdownHandler; private final ShutdownAction shutdownAction; + private final ExpressionOptimizerManager expressionManager; private final RequestBlocker requestBlocker; private final boolean resourceManager; private final boolean catalogServer; @@ -368,6 +370,7 @@ public TestingPrestoServer( procedureTester = injector.getInstance(ProcedureTester.class); splitManager = injector.getInstance(SplitManager.class); pageSourceManager = injector.getInstance(PageSourceManager.class); + expressionManager = injector.getInstance(ExpressionOptimizerManager.class); if (coordinator) { dispatchManager = injector.getInstance(DispatchManager.class); queryManager = injector.getInstance(QueryManager.class); @@ -385,6 +388,7 @@ public TestingPrestoServer( eventListenerManager = ((TestingEventListenerManager) injector.getInstance(EventListenerManager.class)); clusterStateProvider = null; planCheckerProviderManager = injector.getInstance(PlanCheckerProviderManager.class); + expressionManager.loadExpressionOptimizerFactories(); } else if (resourceManager) { dispatchManager = null; @@ -704,6 +708,11 @@ public ShutdownAction getShutdownAction() return shutdownAction; } + public ExpressionOptimizerManager getExpressionManager() + { + return expressionManager; + } + public boolean isCoordinator() { return coordinator; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java b/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java new file mode 100644 index 0000000000000..053f2134412c4 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java @@ -0,0 +1,152 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.expressions; + +import com.facebook.presto.FullConnectorSession; +import com.facebook.presto.Session; +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.nodeManager.PluginNodeManager; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerContext; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; +import com.google.common.collect.ImmutableList; + +import javax.inject.Inject; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; + +import static com.facebook.presto.SystemSessionProperties.getExpressionOptimizerInstance; +import static com.facebook.presto.util.PropertiesUtil.loadProperties; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.io.Files.getNameWithoutExtension; +import static java.lang.Boolean.parseBoolean; +import static java.util.Objects.requireNonNull; + +public class ExpressionOptimizerManager +{ + private static final File EXPRESSION_MANAGER_CONFIGURATION_DIRECTORY = new File("etc/expression-manager/"); + private static final String EXPRESSION_MANAGER_FACTORY_NAME = "expression-manager-factory.name"; + private static final String DEFAULT_PROPERTY = "default"; + public static final String DEFAULT_OPTIMIZER_KEY = "default"; + + private final NodeManager nodeManager; + private final FunctionAndTypeManager functionAndTypeManager; + private final FunctionResolution functionResolution; + private final File configurationDirectory; + + private final Map expressionOptimizerFactories = new ConcurrentHashMap<>(); + private final Map expressionOptimizers = new ConcurrentHashMap<>(); + private final AtomicReference defaultExpressionOptimizer = new AtomicReference<>(); + + @Inject + public ExpressionOptimizerManager(PluginNodeManager nodeManager, FunctionAndTypeManager functionAndTypeManager) + { + this(nodeManager, functionAndTypeManager, EXPRESSION_MANAGER_CONFIGURATION_DIRECTORY); + } + + public ExpressionOptimizerManager(PluginNodeManager nodeManager, FunctionAndTypeManager functionAndTypeManager, File configurationDirectory) + { + requireNonNull(nodeManager, "nodeManager is null"); + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); + this.functionResolution = new FunctionResolution(functionAndTypeManager.getFunctionAndTypeResolver()); + this.configurationDirectory = requireNonNull(configurationDirectory, "configurationDirectory is null"); + expressionOptimizers.put(DEFAULT_OPTIMIZER_KEY, new RowExpressionOptimizer(functionAndTypeManager)); + defaultExpressionOptimizer.set(DEFAULT_OPTIMIZER_KEY); + } + + public void loadExpressionOptimizerFactories() + { + try { + for (File file : listFiles(configurationDirectory)) { + if (file.isFile() && file.getName().endsWith(".properties")) { + loadExpressionOptimizerFactory(file); + } + } + } + catch (IOException e) { + throw new UncheckedIOException("Failed to load expression manager configuration", e); + } + } + + private void loadExpressionOptimizerFactory(File configurationFile) + throws IOException + { + String name = getNameWithoutExtension(configurationFile.getName()); + checkArgument(!isNullOrEmpty(name), "File name is empty, full path: %s", configurationFile.getAbsolutePath()); + checkArgument(!name.equals(DEFAULT_OPTIMIZER_KEY), "Cannot name an expression optimizer instance default"); + + Map properties = new HashMap<>(loadProperties(configurationFile)); + String factoryName = properties.remove(EXPRESSION_MANAGER_FACTORY_NAME); + checkArgument(!isNullOrEmpty(factoryName), "%s does not contain %s", configurationFile, EXPRESSION_MANAGER_FACTORY_NAME); + checkArgument(expressionOptimizerFactories.containsKey(factoryName), + "ExpressionOptimizerFactory %s is not registered, registered factories: ", factoryName, expressionOptimizerFactories.keySet()); + String defaultPropertyValue = properties.remove(DEFAULT_PROPERTY); + checkArgument(defaultPropertyValue == null || "true".equalsIgnoreCase(defaultPropertyValue) || "false".equalsIgnoreCase(defaultPropertyValue), + "Only true or false is allowed for %s, found: %s", DEFAULT_PROPERTY, defaultPropertyValue); + boolean isDefault = parseBoolean(defaultPropertyValue); + + ExpressionOptimizer optimizer = expressionOptimizerFactories.get(factoryName).createOptimizer( + properties, + new ExpressionOptimizerContext(nodeManager, functionAndTypeManager, functionResolution)); + expressionOptimizers.put(name, optimizer); + if (isDefault) { + checkArgument(defaultExpressionOptimizer.get().equals(DEFAULT_OPTIMIZER_KEY), "Default ExpressionOptimizer is already set, found: %s, attempted: %s", defaultExpressionOptimizer.get()); + defaultExpressionOptimizer.set(name); + } + } + + public void addExpressionOptimizerFactory(ExpressionOptimizerFactory expressionOptimizerFactory) + { + String name = expressionOptimizerFactory.getName(); + checkArgument( + expressionOptimizerFactories.putIfAbsent(name, expressionOptimizerFactory) == null, + "ExpressionOptimizerFactory %s is already registered", name); + } + + public ExpressionOptimizer getExpressionOptimizer(ConnectorSession connectorSession) + { + checkArgument(connectorSession instanceof FullConnectorSession, "connectorSession is not an instance of FullConnectorSession"); + Session session = ((FullConnectorSession) connectorSession).getSession(); + String expressionOptimizerInstance = getExpressionOptimizerInstance(session); + if (isNullOrEmpty(expressionOptimizerInstance)) { + expressionOptimizerInstance = defaultExpressionOptimizer.get(); + } + checkArgument(expressionOptimizers.containsKey(expressionOptimizerInstance), "ExpressionOptimizer %s is not registered", expressionOptimizerInstance); + return expressionOptimizers.get(expressionOptimizerInstance); + } + + private static List listFiles(File directory) + { + if (directory.isDirectory()) { + File[] files = directory.listFiles(); + if (files != null) { + return ImmutableList.copyOf(files); + } + } + return ImmutableList.of(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index a900bb364d163..9d7ca9d80e1bb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -22,6 +22,7 @@ import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.iterative.IterativeOptimizer; import com.facebook.presto.sql.planner.iterative.Rule; @@ -220,7 +221,8 @@ public PlanOptimizers( CostComparator costComparator, TaskCountEstimator taskCountEstimator, PartitioningProviderManager partitioningProviderManager, - FeaturesConfig featuresConfig) + FeaturesConfig featuresConfig, + ExpressionOptimizerManager expressionOptimizerManager) { this(metadata, sqlParser, @@ -235,7 +237,8 @@ public PlanOptimizers( costComparator, taskCountEstimator, partitioningProviderManager, - featuresConfig); + featuresConfig, + expressionOptimizerManager); } @PostConstruct @@ -266,7 +269,8 @@ public PlanOptimizers( CostComparator costComparator, TaskCountEstimator taskCountEstimator, PartitioningProviderManager partitioningProviderManager, - FeaturesConfig featuresConfig) + FeaturesConfig featuresConfig, + ExpressionOptimizerManager expressionOptimizerManager) { this.exporter = exporter; ImmutableList.Builder builder = ImmutableList.builder(); @@ -321,7 +325,7 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.>builder() - .addAll(new SimplifyRowExpressions(metadata).rules()) + .addAll(new SimplifyRowExpressions(metadata, expressionOptimizerManager).rules()) .add(new PruneRedundantProjectionAssignments()) .build()); @@ -487,7 +491,7 @@ public PlanOptimizers( estimatedExchangesCostCalculator, ImmutableSet.>builder() .add(new InlineProjectionsOnValues(metadata.getFunctionAndTypeManager())) - .addAll(new SimplifyRowExpressions(metadata).rules()) + .addAll(new SimplifyRowExpressions(metadata, expressionOptimizerManager).rules()) .build()), new IterativeOptimizer( metadata, @@ -846,7 +850,7 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of(new PushTableWriteThroughUnion()))); // Must run before AddExchanges - builder.add(new CteProjectionAndPredicatePushDown(metadata)); // must run before PhysicalCteOptimizer + builder.add(new CteProjectionAndPredicatePushDown(metadata, expressionOptimizerManager, featuresConfig)); // must run before PhysicalCteOptimizer builder.add(new PhysicalCteOptimizer(metadata)); // Must run before AddExchanges builder.add(new StatsRecordingPlanOptimizer(optimizerStats, new AddExchanges(metadata, partitioningProviderManager, featuresConfig.isNativeExecutionEnabled()))); builder.add(new StatsRecordingPlanOptimizer(optimizerStats, new AddExchangesForSingleNodeExecution(metadata))); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java index 9705f7e836fd5..b7192926bb807 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java @@ -13,20 +13,20 @@ */ package com.facebook.presto.sql.planner.iterative.rule; +import com.facebook.presto.Session; import com.facebook.presto.common.type.BooleanType; import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.expressions.RowExpressionRewriter; import com.facebook.presto.expressions.RowExpressionTreeRewriter; import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; -import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.google.common.annotations.VisibleForTesting; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.SERIALIZABLE; @@ -39,44 +39,44 @@ public class SimplifyRowExpressions extends RowExpressionRewriteRuleSet { - public SimplifyRowExpressions(Metadata metadata) + public SimplifyRowExpressions(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager) { - super(new Rewriter(metadata)); + super(new Rewriter(metadata, expressionOptimizerManager)); } private static class Rewriter implements PlanRowExpressionRewriter { - private final RowExpressionOptimizer optimizer; + private final ExpressionOptimizerManager expressionOptimizerManager; private final LogicalExpressionRewriter logicalExpressionRewriter; - public Rewriter(Metadata metadata) + public Rewriter(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager) { requireNonNull(metadata, "metadata is null"); - this.optimizer = new RowExpressionOptimizer(metadata); + requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); + this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); this.logicalExpressionRewriter = new LogicalExpressionRewriter(metadata.getFunctionAndTypeManager()); } @Override public RowExpression rewrite(RowExpression expression, Rule.Context context) { - return rewrite(expression, context.getSession().toConnectorSession()); + return rewrite(expression, context.getSession()); } - private RowExpression rewrite(RowExpression expression, ConnectorSession session) + private RowExpression rewrite(RowExpression expression, Session session) { // Rewrite RowExpression first to reduce depth of RowExpression tree by balancing AND/OR predicates. // It doesn't matter whether we rewrite/optimize first because this will be called by IterativeOptimizer. RowExpression rewritten = RowExpressionTreeRewriter.rewriteWith(logicalExpressionRewriter, expression, true); - RowExpression optimizedRowExpression = optimizer.optimize(rewritten, SERIALIZABLE, session); - return optimizedRowExpression; + return expressionOptimizerManager.getExpressionOptimizer(session.toConnectorSession()).optimize(rewritten, SERIALIZABLE, session.toConnectorSession()); } } @VisibleForTesting - public static RowExpression rewrite(RowExpression expression, Metadata metadata, ConnectorSession session) + public static RowExpression rewrite(RowExpression expression, Metadata metadata, Session session, ExpressionOptimizerManager expressionOptimizerManager) { - return new Rewriter(metadata).rewrite(expression, session); + return new Rewriter(metadata, expressionOptimizerManager).rewrite(expression, session); } private static class LogicalExpressionRewriter diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CteProjectionAndPredicatePushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CteProjectionAndPredicatePushDown.java index d64068aeb3c27..58241d1014245 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CteProjectionAndPredicatePushDown.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/CteProjectionAndPredicatePushDown.java @@ -26,6 +26,8 @@ import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.PlannerUtils; import com.facebook.presto.sql.planner.RowExpressionVariableInliner; import com.facebook.presto.sql.planner.SimplePlanVisitor; @@ -87,10 +89,14 @@ public class CteProjectionAndPredicatePushDown implements PlanOptimizer { private final Metadata metadata; + private final ExpressionOptimizerManager expressionOptimizerManager; + private final FeaturesConfig featuresConfig; - public CteProjectionAndPredicatePushDown(Metadata metadata) + public CteProjectionAndPredicatePushDown(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager, FeaturesConfig featuresConfig) { - this.metadata = metadata; + this.metadata = requireNonNull(metadata, "metadata is null"); + this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); + this.featuresConfig = requireNonNull(featuresConfig, "featuresConfig is null"); } @Override @@ -383,7 +389,7 @@ private PlanNode addFilter(PlanNode node, List predicates) resultPredicate, predicates.get(i)); } } - resultPredicate = SimplifyRowExpressions.rewrite(resultPredicate, metadata, session.toConnectorSession()); + resultPredicate = SimplifyRowExpressions.rewrite(resultPredicate, metadata, session, expressionOptimizerManager); return new FilterNode(node.getSourceLocation(), idAllocator.getNextId(), node, resultPredicate); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java index 15dcc1d43f804..e976f567ecdf8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java @@ -13,8 +13,10 @@ */ package com.facebook.presto.sql.relational; +import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -24,23 +26,30 @@ import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.sql.planner.LiteralEncoder.toRowExpression; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; public final class RowExpressionOptimizer implements ExpressionOptimizer { - private final Metadata metadata; + private final FunctionAndTypeManager functionAndTypeManager; public RowExpressionOptimizer(Metadata metadata) { - this.metadata = requireNonNull(metadata, "metadata is null"); + this(requireNonNull(metadata, "metadata is null").getFunctionAndTypeManager()); + } + + public RowExpressionOptimizer(FunctionMetadataManager functionMetadataManager) + { + checkArgument(functionMetadataManager instanceof FunctionAndTypeManager, "Expected functionMetadataManager to be instance of FunctionAndTypeManager"); + this.functionAndTypeManager = (FunctionAndTypeManager) requireNonNull(functionMetadataManager, "functionMetadataManager is null"); } @Override public RowExpression optimize(RowExpression rowExpression, Level level, ConnectorSession session) { if (level.ordinal() <= OPTIMIZED.ordinal()) { - return toRowExpression(rowExpression.getSourceLocation(), new RowExpressionInterpreter(rowExpression, metadata.getFunctionAndTypeManager(), session, level).optimize(), rowExpression.getType()); + return toRowExpression(rowExpression.getSourceLocation(), new RowExpressionInterpreter(rowExpression, functionAndTypeManager, session, level).optimize(), rowExpression.getType()); } throw new IllegalArgumentException("Not supported optimization level: " + level); } @@ -48,7 +57,7 @@ public RowExpression optimize(RowExpression rowExpression, Level level, Connecto @Override public RowExpression optimize(RowExpression expression, Level level, ConnectorSession session, Function variableResolver) { - RowExpressionInterpreter interpreter = new RowExpressionInterpreter(expression, metadata.getFunctionAndTypeManager(), session, level); + RowExpressionInterpreter interpreter = new RowExpressionInterpreter(expression, functionAndTypeManager, session, level); return toRowExpression(expression.getSourceLocation(), interpreter.optimize(variableResolver::apply), expression.getType()); } } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 7bf878e0b4812..31dba80851dfd 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -107,6 +107,7 @@ import com.facebook.presto.metadata.SchemaPropertyManager; import com.facebook.presto.metadata.Split; import com.facebook.presto.metadata.TablePropertyManager; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.operator.Driver; import com.facebook.presto.operator.DriverContext; import com.facebook.presto.operator.DriverFactory; @@ -169,6 +170,7 @@ import com.facebook.presto.sql.analyzer.JavaFeaturesConfig; import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -355,6 +357,7 @@ public class LocalQueryRunner private static ExecutorService metadataExtractorExecutor = newCachedThreadPool(threadsNamed("query-execution-%s")); private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private ExpressionOptimizerManager expressionOptimizerManager; private List additionalOptimizer = ImmutableList.of(); @@ -452,6 +455,10 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, this.planFragmenter = new PlanFragmenter(this.metadata, this.nodePartitioningManager, new QueryManagerConfig(), featuresConfig, planCheckerProviderManager); this.joinCompiler = new JoinCompiler(metadata); this.pageIndexerFactory = new GroupByHashPageIndexerFactory(joinCompiler); + + NodeInfo nodeInfo = new NodeInfo("test"); + expressionOptimizerManager = new ExpressionOptimizerManager(new PluginNodeManager(nodeManager, nodeInfo.getEnvironment()), getFunctionAndTypeManager()); + this.statsNormalizer = new StatsNormalizer(); this.scalarStatsCalculator = new ScalarStatsCalculator(metadata); this.filterStatsCalculator = new FilterStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer); @@ -468,7 +475,6 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, this.expressionCompiler = new ExpressionCompiler(metadata, pageFunctionCompiler); this.joinFilterFunctionCompiler = new JoinFilterFunctionCompiler(metadata); - NodeInfo nodeInfo = new NodeInfo("test"); NodeVersion nodeVersion = new NodeVersion("testversion"); this.connectorManager = new ConnectorManager( metadata, @@ -531,7 +537,8 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, historyBasedPlanStatisticsManager, new TracerProviderManager(new TracingConfig()), new NodeStatusNotificationManager(), - planCheckerProviderManager); + planCheckerProviderManager, + expressionOptimizerManager); connectorManager.addConnectorFactory(globalSystemConnectorFactory); connectorManager.createConnection(GlobalSystemConnector.NAME, GlobalSystemConnector.NAME, ImmutableMap.of()); @@ -710,6 +717,12 @@ public TestingAccessControlManager getAccessControl() return accessControl; } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + return expressionOptimizerManager; + } + public ExecutorService getExecutor() { return notificationExecutor; @@ -1130,7 +1143,8 @@ public List getPlanOptimizers(boolean noExchange) new CostComparator(featuresConfig), taskCountEstimator, partitioningProviderManager, - featuresConfig).getPlanningTimeOptimizers()); + featuresConfig, + expressionOptimizerManager).getPlanningTimeOptimizers()); return planOptimizers.build(); } diff --git a/presto-main/src/main/java/com/facebook/presto/testing/QueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/QueryRunner.java index d2121ce335f86..e62447a403e94 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/QueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/QueryRunner.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.Plan; @@ -63,6 +64,8 @@ public interface QueryRunner TestingAccessControlManager getAccessControl(); + ExpressionOptimizerManager getExpressionManager(); + MaterializedResult execute(@Language("SQL") String sql); MaterializedResult execute(Session session, @Language("SQL") String sql); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java index 37295146c4296..5f751b22d10e2 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java @@ -1677,7 +1677,7 @@ private static Object optimize(Expression expression) private static Object optimize(RowExpression expression, Level level) { - return new RowExpressionInterpreter(expression, METADATA.getFunctionAndTypeManager(), TEST_SESSION.toConnectorSession(), level).optimize(variable -> { + return new RowExpressionInterpreter(expression, METADATA, TEST_SESSION.toConnectorSession(), level).optimize(variable -> { Symbol symbol = new Symbol(variable.getName()); Object value = symbolConstant(symbol); if (value == null) { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/expressions/TestExpressionOptimizerManager.java b/presto-main/src/test/java/com/facebook/presto/sql/expressions/TestExpressionOptimizerManager.java new file mode 100644 index 0000000000000..ceebb6e422df9 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/expressions/TestExpressionOptimizerManager.java @@ -0,0 +1,184 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.expressions; + +import com.facebook.presto.Session; +import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.metadata.InMemoryNodeManager; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.nodeManager.PluginNodeManager; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerContext; +import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; +import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.google.common.collect.ImmutableMap; +import io.airlift.slice.Slices; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.Properties; + +import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static java.lang.String.format; +import static java.nio.file.Files.createTempDirectory; +import static java.nio.file.Files.newOutputStream; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +@Test(singleThreaded = true) +public class TestExpressionOptimizerManager +{ + private static final MetadataManager METADATA = MetadataManager.createTestMetadataManager(); + private static final TestingRowExpressionTranslator TRANSLATOR = new TestingRowExpressionTranslator(METADATA); + private File directory; + private ExpressionOptimizerManager manager; + + @BeforeMethod + public void setUp() + throws Exception + { + directory = createTempDirectory("test-optimizers").toFile(); + directory.deleteOnExit(); + + InMemoryNodeManager nodeManager = new InMemoryNodeManager(); + PluginNodeManager pluginNodeManager = new PluginNodeManager(nodeManager); + manager = new ExpressionOptimizerManager( + pluginNodeManager, + METADATA.getFunctionAndTypeManager(), + directory); + } + + @AfterMethod + public void tearDown() + throws Exception + { + deleteRecursively(directory.toPath(), ALLOW_INSECURE); + } + + @Test + public void testBasicIntegration() + throws Exception + { + createPropertiesFile("foo.properties", ImmutableMap.of("expression-manager-factory.name", "foo")); + createPropertiesFile("bar.properties", ImmutableMap.of("expression-manager-factory.name", "bar")); + + manager.addExpressionOptimizerFactory(getExpressionOptimizerFactory("foo")); + manager.addExpressionOptimizerFactory(getExpressionOptimizerFactory("bar")); + manager.loadExpressionOptimizerFactories(); + + assertOptimizedExpression("1+1", "2", ImmutableMap.of()); + + // Override the default optimizer based on session property + assertOptimizedExpression("1+1", "'foo'", ImmutableMap.of("expression_optimizer_instance", "foo")); + assertOptimizedExpression("1+1", "'bar'", ImmutableMap.of("expression_optimizer_instance", "bar")); + } + + @Test + public void testDefaultMayBeConfigured() + throws Exception + { + createPropertiesFile("foo.properties", ImmutableMap.of("expression-manager-factory.name", "foo", "default", "true")); + createPropertiesFile("bar.properties", ImmutableMap.of("expression-manager-factory.name", "bar")); + + manager.addExpressionOptimizerFactory(getExpressionOptimizerFactory("foo")); + manager.addExpressionOptimizerFactory(getExpressionOptimizerFactory("bar")); + manager.loadExpressionOptimizerFactories(); + + assertOptimizedExpression("2+2", "'foo'", ImmutableMap.of()); + + // Override the default optimizer based on session property + assertOptimizedExpression("2+2", "'bar'", ImmutableMap.of("expression_optimizer_instance", "bar")); + assertOptimizedExpression("2+2", "4", ImmutableMap.of("expression_optimizer_instance", "default")); + } + + @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "Cannot name an expression optimizer instance default") + public void testNoNewOptimizerInstanceCalledDefault() + throws Exception + { + createPropertiesFile("default.properties", ImmutableMap.of("expression-manager-factory.name", "default")); + + manager.addExpressionOptimizerFactory(getExpressionOptimizerFactory("default")); + manager.loadExpressionOptimizerFactories(); + } + + @Test + public void testNoFactoryName() + throws Exception + { + createPropertiesFile("foo.properties", ImmutableMap.of()); + + manager.addExpressionOptimizerFactory(getExpressionOptimizerFactory("foo")); + assertThrows(IllegalArgumentException.class, () -> manager.loadExpressionOptimizerFactories()); + } + + @Test + public void testNoFactoryRegistered() + throws Exception + { + createPropertiesFile("foo.properties", ImmutableMap.of("expression-manager-factory.name", "foo")); + assertThrows(IllegalArgumentException.class, () -> manager.loadExpressionOptimizerFactories()); + } + + private void assertOptimizedExpression(String originalExpression, String optimizedExpression, Map systemProperties) + { + Session.SessionBuilder sessionBuilder = testSessionBuilder(); + systemProperties.forEach(sessionBuilder::setSystemProperty); + Session session = sessionBuilder.build(); + assertEquals(manager.getExpressionOptimizer(session.toConnectorSession()).optimize(expression(originalExpression), OPTIMIZED, session.toConnectorSession()), + expression(optimizedExpression)); + } + + private static RowExpression expression(String expression) + { + return TRANSLATOR.translate(expression, ImmutableMap.of()); + } + + private void createPropertiesFile(String fileName, Map propertiesMap) + throws IOException + { + File newProperties = new File(directory, fileName); + newProperties.deleteOnExit(); + Properties properties = new Properties(); + properties.putAll(propertiesMap); + properties.store(newOutputStream(newProperties.toPath()), null); + } + + public ExpressionOptimizerFactory getExpressionOptimizerFactory(String name) + { + return new ExpressionOptimizerFactory() { + @Override + public ExpressionOptimizer createOptimizer(Map config, ExpressionOptimizerContext context) + { + return (expression, level, session, variableResolver) -> constant( + Slices.utf8Slice(name), + METADATA.getType(TypeSignature.parseTypeSignature(format("varchar(%s)", name.length())))); + } + + @Override + public String getName() + { + return name; + } + }; + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java index d3c644b793ca6..f44809325bc02 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java @@ -14,10 +14,18 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.Session; +import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.block.SortOrder; +import com.facebook.presto.common.type.StandardTypes; import com.facebook.presto.functionNamespace.FunctionNamespaceManagerPlugin; import com.facebook.presto.functionNamespace.json.JsonFileBasedFunctionNamespaceManagerFactory; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.AggregationFunctionMetadata; +import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.Parameter; +import com.facebook.presto.spi.function.RoutineCharacteristics; +import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.FilterNode; @@ -76,8 +84,12 @@ import static com.facebook.presto.common.block.SortOrder.ASC_NULLS_LAST; import static com.facebook.presto.common.predicate.Domain.singleValue; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.VarcharType.createVarcharType; import static com.facebook.presto.spi.StandardErrorCode.INVALID_LIMIT_CLAUSE; +import static com.facebook.presto.spi.function.FunctionVersion.notVersioned; +import static com.facebook.presto.spi.function.RoutineCharacteristics.Determinism.DETERMINISTIC; +import static com.facebook.presto.spi.function.RoutineCharacteristics.Language.CPP; import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; @@ -88,8 +100,6 @@ import static com.facebook.presto.spi.plan.JoinType.RIGHT; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; -import static com.facebook.presto.sql.TestExpressionInterpreter.AVG_UDAF_CPP; -import static com.facebook.presto.sql.TestExpressionInterpreter.SQUARE_UDF_CPP; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.ELIMINATE_CROSS_JOINS; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.any; @@ -144,6 +154,26 @@ public class TestLogicalPlanner extends BasePlanTest { + public static final SqlInvokedFunction SQUARE_UDF_CPP = new SqlInvokedFunction( + QualifiedObjectName.valueOf(new CatalogSchemaName("json", "test_schema"), "square"), + ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.BIGINT))), + parseTypeSignature(StandardTypes.BIGINT), + "Integer square", + RoutineCharacteristics.builder().setDeterminism(DETERMINISTIC).setLanguage(CPP).build(), + "", + notVersioned()); + + public static final SqlInvokedFunction AVG_UDAF_CPP = new SqlInvokedFunction( + QualifiedObjectName.valueOf(new CatalogSchemaName("json", "test_schema"), "avg"), + ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.DOUBLE))), + parseTypeSignature(StandardTypes.DOUBLE), + "Returns mean of doubles", + RoutineCharacteristics.builder().setDeterminism(DETERMINISTIC).setLanguage(CPP).build(), + "", + notVersioned(), + FunctionKind.AGGREGATE, + Optional.of(new AggregationFunctionMetadata(parseTypeSignature("ROW(double, int)"), false))); + // TODO: Use com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder#tableScan with required node/stream // partitioning to properly test aggregation, window function and join. diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java index b4ad658409b82..cea4abb4e0787 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/OptimizerAssert.java @@ -16,13 +16,16 @@ import com.facebook.presto.Session; import com.facebook.presto.cost.StatsAndCosts; import com.facebook.presto.cost.StatsCalculator; +import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.Optimizer; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.TypeProvider; @@ -170,7 +173,11 @@ private List getMinimalOptimizers() new RuleStatsRecorder(), queryRunner.getStatsCalculator(), queryRunner.getCostCalculator(), - new SimplifyRowExpressions(metadata).rules())); + new SimplifyRowExpressions( + metadata, + new ExpressionOptimizerManager( + new PluginNodeManager(new InMemoryNodeManager()), + queryRunner.getFunctionAndTypeManager())).rules())); } private void inTransaction(Function transactionSessionConsumer) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveMapCastRule.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveMapCastRule.java index 19638181d35e4..88b0c4a771f56 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveMapCastRule.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveMapCastRule.java @@ -37,7 +37,7 @@ public class TestRemoveMapCastRule public void testSubscriptCast() { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll(new RemoveMapCastRule(getFunctionManager()).rules()).build()) + ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager()).rules()).addAll(new RemoveMapCastRule(getFunctionManager()).rules()).build()) .setSystemProperty(REMOVE_MAP_CAST, "true") .on(p -> { VariableReferenceExpression a = p.variable("a", DOUBLE); @@ -57,7 +57,7 @@ public void testSubscriptCast() public void testElementAtCast() { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll(new RemoveMapCastRule(getFunctionManager()).rules()).build()) + ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager()).rules()).addAll(new RemoveMapCastRule(getFunctionManager()).rules()).build()) .setSystemProperty(REMOVE_MAP_CAST, "true") .on(p -> { VariableReferenceExpression a = p.variable("a", DOUBLE); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteConstantArrayContainsToInExpression.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteConstantArrayContainsToInExpression.java index 47c50c31a7308..3540dfaeb9fba 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteConstantArrayContainsToInExpression.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteConstantArrayContainsToInExpression.java @@ -36,7 +36,7 @@ public class TestRewriteConstantArrayContainsToInExpression public void testNoNull() { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll( + ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager()).rules()).addAll( new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()).build()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") .on(p -> { @@ -101,7 +101,7 @@ public void testDoesNotFireForEmpty() public void testNotFire() { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll( + ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager()).rules()).addAll( new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()).build()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") .on(p -> { @@ -122,7 +122,7 @@ public void testNotFire() public void testWithNull() { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll( + ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager()).rules()).addAll( new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()).build()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") .on(p -> { @@ -142,7 +142,7 @@ public void testWithNull() public void testLambda() { tester().assertThat( - ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata()).rules()).addAll( + ImmutableSet.>builder().addAll(new SimplifyRowExpressions(getMetadata(), getExpressionManager()).rules()).addAll( new RewriteConstantArrayContainsToInExpression(getFunctionManager()).rules()).build()) .setSystemProperty(REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION, "true") .on(p -> { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java index 96da619e3ccb1..7817b311053df 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java @@ -17,11 +17,14 @@ import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.expressions.RowExpressionRewriter; import com.facebook.presto.expressions.RowExpressionTreeRewriter; +import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Expression; @@ -181,9 +184,12 @@ private static void assertSimplifies(String expression, String rowExpressionExpe { Expression actualExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(expression)); + InMemoryNodeManager nodeManager = new InMemoryNodeManager(); + ExpressionOptimizerManager expressionOptimizerManager = new ExpressionOptimizerManager(new PluginNodeManager(nodeManager), METADATA.getFunctionAndTypeManager()); + TestingRowExpressionTranslator translator = new TestingRowExpressionTranslator(METADATA); RowExpression actualRowExpression = translator.translate(actualExpression, TypeProvider.viewOf(TYPES)); - RowExpression simplifiedRowExpression = SimplifyRowExpressions.rewrite(actualRowExpression, METADATA, TEST_SESSION.toConnectorSession()); + RowExpression simplifiedRowExpression = SimplifyRowExpressions.rewrite(actualRowExpression, METADATA, TEST_SESSION, expressionOptimizerManager); Expression expectedByRowExpression = rewriteIdentifiersToSymbolReferences(SQL_PARSER.createExpression(rowExpressionExpected)); RowExpression simplifiedByExpression = translator.translate(expectedByRowExpression, TypeProvider.viewOf(TYPES)); assertEquals(normalize(simplifiedRowExpression), normalize(simplifiedByExpression)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java index 04ce733e96aab..0dafb0c96b8b6 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/BaseRuleTest.java @@ -16,6 +16,7 @@ import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.Plugin; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.Plan; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; @@ -84,4 +85,9 @@ protected void assertNodePresentInPlan(Plan plan, Class nodeClass) .matches(), "Expected " + nodeClass.toString() + " in plan after optimization. "); } + + protected ExpressionOptimizerManager getExpressionManager() + { + return tester.getExpressionManager(); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java index 3c9540ef42b2e..33d46ff6b885e 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleTester.java @@ -26,6 +26,7 @@ import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.assertions.OptimizerAssert; @@ -61,6 +62,7 @@ public class RuleTester private final PageSourceManager pageSourceManager; private final AccessControl accessControl; private final SqlParser sqlParser; + private ExpressionOptimizerManager expressionOptimizerManager; public RuleTester() { @@ -107,6 +109,7 @@ public RuleTester(List plugins, Map sessionProperties, S connectorFactory, ImmutableMap.of()); plugins.stream().forEach(queryRunner::installPlugin); + expressionOptimizerManager = queryRunner.getExpressionManager(); this.metadata = queryRunner.getMetadata(); this.transactionManager = queryRunner.getTransactionManager(); @@ -197,4 +200,9 @@ public List> getTableConstraints(TableHandle table return metadata.getTableMetadata(transactionSession, tableHandle).getMetadata().getTableConstraintsHolder().getTableConstraintsWithColumnHandles(); }); } + + public ExpressionOptimizerManager getExpressionManager() + { + return expressionOptimizerManager; + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestCteProjectionAndPredicatePushdown.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestCteProjectionAndPredicatePushdown.java index f36c3aaabcf15..0e73b6b3d5516 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestCteProjectionAndPredicatePushdown.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestCteProjectionAndPredicatePushdown.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.sql.Optimizer; +import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; @@ -144,7 +145,7 @@ private void assertCtePlan(String sql, PlanMatchPattern pattern) new RemoveIdentityProjectionsBelowProjection(), new PruneRedundantProjectionAssignments())), new PruneUnreferencedOutputs(), - new CteProjectionAndPredicatePushDown(metadata)); + new CteProjectionAndPredicatePushDown(metadata, getQueryRunner().getExpressionManager(), new FeaturesConfig())); assertPlan(sql, getSession(), Optimizer.PlanStage.OPTIMIZED, pattern, optimizers); } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunner.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunner.java index 9cd88e2ade8db..f30a34002de29 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunner.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunner.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; @@ -233,6 +234,12 @@ public TestingAccessControlManager getAccessControl() throw new UnsupportedOperationException(); } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + throw new UnsupportedOperationException(); + } + @Override public MaterializedResult execute(String sql) { diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java index ed017e654cceb..b84557391a8b7 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java @@ -80,6 +80,7 @@ import com.facebook.presto.metadata.StaticFunctionNamespaceStore; import com.facebook.presto.metadata.StaticFunctionNamespaceStoreConfig; import com.facebook.presto.metadata.TablePropertyManager; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.operator.FileFragmentResultCacheConfig; import com.facebook.presto.operator.FileFragmentResultCacheManager; import com.facebook.presto.operator.FragmentCacheStats; @@ -173,6 +174,7 @@ import com.facebook.presto.sql.analyzer.MetadataExtractorMBean; import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -346,6 +348,9 @@ protected void setup(Binder binder) binder.bind(AnalyzePropertyManager.class).in(Scopes.SINGLETON); binder.bind(QuerySessionSupplier.class).in(Scopes.SINGLETON); + // expression manager + binder.bind(ExpressionOptimizerManager.class).in(Scopes.SINGLETON); + // tracer provider managers binder.bind(TracerProviderManager.class).in(Scopes.SINGLETON); @@ -508,6 +513,7 @@ protected void setup(Binder binder) // TODO: Decouple and remove: required by ConnectorManager binder.bind(InternalNodeManager.class).toInstance(new PrestoSparkInternalNodeManager()); + binder.bind(PluginNodeManager.class); // TODO: Decouple and remove: required by PluginManager binder.bind(InternalResourceGroupManager.class).in(Scopes.SINGLETON); diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java index f85c08ef58eb1..2ddde52dfa793 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java @@ -64,6 +64,7 @@ import com.facebook.presto.spi.security.PrincipalType; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; @@ -504,6 +505,12 @@ public TestingAccessControlManager getAccessControl() return testingAccessControlManager; } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + throw new UnsupportedOperationException(); + } + public HistoryBasedPlanStatisticsManager getHistoryBasedPlanStatisticsManager() { return historyBasedPlanStatisticsManager; diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java index 303ae0dfe7801..bedd15a5d9e10 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.tests; +import com.facebook.airlift.node.NodeInfo; import com.facebook.presto.Session; import com.facebook.presto.common.type.Type; import com.facebook.presto.cost.CostCalculator; @@ -21,11 +22,14 @@ import com.facebook.presto.cost.CostComparator; import com.facebook.presto.cost.TaskCountEstimator; import com.facebook.presto.execution.QueryManagerConfig; +import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.security.AccessDeniedException; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.QueryExplainer; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.Plan; @@ -74,6 +78,7 @@ public abstract class AbstractTestQueryFramework { + private static final NodeInfo NODE_INFO = new NodeInfo("test"); private QueryRunner queryRunner; private ExpectedQueryRunner expectedQueryRunner; private SqlParser sqlParser; @@ -568,7 +573,10 @@ private QueryExplainer getQueryExplainer() new CostComparator(featuresConfig), taskCountEstimator, new PartitioningProviderManager(), - featuresConfig) + featuresConfig, + new ExpressionOptimizerManager( + new PluginNodeManager(new InMemoryNodeManager()), + queryRunner.getMetadata().getFunctionAndTypeManager())) .getPlanningTimeOptimizers(); return new QueryExplainer( optimizers, diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index 62502743809b1..5674d59e752b7 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -42,6 +42,7 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; @@ -626,6 +627,13 @@ public PlanCheckerProviderManager getPlanCheckerProviderManager() return coordinators.get(0).getPlanCheckerProviderManager(); } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + checkState(coordinators.size() == 1, "Expected a single coordinator"); + return coordinators.get(0).getExpressionManager(); + } + public TestingPrestoServer getCoordinator() { checkState(coordinators.size() == 1, "Expected a single coordinator"); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java index 980fb4f991bcd..00993ae054951 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java @@ -27,6 +27,7 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; @@ -188,6 +189,12 @@ public TestingAccessControlManager getAccessControl() return server.getAccessControl(); } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + return server.getExpressionManager(); + } + public TestingPrestoServer getServer() { return server; diff --git a/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java b/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java index c9a4f82b7cd84..45c030cff3ac7 100644 --- a/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java +++ b/presto-thrift-connector/src/test/java/com/facebook/presto/connector/thrift/integration/ThriftQueryRunner.java @@ -34,6 +34,7 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; @@ -254,6 +255,12 @@ public TestingAccessControlManager getAccessControl() return source.getAccessControl(); } + @Override + public ExpressionOptimizerManager getExpressionManager() + { + return source.getExpressionManager(); + } + @Override public MaterializedResult execute(String sql) { From 2cf08d29cd174d6190d6635c3bbe607c2dc61fa6 Mon Sep 17 00:00:00 2001 From: Tim Meehan Date: Fri, 22 Nov 2024 16:23:23 -0500 Subject: [PATCH 3/3] Use ExpressionOptimizerProvider The runtime should consolidate to the `ExpressionOptimizerProvider` factory so that it can be customized without significant refactoring. --- .../presto/plugin/jdbc/JdbcConnector.java | 2 +- .../optimization/JdbcComputePushdown.java | 10 ++--- .../JdbcPlanOptimizerProvider.java | 10 ++--- .../optimization/TestJdbcComputePushdown.java | 2 +- .../clickhouse/ClickHouseConnector.java | 3 +- .../ClickHouseComputePushdown.java | 10 ++--- .../ClickHousePlanOptimizerProvider.java | 13 +++---- .../facebook/presto/hive/MetadataUtils.java | 2 +- .../rule/BaseSubfieldExtractionRewriter.java | 6 +-- .../presto/hive/FilteringPageSource.java | 2 +- .../facebook/presto/hive/HiveMetadata.java | 2 +- .../presto/hive/HivePageSourceProvider.java | 2 +- .../orc/OrcSelectivePageSourceFactory.java | 2 +- .../presto/hive/AbstractTestHiveClient.java | 2 +- .../facebook/presto/hive/HiveTestUtils.java | 4 +- .../optimizer/IcebergMetadataOptimizer.java | 2 +- .../optimizer/IcebergPlanOptimizer.java | 4 +- .../presto/connector/ConnectorManager.java | 7 +++- .../presto/cost/ScalarStatsCalculator.java | 14 +++++-- .../ExpressionOptimizerManager.java | 3 ++ .../presto/sql/planner/PlanOptimizers.java | 4 +- .../optimizations/PredicatePushDown.java | 38 ++++++++++++------ .../optimizations/PushdownSubfields.java | 13 ++++--- .../ConnectorRowExpressionService.java | 11 +++--- .../presto/testing/LocalQueryRunner.java | 3 +- .../testing/TestingConnectorContext.java | 6 ++- ...AbstractTestComparisonStatsCalculator.java | 3 +- .../AbstractTestFilterStatsCalculator.java | 3 +- ...ConnectorFilterStatsCalculatorService.java | 8 +++- .../presto/cost/TestJoinStatsRule.java | 6 ++- .../cost/TestScalarStatsCalculator.java | 6 ++- .../InMemoryExpressionOptimizerProvider.java | 39 +++++++++++++++++++ .../sql/planner/TestPredicatePushdown.java | 7 ++-- .../optimizations/TestReorderWindows.java | 3 +- .../rule/ParquetDereferencePushDown.java | 2 +- .../relation/ExpressionOptimizerProvider.java | 21 ++++++++++ .../spi/relation/RowExpressionService.java | 3 +- 37 files changed, 194 insertions(+), 84 deletions(-) create mode 100644 presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizerProvider.java diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java index 3eb3ef0112375..511899a2e03f9 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java @@ -109,7 +109,7 @@ public ConnectorPlanOptimizerProvider getConnectorPlanOptimizerProvider() functionManager, functionResolution, rowExpressionService.getDeterminismEvaluator(), - rowExpressionService.getExpressionOptimizer()); + rowExpressionService); } @Override diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java index e0ad9a3282f38..8d8db43d97654 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java @@ -29,7 +29,7 @@ import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.relation.DeterminismEvaluator; -import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.RowExpression; import java.util.Optional; @@ -44,7 +44,7 @@ public class JdbcComputePushdown implements ConnectorPlanOptimizer { - private final ExpressionOptimizer expressionOptimizer; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final JdbcFilterToSqlTranslator jdbcFilterToSqlTranslator; private final LogicalRowExpressions logicalRowExpressions; @@ -52,7 +52,7 @@ public JdbcComputePushdown( FunctionMetadataManager functionMetadataManager, StandardFunctionResolution functionResolution, DeterminismEvaluator determinismEvaluator, - ExpressionOptimizer expressionOptimizer, + ExpressionOptimizerProvider expressionOptimizerProvider, String identifierQuote, Set> functionTranslators) { @@ -62,7 +62,7 @@ public JdbcComputePushdown( requireNonNull(determinismEvaluator, "determinismEvaluator is null"); requireNonNull(functionResolution, "functionResolution is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.jdbcFilterToSqlTranslator = new JdbcFilterToSqlTranslator( functionMetadataManager, buildFunctionTranslator(functionTranslators), @@ -106,7 +106,7 @@ public PlanNode visitFilter(FilterNode node, RewriteContext context) TableHandle oldTableHandle = oldTableScanNode.getTable(); JdbcTableHandle oldConnectorTable = (JdbcTableHandle) oldTableHandle.getConnectorHandle(); - RowExpression predicate = expressionOptimizer.optimize(node.getPredicate(), OPTIMIZED, session); + RowExpression predicate = expressionOptimizerProvider.getExpressionOptimizer(session).optimize(node.getPredicate(), OPTIMIZED, session); predicate = logicalRowExpressions.convertToConjunctiveNormalForm(predicate); TranslatedExpression jdbcExpression = translateWith( predicate, diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java index 9dcd399b66472..5edbfd21eb7f5 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java @@ -20,7 +20,7 @@ import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.relation.DeterminismEvaluator; -import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; @@ -34,7 +34,7 @@ public class JdbcPlanOptimizerProvider private final FunctionMetadataManager functionManager; private final StandardFunctionResolution functionResolution; private final DeterminismEvaluator determinismEvaluator; - private final ExpressionOptimizer expressionOptimizer; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final String identifierQuote; @Inject @@ -43,12 +43,12 @@ public JdbcPlanOptimizerProvider( FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution, DeterminismEvaluator determinismEvaluator, - ExpressionOptimizer expressionOptimizer) + ExpressionOptimizerProvider expressionOptimizerProvider) { this.functionManager = requireNonNull(functionManager, "functionManager is null"); this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.identifierQuote = jdbcClient.getIdentifierQuote(); } @@ -65,7 +65,7 @@ public Set getPhysicalPlanOptimizers() functionManager, functionResolution, determinismEvaluator, - expressionOptimizer, + expressionOptimizerProvider, identifierQuote, getFunctionTranslators())); } diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java index 726f89f6631ba..34669ff0f1fa2 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java @@ -105,7 +105,7 @@ public TestJdbcComputePushdown() functionAndTypeManager, functionResolution, determinismEvaluator, - new RowExpressionOptimizer(METADATA), + (ConnectorSession session) -> new RowExpressionOptimizer(METADATA), "'", getFunctionTranslators()); } diff --git a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseConnector.java b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseConnector.java index 421b95d1cb65c..a3ced140b87da 100755 --- a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseConnector.java +++ b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseConnector.java @@ -113,8 +113,7 @@ public ConnectorPlanOptimizerProvider getConnectorPlanOptimizerProvider() clickHouseClient, functionManager, functionResolution, - rowExpressionService.getDeterminismEvaluator(), - rowExpressionService.getExpressionOptimizer(), + rowExpressionService, clickhouseQueryGenerator); } diff --git a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/optimization/ClickHouseComputePushdown.java b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/optimization/ClickHouseComputePushdown.java index c025db58fa542..402aa6a912bef 100755 --- a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/optimization/ClickHouseComputePushdown.java +++ b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/optimization/ClickHouseComputePushdown.java @@ -35,8 +35,8 @@ import com.facebook.presto.spi.plan.PlanVisitor; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.relation.DeterminismEvaluator; -import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionService; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.google.common.collect.ImmutableList; @@ -57,7 +57,7 @@ public class ClickHouseComputePushdown implements ConnectorPlanOptimizer { - private final ExpressionOptimizer expressionOptimizer; + private final RowExpressionService rowExpressionService; private final ClickHouseFilterToSqlTranslator clickHouseFilterToSqlTranslator; private final LogicalRowExpressions logicalRowExpressions; private final ClickHouseQueryGenerator clickhouseQueryGenerator; @@ -67,7 +67,7 @@ public ClickHouseComputePushdown( FunctionMetadataManager functionMetadataManager, StandardFunctionResolution functionResolution, DeterminismEvaluator determinismEvaluator, - ExpressionOptimizer expressionOptimizer, + RowExpressionService rowExpressionService, String identifierQuote, Set> functionTranslators, ClickHouseQueryGenerator clickhouseQueryGenerator) @@ -78,7 +78,7 @@ public ClickHouseComputePushdown( requireNonNull(determinismEvaluator, "determinismEvaluator is null"); requireNonNull(functionResolution, "functionResolution is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); this.clickHouseFilterToSqlTranslator = new ClickHouseFilterToSqlTranslator( functionMetadataManager, buildFunctionTranslator(functionTranslators), @@ -256,7 +256,7 @@ public PlanNode visitFilter(FilterNode node, Void context) TableHandle oldTableHandle = oldTableScanNode.getTable(); ClickHouseTableHandle oldConnectorTable = (ClickHouseTableHandle) oldTableHandle.getConnectorHandle(); - RowExpression predicate = expressionOptimizer.optimize(node.getPredicate(), OPTIMIZED, session); + RowExpression predicate = rowExpressionService.getExpressionOptimizer(session).optimize(node.getPredicate(), OPTIMIZED, session); predicate = logicalRowExpressions.convertToConjunctiveNormalForm(predicate); TranslatedExpression clickHouseExpression = translateWith( predicate, diff --git a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/optimization/ClickHousePlanOptimizerProvider.java b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/optimization/ClickHousePlanOptimizerProvider.java index 60d59692971c8..8643b7acba550 100755 --- a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/optimization/ClickHousePlanOptimizerProvider.java +++ b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/optimization/ClickHousePlanOptimizerProvider.java @@ -20,7 +20,7 @@ import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.relation.DeterminismEvaluator; -import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpressionService; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; @@ -33,8 +33,8 @@ public class ClickHousePlanOptimizerProvider { private final FunctionMetadataManager functionManager; private final StandardFunctionResolution functionResolution; + private final RowExpressionService rowExpressionService; private final DeterminismEvaluator determinismEvaluator; - private final ExpressionOptimizer expressionOptimizer; private final String identifierQuote; private final ClickHouseQueryGenerator clickhouseQueryGenerator; @@ -43,16 +43,15 @@ public ClickHousePlanOptimizerProvider( ClickHouseClient clickHouseClient, FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution, - DeterminismEvaluator determinismEvaluator, - ExpressionOptimizer expressionOptimizer, + RowExpressionService rowExpressionService, ClickHouseQueryGenerator clickhouseQueryGenerator) { this.functionManager = requireNonNull(functionManager, "functionManager is null"); this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); - this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.rowExpressionService = requireNonNull(rowExpressionService, "rowExpressionService is null"); this.identifierQuote = clickHouseClient.getIdentifierQuote(); this.clickhouseQueryGenerator = clickhouseQueryGenerator; + this.determinismEvaluator = rowExpressionService.getDeterminismEvaluator(); } @Override @@ -68,7 +67,7 @@ public Set getPhysicalPlanOptimizers() functionManager, functionResolution, determinismEvaluator, - expressionOptimizer, + rowExpressionService, identifierQuote, getFunctionTranslators(), clickhouseQueryGenerator)); diff --git a/presto-hive-common/src/main/java/com/facebook/presto/hive/MetadataUtils.java b/presto-hive-common/src/main/java/com/facebook/presto/hive/MetadataUtils.java index 361709b7e496b..2b76980e66ab0 100644 --- a/presto-hive-common/src/main/java/com/facebook/presto/hive/MetadataUtils.java +++ b/presto-hive-common/src/main/java/com/facebook/presto/hive/MetadataUtils.java @@ -85,7 +85,7 @@ public static RowExpression getSubfieldPredicate( StandardFunctionResolution functionResolution, RowExpressionService rowExpressionService) { - SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session); + SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(session), session); return rowExpressionService.getDomainTranslator().toPredicate( layoutHandle.getDomainPredicate() diff --git a/presto-hive-common/src/main/java/com/facebook/presto/hive/rule/BaseSubfieldExtractionRewriter.java b/presto-hive-common/src/main/java/com/facebook/presto/hive/rule/BaseSubfieldExtractionRewriter.java index add6ddb6b7a60..fe541753738b2 100644 --- a/presto-hive-common/src/main/java/com/facebook/presto/hive/rule/BaseSubfieldExtractionRewriter.java +++ b/presto-hive-common/src/main/java/com/facebook/presto/hive/rule/BaseSubfieldExtractionRewriter.java @@ -217,7 +217,7 @@ public ConnectorPushdownFilterResult pushdownFilter( ExtractionResult decomposedFilter = rowExpressionService.getDomainTranslator() .fromPredicate(session, filter, new SubfieldExtractor( functionResolution, - rowExpressionService.getExpressionOptimizer(), + rowExpressionService.getExpressionOptimizer(session), session).toColumnExtractor()); if (currentLayoutHandle.isPresent()) { @@ -231,7 +231,7 @@ public ConnectorPushdownFilterResult pushdownFilter( return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT); } - RowExpression optimizedRemainingExpression = rowExpressionService.getExpressionOptimizer() + RowExpression optimizedRemainingExpression = rowExpressionService.getExpressionOptimizer(session) .optimize(decomposedFilter.getRemainingExpression(), OPTIMIZED, session); if (optimizedRemainingExpression instanceof ConstantExpression) { ConstantExpression constantExpression = (ConstantExpression) optimizedRemainingExpression; @@ -438,7 +438,7 @@ private boolean isCandidate(Map bindings) // spurious query failures for partitions that would otherwise be filtered out. RowExpression optimized; try { - optimized = evaluator.getExpressionOptimizer().optimize(expression, OPTIMIZED, session, variableResolver); + optimized = evaluator.getExpressionOptimizer(session).optimize(expression, OPTIMIZED, session, variableResolver); } catch (PrestoException e) { propagateIfUnhandled(e); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/FilteringPageSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/FilteringPageSource.java index 07f47603e0523..0890c00278fcd 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/FilteringPageSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/FilteringPageSource.java @@ -118,7 +118,7 @@ public FilteringPageSource( columnHandle -> new VariableReferenceExpression(Optional.empty(), columnHandle.getName(), columnHandle.getHiveType().getType(typeManager)), columnHandle -> new InputReferenceExpression(Optional.empty(), columnHandle.getHiveColumnIndex(), columnHandle.getHiveType().getType(typeManager)))); - RowExpression optimizedRemainingPredicate = rowExpressionService.getExpressionOptimizer().optimize(remainingPredicate, OPTIMIZED, session); + RowExpression optimizedRemainingPredicate = rowExpressionService.getExpressionOptimizer(session).optimize(remainingPredicate, OPTIMIZED, session); if (TRUE_CONSTANT.equals(optimizedRemainingPredicate)) { this.filterFunction = null; } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index 3d3a0eacd4d44..b3319563d7469 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -888,7 +888,7 @@ public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTab .transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null) .transform(allColumns::get))); - SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session); + SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(session), session); RowExpression domainPredicate = rowExpressionService.getDomainTranslator().toPredicate( hiveLayoutHandle.getDomainPredicate() diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java index 7065dc6720811..85d00b8b9571c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java @@ -129,7 +129,7 @@ public HivePageSourceProvider( this.optimizedRowExpressionCache = CacheBuilder.newBuilder() .recordStats() .maximumSize(10_000) - .build(CacheLoader.from(cacheKey -> rowExpressionService.getExpressionOptimizer().optimize(cacheKey.rowExpression, OPTIMIZED, cacheKey.session))); + .build(CacheLoader.from(cacheKey -> rowExpressionService.getExpressionOptimizer(cacheKey.session).optimize(cacheKey.rowExpression, OPTIMIZED, cacheKey.session))); } @Override diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java index 6cc1054b59a37..29c680aac3762 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java @@ -425,7 +425,7 @@ private static Map> collectRequiredSubfields(List outputSubfields.put(column.getHiveColumnIndex(), new HashSet<>(column.getRequiredSubfields()))); Map> predicateSubfields = new HashMap<>(); - SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session); + SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(session), session); remainingPredicate.accept( new RequiredSubfieldsExtractor(subfieldExtractor), subfield -> predicateSubfields.computeIfAbsent(columnIndices.get(subfield.getRootName()), v -> new HashSet<>()).add(subfield)); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index d622e36da67e5..28c4fc5146388 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -524,7 +524,7 @@ private static RowType toRowType(List columns) }).collect(toList())) .build(); - private static final SubfieldExtractor SUBFIELD_EXTRACTOR = new SubfieldExtractor(FUNCTION_RESOLUTION, ROW_EXPRESSION_SERVICE.getExpressionOptimizer(), SESSION); + private static final SubfieldExtractor SUBFIELD_EXTRACTOR = new SubfieldExtractor(FUNCTION_RESOLUTION, ROW_EXPRESSION_SERVICE.getExpressionOptimizer(SESSION), SESSION); private static final TypeProvider TYPE_PROVIDER_AFTER = TypeProvider.copyOf(MISMATCH_SCHEMA_TABLE_AFTER.stream() .collect(toImmutableMap(ColumnMetadata::getName, ColumnMetadata::getType))); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java b/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java index a6fdaa9f73677..37b3b35139a27 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java @@ -126,7 +126,7 @@ public DomainTranslator getDomainTranslator() } @Override - public ExpressionOptimizer getExpressionOptimizer() + public ExpressionOptimizer getExpressionOptimizer(ConnectorSession session) { return new RowExpressionOptimizer(METADATA); } @@ -151,7 +151,7 @@ public String formatRowExpression(ConnectorSession session, RowExpression expres }; public static final FilterStatsCalculatorService FILTER_STATS_CALCULATOR_SERVICE = new ConnectorFilterStatsCalculatorService( - new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA), new StatsNormalizer())); + new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA, ROW_EXPRESSION_SERVICE), new StatsNormalizer())); public static final HiveClientConfig HIVE_CLIENT_CONFIG = new HiveClientConfig(); public static final MetastoreClientConfig METASTORE_CLIENT_CONFIG = new MetastoreClientConfig(); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergMetadataOptimizer.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergMetadataOptimizer.java index f21d39f956473..3e35e1d6afa4a 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergMetadataOptimizer.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergMetadataOptimizer.java @@ -356,7 +356,7 @@ else if (scalarFunctionName.equals("least")) { throw new PrestoException(StandardErrorCode.NOT_SUPPORTED, "unsupported function: " + scalarFunctionName); } - RowExpression reducedValue = rowExpressionService.getExpressionOptimizer().optimize( + RowExpression reducedValue = rowExpressionService.getExpressionOptimizer(connectorSession).optimize( new CallExpression( Optional.empty(), scalarFunctionName, diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizer.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizer.java index 35f8c6b356bb7..25438e4d12722 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizer.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizer.java @@ -154,7 +154,7 @@ public PlanNode visitFilter(FilterNode filter, RewriteContext context) //TODO we should optimize the filter expression DomainTranslator.ExtractionResult decomposedFilter = rowExpressionService.getDomainTranslator() - .fromPredicate(session, filterPredicate, new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session).toColumnExtractor()); + .fromPredicate(session, filterPredicate, new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(session), session).toColumnExtractor()); // Only pushdown the range filters which apply to entire columns, because iceberg does not accept the filters on the subfields in nested structures TupleDomain entireColumnDomain = decomposedFilter.getTupleDomain() @@ -167,7 +167,7 @@ public PlanNode visitFilter(FilterNode filter, RewriteContext context) Table icebergTable = getIcebergTable(metadata, session, tableHandle.getSchemaTableName()); // Get predicate expression on subfield - SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session); + SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(session), session); Map columnTypes = nameToColumnHandlesMapping.entrySet().stream() .collect(toImmutableMap(entry -> entry.getKey(), entry -> entry.getValue().getType())); TupleDomain subfieldTupleDomain = decomposedFilter.getTupleDomain() diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java index 6ed7888b33d54..56f3e15d72aa6 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -62,12 +62,12 @@ import com.facebook.presto.split.RecordPageSourceProvider; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.planPrinter.RowExpressionFormatter; import com.facebook.presto.sql.relational.ConnectorRowExpressionService; import com.facebook.presto.sql.relational.FunctionResolution; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -116,6 +116,7 @@ public class ConnectorManager private final PageIndexerFactory pageIndexerFactory; private final NodeInfo nodeInfo; private final TransactionManager transactionManager; + private final ExpressionOptimizerManager expressionOptimizerManager; private final DomainTranslator domainTranslator; private final PredicateCompiler predicateCompiler; private final DeterminismEvaluator determinismEvaluator; @@ -151,6 +152,7 @@ public ConnectorManager( PageSorter pageSorter, PageIndexerFactory pageIndexerFactory, TransactionManager transactionManager, + ExpressionOptimizerManager expressionOptimizerManager, DomainTranslator domainTranslator, PredicateCompiler predicateCompiler, DeterminismEvaluator determinismEvaluator, @@ -176,6 +178,7 @@ public ConnectorManager( this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); this.nodeInfo = requireNonNull(nodeInfo, "nodeInfo is null"); this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); this.predicateCompiler = requireNonNull(predicateCompiler, "predicateCompiler is null"); this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); @@ -382,7 +385,7 @@ private Connector createConnector(ConnectorId connectorId, ConnectorFactory fact pageIndexerFactory, new ConnectorRowExpressionService( domainTranslator, - new RowExpressionOptimizer(metadataManager), + expressionOptimizerManager, predicateCompiler, determinismEvaluator, new RowExpressionFormatter(metadataManager.getFunctionAndTypeManager())), diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java index 044785ca0a440..a25351ca7a88a 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.InputReferenceExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -31,11 +32,11 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.analyzer.ExpressionAnalyzer; import com.facebook.presto.sql.analyzer.Scope; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ExpressionInterpreter; import com.facebook.presto.sql.planner.NoOpVariableResolver; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.relational.FunctionResolution; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.ArithmeticUnaryExpression; import com.facebook.presto.sql.tree.AstVisitor; @@ -78,11 +79,18 @@ public class ScalarStatsCalculator { private final Metadata metadata; + private final ExpressionOptimizerProvider expressionOptimizerProvider; @Inject - public ScalarStatsCalculator(Metadata metadata) + public ScalarStatsCalculator(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager) + { + this(metadata, (ExpressionOptimizerProvider) expressionOptimizerManager); + } + + public ScalarStatsCalculator(Metadata metadata, ExpressionOptimizerProvider expressionOptimizerProvider) { this.metadata = requireNonNull(metadata, "metadata can not be null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerManager can not be null"); } @Deprecated @@ -126,7 +134,7 @@ public VariableStatsEstimate visitCall(CallExpression call, Void context) return computeArithmeticBinaryStatistics(call, context); } - RowExpression value = new RowExpressionOptimizer(metadata).optimize(call, OPTIMIZED, session); + RowExpression value = expressionOptimizerProvider.getExpressionOptimizer(session).optimize(call, OPTIMIZED, session); if (isNull(value)) { return nullStatsEstimate(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java b/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java index 053f2134412c4..c67d02de2f055 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/expressions/ExpressionOptimizerManager.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.sql.planner.ExpressionOptimizerContext; import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; import com.facebook.presto.sql.relational.FunctionResolution; @@ -46,6 +47,7 @@ import static java.util.Objects.requireNonNull; public class ExpressionOptimizerManager + implements ExpressionOptimizerProvider { private static final File EXPRESSION_MANAGER_CONFIGURATION_DIRECTORY = new File("etc/expression-manager/"); private static final String EXPRESSION_MANAGER_FACTORY_NAME = "expression-manager-factory.name"; @@ -127,6 +129,7 @@ public void addExpressionOptimizerFactory(ExpressionOptimizerFactory expressionO "ExpressionOptimizerFactory %s is already registered", name); } + @Override public ExpressionOptimizer getExpressionOptimizer(ConnectorSession connectorSession) { checkArgument(connectorSession instanceof FullConnectorSession, "connectorSession is not an instance of FullConnectorSession"); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index 9d7ca9d80e1bb..ce1cc1bd4278f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -350,7 +350,7 @@ public PlanOptimizers( estimatedExchangesCostCalculator, new RewriteConstantArrayContainsToInExpression(metadata.getFunctionAndTypeManager()).rules()); - PlanOptimizer predicatePushDown = new StatsRecordingPlanOptimizer(optimizerStats, new PredicatePushDown(metadata, sqlParser, featuresConfig.isNativeExecutionEnabled())); + PlanOptimizer predicatePushDown = new StatsRecordingPlanOptimizer(optimizerStats, new PredicatePushDown(metadata, sqlParser, expressionOptimizerManager, featuresConfig.isNativeExecutionEnabled())); PlanOptimizer prefilterForLimitingAggregation = new StatsRecordingPlanOptimizer(optimizerStats, new PrefilterForLimitingAggregation(metadata, statsCalculator)); builder.add( @@ -731,7 +731,7 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of(new RemoveRedundantIdentityProjections(), new PruneRedundantProjectionAssignments())), - new PushdownSubfields(metadata)); + new PushdownSubfields(metadata, expressionOptimizerManager)); builder.add(predicatePushDown); // Run predicate push down one more time in case we can leverage new information from layouts' effective predicate builder.add(simplifyRowExpressionOptimizer); // Should be always run after PredicatePushDown diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java index 087cab8bcc77c..b5a4edaf0351f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java @@ -45,6 +45,7 @@ import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.parser.SqlParser; @@ -65,7 +66,6 @@ import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableList; @@ -132,13 +132,15 @@ public class PredicatePushDown private final SqlParser sqlParser; private final RowExpressionDomainTranslator rowExpressionDomainTranslator; private final boolean nativeExecution; + private final ExpressionOptimizerProvider expressionOptimizerProvider; - public PredicatePushDown(Metadata metadata, SqlParser sqlParser, boolean nativeExecution) + public PredicatePushDown(Metadata metadata, SqlParser sqlParser, ExpressionOptimizerProvider expressionOptimizerProvider, boolean nativeExecution) { this.metadata = requireNonNull(metadata, "metadata is null"); rowExpressionDomainTranslator = new RowExpressionDomainTranslator(metadata); this.effectivePredicateExtractor = new EffectivePredicateExtractor(rowExpressionDomainTranslator, metadata.getFunctionAndTypeManager()); this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.nativeExecution = nativeExecution; } @@ -150,7 +152,7 @@ public PlanOptimizerResult optimize(PlanNode plan, Session session, TypeProvider requireNonNull(types, "types is null"); requireNonNull(idAllocator, "idAllocator is null"); - Rewriter rewriter = new Rewriter(variableAllocator, idAllocator, metadata, effectivePredicateExtractor, rowExpressionDomainTranslator, sqlParser, session, nativeExecution); + Rewriter rewriter = new Rewriter(variableAllocator, idAllocator, metadata, effectivePredicateExtractor, rowExpressionDomainTranslator, expressionOptimizerProvider, sqlParser, session, nativeExecution); PlanNode rewrittenPlan = SimplePlanRewriter.rewriteWith(rewriter, plan, TRUE_CONSTANT); return PlanOptimizerResult.optimizerResult(rewrittenPlan, rewriter.isPlanChanged()); } @@ -184,6 +186,7 @@ private static class Rewriter private final Metadata metadata; private final EffectivePredicateExtractor effectivePredicateExtractor; private final RowExpressionDomainTranslator rowExpressionDomainTranslator; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final Session session; private final boolean nativeExecution; private final ExpressionEquivalence expressionEquivalence; @@ -199,6 +202,7 @@ private Rewriter( Metadata metadata, EffectivePredicateExtractor effectivePredicateExtractor, RowExpressionDomainTranslator rowExpressionDomainTranslator, + ExpressionOptimizerProvider expressionOptimizerProvider, SqlParser sqlParser, Session session, boolean nativeExecution) @@ -208,6 +212,7 @@ private Rewriter( this.metadata = requireNonNull(metadata, "metadata is null"); this.effectivePredicateExtractor = requireNonNull(effectivePredicateExtractor, "effectivePredicateExtractor is null"); this.rowExpressionDomainTranslator = rowExpressionDomainTranslator; + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.session = requireNonNull(session, "session is null"); this.nativeExecution = nativeExecution; this.expressionEquivalence = new ExpressionEquivalence(metadata, sqlParser); @@ -1159,7 +1164,8 @@ private RowExpression getPostJoinPredicate() } } - private InnerJoinPushDownResult processInnerJoin(RowExpression inheritedPredicate, + private InnerJoinPushDownResult processInnerJoin( + RowExpression inheritedPredicate, RowExpression leftEffectivePredicate, RowExpression rightEffectivePredicate, RowExpression joinPredicate, @@ -1283,6 +1289,7 @@ private InnerJoinPushDownResult processInnerJoin(RowExpression inheritedPredicat joinConjuncts.addAll(allInference.generateEqualitiesPartitionedBy(in(leftVariables)::apply).getScopeStraddlingEqualities()); // scope straddling equalities get dropped in as part of the join predicate return new Rewriter.InnerJoinPushDownResult( + expressionOptimizerProvider, logicalRowExpressions.combineConjuncts(leftPushDownConjuncts.build()), logicalRowExpressions.combineConjuncts(rightPushDownConjuncts.build()), logicalRowExpressions.combineConjuncts(joinConjuncts.build()), TRUE_CONSTANT); @@ -1294,13 +1301,20 @@ private static class InnerJoinPushDownResult private final RowExpression rightPredicate; private final RowExpression joinPredicate; private final RowExpression postJoinPredicate; - - private InnerJoinPushDownResult(RowExpression leftPredicate, RowExpression rightPredicate, RowExpression joinPredicate, RowExpression postJoinPredicate) + private final ExpressionOptimizerProvider expressionOptimizerProvider; + + private InnerJoinPushDownResult( + ExpressionOptimizerProvider expressionOptimizerProvider, + RowExpression leftPredicate, + RowExpression rightPredicate, + RowExpression joinPredicate, + RowExpression postJoinPredicate) { - this.leftPredicate = leftPredicate; - this.rightPredicate = rightPredicate; - this.joinPredicate = joinPredicate; - this.postJoinPredicate = postJoinPredicate; + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); + this.leftPredicate = requireNonNull(leftPredicate, "leftPredicate is null"); + this.rightPredicate = requireNonNull(rightPredicate, "rightPredicate is null"); + this.joinPredicate = requireNonNull(joinPredicate, "joinPredicate is null"); + this.postJoinPredicate = requireNonNull(postJoinPredicate, "postJoinPredicate is null"); } private RowExpression getLeftPredicate() @@ -1425,7 +1439,7 @@ private boolean canConvertOuterToInner(List innerVa // Temporary implementation for joins because the SimplifyExpressions optimizers can not run properly on join clauses private RowExpression simplifyExpression(RowExpression expression) { - return new RowExpressionOptimizer(metadata).optimize(expression, ExpressionOptimizer.Level.SERIALIZABLE, session.toConnectorSession()); + return expressionOptimizerProvider.getExpressionOptimizer(session.toConnectorSession()).optimize(expression, ExpressionOptimizer.Level.SERIALIZABLE, session.toConnectorSession()); } private boolean areExpressionsEquivalent(RowExpression leftExpression, RowExpression rightExpression) @@ -1440,7 +1454,7 @@ private RowExpression nullInputEvaluator(final Collection constantNull(variable.getSourceLocation(), variable.getType())))); - return new RowExpressionOptimizer(metadata).optimize(expression, ExpressionOptimizer.Level.OPTIMIZED, session.toConnectorSession()); + return expressionOptimizerProvider.getExpressionOptimizer(session.toConnectorSession()).optimize(expression, ExpressionOptimizer.Level.OPTIMIZED, session.toConnectorSession()); } private Predicate joinEqualityExpression(final Collection leftVariables) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java index ada7285410e25..fd5d7087c39a9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java @@ -57,6 +57,7 @@ import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; @@ -71,7 +72,6 @@ import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.relational.FunctionResolution; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -111,11 +111,13 @@ public class PushdownSubfields public static final QualifiedObjectName ELEMENT_AT = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "element_at"); public static final QualifiedObjectName CAST = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "$operator$cast"); private final Metadata metadata; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private boolean isEnabledForTesting; - public PushdownSubfields(Metadata metadata) + public PushdownSubfields(Metadata metadata, ExpressionOptimizerProvider expressionOptimizerProvider) { this.metadata = requireNonNull(metadata, "metadata is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); } @Override @@ -141,7 +143,7 @@ public PlanOptimizerResult optimize(PlanNode plan, Session session, TypeProvider return PlanOptimizerResult.optimizerResult(plan, false); } - Rewriter rewriter = new Rewriter(session, metadata); + Rewriter rewriter = new Rewriter(session, metadata, expressionOptimizerProvider); PlanNode rewrittenPlan = SimplePlanRewriter.rewriteWith(rewriter, plan, new Rewriter.Context()); return PlanOptimizerResult.optimizerResult(rewrittenPlan, rewriter.isPlanChanged()); } @@ -157,12 +159,13 @@ private static class Rewriter private static final QualifiedObjectName ARBITRARY_AGGREGATE_FUNCTION = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "arbitrary"); private boolean planChanged; - public Rewriter(Session session, Metadata metadata) + public Rewriter(Session session, Metadata metadata, ExpressionOptimizerProvider expressionOptimizerProvider) { this.session = requireNonNull(session, "session is null"); this.metadata = requireNonNull(metadata, "metadata is null"); + requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.functionResolution = new FunctionResolution(metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver()); - this.expressionOptimizer = new RowExpressionOptimizer(metadata); + this.expressionOptimizer = expressionOptimizerProvider.getExpressionOptimizer(session.toConnectorSession()); this.subfieldExtractor = new SubfieldExtractor( functionResolution, expressionOptimizer, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java index a8ab25fefc51f..ca423293bfb8e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.PredicateCompiler; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; @@ -28,20 +29,20 @@ public final class ConnectorRowExpressionService implements RowExpressionService { private final DomainTranslator domainTranslator; - private final ExpressionOptimizer expressionOptimizer; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final PredicateCompiler predicateCompiler; private final DeterminismEvaluator determinismEvaluator; private final RowExpressionFormatter rowExpressionFormatter; public ConnectorRowExpressionService( DomainTranslator domainTranslator, - ExpressionOptimizer expressionOptimizer, + ExpressionOptimizerProvider expressionOptimizerProvider, PredicateCompiler predicateCompiler, DeterminismEvaluator determinismEvaluator, RowExpressionFormatter rowExpressionFormatter) { this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.predicateCompiler = requireNonNull(predicateCompiler, "predicateCompiler is null"); this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); this.rowExpressionFormatter = requireNonNull(rowExpressionFormatter, "rowExpressionFormatter is null"); @@ -54,9 +55,9 @@ public DomainTranslator getDomainTranslator() } @Override - public ExpressionOptimizer getExpressionOptimizer() + public ExpressionOptimizer getExpressionOptimizer(ConnectorSession session) { - return expressionOptimizer; + return expressionOptimizerProvider.getExpressionOptimizer(session); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 31dba80851dfd..f38172e5e88c4 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -460,7 +460,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, expressionOptimizerManager = new ExpressionOptimizerManager(new PluginNodeManager(nodeManager, nodeInfo.getEnvironment()), getFunctionAndTypeManager()); this.statsNormalizer = new StatsNormalizer(); - this.scalarStatsCalculator = new ScalarStatsCalculator(metadata); + this.scalarStatsCalculator = new ScalarStatsCalculator(metadata, expressionOptimizerManager); this.filterStatsCalculator = new FilterStatsCalculator(metadata, scalarStatsCalculator, statsNormalizer); this.historyBasedPlanStatisticsManager = new HistoryBasedPlanStatisticsManager(objectMapper, createTestingSessionPropertyManager(), metadata, new HistoryBasedOptimizationConfig(), featuresConfig, new NodeVersion("1")); this.fragmentStatsProvider = new FragmentStatsProvider(); @@ -495,6 +495,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, pageSorter, pageIndexerFactory, transactionManager, + expressionOptimizerManager, new RowExpressionDomainTranslator(metadata), new RowExpressionPredicateCompiler(metadata), new RowExpressionDeterminismEvaluator(metadata.getFunctionAndTypeManager()), diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java index 7cef5e6c1cd2f..e7f69f06d1e7f 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java @@ -41,6 +41,7 @@ import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.PredicateCompiler; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; @@ -66,7 +67,8 @@ public class TestingConnectorContext private final DomainTranslator domainTranslator = new RowExpressionDomainTranslator(metadata); private final PredicateCompiler predicateCompiler = new RowExpressionPredicateCompiler(metadata); private final DeterminismEvaluator determinismEvaluator = new RowExpressionDeterminismEvaluator(functionAndTypeManager); - private final FilterStatsCalculatorService filterStatsCalculatorService = new ConnectorFilterStatsCalculatorService(new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer())); + private final ExpressionOptimizerProvider expressionOptimizerProvider = (ConnectorSession session) -> new RowExpressionOptimizer(metadata); + private final FilterStatsCalculatorService filterStatsCalculatorService = new ConnectorFilterStatsCalculatorService(new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata, expressionOptimizerProvider), new StatsNormalizer())); private final BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(); @Override @@ -117,7 +119,7 @@ public DomainTranslator getDomainTranslator() } @Override - public ExpressionOptimizer getExpressionOptimizer() + public ExpressionOptimizer getExpressionOptimizer(ConnectorSession session) { return new RowExpressionOptimizer(metadata); } diff --git a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java index 66c2ceef405fd..865cf0d018e8b 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java @@ -18,6 +18,7 @@ import com.facebook.presto.common.type.VarcharType; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.ComparisonExpression; @@ -82,7 +83,7 @@ public void setUp() throws Exception { MetadataManager metadata = MetadataManager.createTestMetadataManager(); - filterStatsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer()); + filterStatsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata, new InMemoryExpressionOptimizerProvider(metadata)), new StatsNormalizer()); uStats = VariableStatsEstimate.builder() .setAverageRowSize(8.0) diff --git a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java index c8f919326f92d..aef0e39f5629d 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Expression; @@ -146,7 +147,7 @@ public void setUp() .build()); MetadataManager metadata = MetadataManager.createTestMetadataManager(); - statsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer()); + statsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata, new InMemoryExpressionOptimizerProvider(metadata)), new StatsNormalizer()); translator = new TestingRowExpressionTranslator(MetadataManager.createTestMetadataManager()); } diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java b/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java index 584f55140e3a7..8c8a9fe1acc02 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.statistics.DoubleRange; import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.TableStatistics; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Expression; @@ -56,7 +57,12 @@ public void setUp() { session = testSessionBuilder().build(); MetadataManager metadata = MetadataManager.createTestMetadataManager(); - FilterStatsCalculator statsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer()); + FilterStatsCalculator statsCalculator = new FilterStatsCalculator( + metadata, + new ScalarStatsCalculator( + metadata, + new InMemoryExpressionOptimizerProvider(metadata)), + new StatsNormalizer()); statsCalculatorService = new ConnectorFilterStatsCalculatorService(statsCalculator); xStats = ColumnStatistics.builder() .setDistinctValuesCount(Estimate.of(40)) diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java b/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java index 0f642ae517561..5d757b68d1e56 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java @@ -18,8 +18,10 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.JoinType; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -90,8 +92,10 @@ public class TestJoinStatsRule private static final MetadataManager METADATA = createTestMetadataManager(); private static final StatsNormalizer NORMALIZER = new StatsNormalizer(); + + private static final ExpressionOptimizerProvider EXPRESSION_OPTIMIZER_PROVIDER = new InMemoryExpressionOptimizerProvider(METADATA); private static final JoinStatsRule JOIN_STATS_RULE = new JoinStatsRule( - new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA), NORMALIZER), + new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA, EXPRESSION_OPTIMIZER_PROVIDER), NORMALIZER), NORMALIZER, 1.0); diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java index c7951ffbeebb1..afe69f74e8fe7 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.LiteralEncoder; @@ -68,9 +69,10 @@ public class TestScalarStatsCalculator @BeforeClass public void setUp() { - calculator = new ScalarStatsCalculator(MetadataManager.createTestMetadataManager()); + MetadataManager metadata = createTestMetadataManager(); + calculator = new ScalarStatsCalculator(metadata, new InMemoryExpressionOptimizerProvider(metadata)); session = testSessionBuilder().build(); - translator = new TestingRowExpressionTranslator(MetadataManager.createTestMetadataManager()); + translator = new TestingRowExpressionTranslator(metadata); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java b/presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java new file mode 100644 index 0000000000000..95b0d9a09d5b8 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; + +import static java.util.Objects.requireNonNull; + +public class InMemoryExpressionOptimizerProvider + implements ExpressionOptimizerProvider +{ + private final Metadata metadata; + + public InMemoryExpressionOptimizerProvider(Metadata metadata) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + } + + @Override + public ExpressionOptimizer getExpressionOptimizer(ConnectorSession session) + { + return new RowExpressionOptimizer(metadata); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java index df925d425befc..f3aaca8d2d19f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.WindowNode; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; @@ -476,7 +477,7 @@ public void testNoPushdownWithTry() public void testPredicatePushDownCanReduceInnerToCrossJoin() { RuleTester tester = new RuleTester(); - tester.assertThat(new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), false)) + tester.assertThat(new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), new InMemoryExpressionOptimizerProvider(tester.getMetadata()), false)) .on(p -> p.join(INNER, p.filter(p.comparison(OperatorType.EQUAL, p.variable("a1"), constant(1L, INTEGER)), @@ -509,7 +510,7 @@ public void testPredicatePushDownCanReduceInnerToCrossJoin() public void testPredicatePushdownDoesNotAddProjectsBetweenJoinNodes() { RuleTester tester = new RuleTester(); - PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), false); + PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), new InMemoryExpressionOptimizerProvider(tester.getMetadata()), false); tester.assertThat(predicatePushDownOptimizer) .on("SELECT 1 " + @@ -589,7 +590,7 @@ public void testPredicatePushdownDoesNotAddProjectsBetweenJoinNodes() public void testDomainFiltersCanBeInferredForLargeDisjunctiveFilters() { RuleTester tester = new RuleTester(emptyList(), ImmutableMap.of(GENERATE_DOMAIN_FILTERS, "true")); - PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), false); + PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), new InMemoryExpressionOptimizerProvider(tester.getMetadata()), false); // For Inner Join tester.assertThat(predicatePushDownOptimizer) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java index e35960c601fd8..1ea58b5627d57 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java @@ -15,6 +15,7 @@ import com.facebook.presto.common.block.SortOrder; import com.facebook.presto.spi.plan.WindowNode; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.ExpectedValueProvider; @@ -322,7 +323,7 @@ private void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern patter { List optimizers = ImmutableList.of( new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), - new PredicatePushDown(getMetadata(), getQueryRunner().getSqlParser(), false), + new PredicatePushDown(getMetadata(), getQueryRunner().getSqlParser(), new InMemoryExpressionOptimizerProvider(getMetadata()), false), new IterativeOptimizer( getMetadata(), new RuleStatsRecorder(), diff --git a/presto-parquet/src/main/java/com/facebook/presto/parquet/rule/ParquetDereferencePushDown.java b/presto-parquet/src/main/java/com/facebook/presto/parquet/rule/ParquetDereferencePushDown.java index 17f156cc0fff5..92e712df6ebe2 100644 --- a/presto-parquet/src/main/java/com/facebook/presto/parquet/rule/ParquetDereferencePushDown.java +++ b/presto-parquet/src/main/java/com/facebook/presto/parquet/rule/ParquetDereferencePushDown.java @@ -328,7 +328,7 @@ public PlanNode visitProject(ProjectNode project, RewriteContext context) Map dereferenceToNestedColumnMap = extractDereferences( baseColumnHandles, session, - rowExpressionService.getExpressionOptimizer(), + rowExpressionService.getExpressionOptimizer(session), new HashSet<>(project.getAssignments().getExpressions())); if (dereferenceToNestedColumnMap.isEmpty()) { return visitPlan(project, context); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizerProvider.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizerProvider.java new file mode 100644 index 0000000000000..13c8cde41ed4a --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/ExpressionOptimizerProvider.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.relation; + +import com.facebook.presto.spi.ConnectorSession; + +public interface ExpressionOptimizerProvider +{ + ExpressionOptimizer getExpressionOptimizer(ConnectorSession session); +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java b/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java index 12710f71b0f76..88d80abfcb366 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/relation/RowExpressionService.java @@ -19,10 +19,11 @@ * A set of services/utilities that are helpful for connectors to operate on row expressions */ public interface RowExpressionService + extends ExpressionOptimizerProvider { DomainTranslator getDomainTranslator(); - ExpressionOptimizer getExpressionOptimizer(); + ExpressionOptimizer getExpressionOptimizer(ConnectorSession session); PredicateCompiler getPredicateCompiler();