Skip to content

Commit

Permalink
Add DelegatingRowExpressionOptimizer
Browse files Browse the repository at this point in the history
  • Loading branch information
tdcmeehan committed Dec 11, 2024
1 parent a446157 commit 5a6ccb9
Show file tree
Hide file tree
Showing 39 changed files with 1,366 additions and 290 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,7 @@ public final class SystemSessionProperties
public static final String WARN_ON_COMMON_NAN_PATTERNS = "warn_on_common_nan_patterns";
public static final String INLINE_PROJECTIONS_ON_VALUES = "inline_projections_on_values";
public static final String INCLUDE_VALUES_NODE_IN_CONNECTOR_OPTIMIZER = "include_values_node_in_connector_optimizer";
public static final String DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED = "delegating_row_expression_optimizer_enabled";

// TODO: Native execution related session properties that are temporarily put here. They will be relocated in the future.
public static final String NATIVE_AGGREGATION_SPILL_ALL = "native_aggregation_spill_all";
Expand Down Expand Up @@ -1829,6 +1830,11 @@ public SystemSessionProperties(
NATIVE_MIN_COLUMNAR_ENCODING_CHANNELS_TO_PREFER_ROW_WISE_ENCODING,
"Minimum number of columnar encoding channels to consider row wise encoding for partitioned exchange. Native execution only",
queryManagerConfig.getMinColumnarEncodingChannelsToPreferRowWiseEncoding(),
false),
booleanProperty(
DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED,
"Enable delegating row optimizer",
featuresConfig.isDelegatingRowExpressionOptimizerEnabled(),
false));
}

Expand Down Expand Up @@ -3110,4 +3116,9 @@ public static int getMinColumnarEncodingChannelsToPreferRowWiseEncoding(Session
{
return session.getSystemProperty(NATIVE_MIN_COLUMNAR_ENCODING_CHANNELS_TO_PREFER_ROW_WISE_ENCODING, Integer.class);
}

public static boolean isDelegatingRowExpressionOptimizerEnabled(Session session)
{
return session.getSystemProperty(DELEGATING_ROW_EXPRESSION_OPTIMIZER_ENABLED, Boolean.class);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@
import com.facebook.presto.spi.relation.VariableReferenceExpression;
import com.facebook.presto.sql.analyzer.ExpressionAnalyzer;
import com.facebook.presto.sql.analyzer.Scope;
import com.facebook.presto.sql.expressions.ExpressionOptimizerManager;
import com.facebook.presto.sql.planner.ExpressionInterpreter;
import com.facebook.presto.sql.planner.NoOpVariableResolver;
import com.facebook.presto.sql.planner.TypeProvider;
import com.facebook.presto.sql.relational.FunctionResolution;
import com.facebook.presto.sql.relational.RowExpressionOptimizer;
import com.facebook.presto.sql.tree.ArithmeticBinaryExpression;
import com.facebook.presto.sql.tree.ArithmeticUnaryExpression;
import com.facebook.presto.sql.tree.AstVisitor;
Expand Down Expand Up @@ -78,11 +78,13 @@
public class ScalarStatsCalculator
{
private final Metadata metadata;
private final ExpressionOptimizerManager expressionOptimizerManager;

@Inject
public ScalarStatsCalculator(Metadata metadata)
public ScalarStatsCalculator(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager)
{
this.metadata = requireNonNull(metadata, "metadata can not be null");
this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager can not be null");
}

@Deprecated
Expand Down Expand Up @@ -126,7 +128,7 @@ public VariableStatsEstimate visitCall(CallExpression call, Void context)
return computeArithmeticBinaryStatistics(call, context);
}

RowExpression value = new RowExpressionOptimizer(metadata).optimize(call, OPTIMIZED, session);
RowExpression value = expressionOptimizerManager.getExpressionOptimizer().optimize(call, OPTIMIZED, session);

if (isNull(value)) {
return nullStatsEstimate();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,15 @@
import com.facebook.presto.spi.security.SystemAccessControlFactory;
import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory;
import com.facebook.presto.spi.session.WorkerSessionPropertyProviderFactory;
import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory;
import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider;
import com.facebook.presto.spi.storage.TempStorageFactory;
import com.facebook.presto.spi.tracing.TracerProvider;
import com.facebook.presto.spi.ttl.ClusterTtlProviderFactory;
import com.facebook.presto.spi.ttl.NodeTtlFetcherFactory;
import com.facebook.presto.sql.analyzer.AnalyzerProviderManager;
import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager;
import com.facebook.presto.sql.expressions.ExpressionOptimizerManager;
import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager;
import com.facebook.presto.storage.TempStorageManager;
import com.facebook.presto.tracing.TracerProviderManager;
Expand Down Expand Up @@ -135,6 +137,7 @@ public class PluginManager
private final QueryPreparerProviderManager queryPreparerProviderManager;
private final NodeStatusNotificationManager nodeStatusNotificationManager;
private final PlanCheckerProviderManager planCheckerProviderManager;
private final ExpressionOptimizerManager expressionOptimizerManager;

@Inject
public PluginManager(
Expand All @@ -157,7 +160,8 @@ public PluginManager(
HistoryBasedPlanStatisticsManager historyBasedPlanStatisticsManager,
TracerProviderManager tracerProviderManager,
NodeStatusNotificationManager nodeStatusNotificationManager,
PlanCheckerProviderManager planCheckerProviderManager)
PlanCheckerProviderManager planCheckerProviderManager,
ExpressionOptimizerManager expressionOptimizerManager)
{
requireNonNull(nodeInfo, "nodeInfo is null");
requireNonNull(config, "config is null");
Expand Down Expand Up @@ -190,6 +194,7 @@ public PluginManager(
this.queryPreparerProviderManager = requireNonNull(queryPreparerProviderManager, "queryPreparerProviderManager is null");
this.nodeStatusNotificationManager = requireNonNull(nodeStatusNotificationManager, "nodeStatusNotificationManager is null");
this.planCheckerProviderManager = requireNonNull(planCheckerProviderManager, "planCheckerProviderManager is null");
this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionManager is null");
}

public void loadPlugins()
Expand Down Expand Up @@ -372,6 +377,11 @@ public void installCoordinatorPlugin(CoordinatorPlugin plugin)
log.info("Registering plan checker provider factory %s", planCheckerProviderFactory.getName());
planCheckerProviderManager.addPlanCheckerProviderFactory(planCheckerProviderFactory);
}

for (ExpressionOptimizerFactory expressionOptimizerFactory : plugin.getExpressionOptimizerFactories()) {
log.info("Registering expression optimizer factory %s", expressionOptimizerFactory.getName());
expressionOptimizerManager.addExpressionOptimizerFactory(expressionOptimizerFactory);
}
}

private URLClassLoader buildClassLoader(String plugin)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import com.facebook.presto.server.security.PasswordAuthenticatorManager;
import com.facebook.presto.server.security.ServerSecurityModule;
import com.facebook.presto.sql.analyzer.FeaturesConfig;
import com.facebook.presto.sql.expressions.ExpressionOptimizerManager;
import com.facebook.presto.sql.parser.SqlParserOptions;
import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager;
import com.facebook.presto.storage.TempStorageManager;
Expand Down Expand Up @@ -190,6 +191,8 @@ public void run()
PluginNodeManager pluginNodeManager = new PluginNodeManager(nodeManager, nodeInfo.getEnvironment());
planCheckerProviderManager.loadPlanCheckerProviders(pluginNodeManager);

injector.getInstance(ExpressionOptimizerManager.class).loadExpressionOptimizerFactory();

startAssociatedProcesses(injector);

injector.getInstance(Announcer.class).start();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@
import com.facebook.presto.sql.analyzer.MetadataExtractorMBean;
import com.facebook.presto.sql.analyzer.QueryExplainer;
import com.facebook.presto.sql.analyzer.QueryPreparerProviderManager;
import com.facebook.presto.sql.expressions.ExpressionOptimizerManager;
import com.facebook.presto.sql.gen.ExpressionCompiler;
import com.facebook.presto.sql.gen.JoinCompiler;
import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler;
Expand Down Expand Up @@ -359,6 +360,9 @@ else if (serverConfig.isCoordinator()) {
binder.bind(SystemSessionProperties.class).in(Scopes.SINGLETON);
binder.bind(SessionPropertyDefaults.class).in(Scopes.SINGLETON);

// expression manager
binder.bind(ExpressionOptimizerManager.class).in(Scopes.SINGLETON);

// schema properties
binder.bind(SchemaPropertyManager.class).in(Scopes.SINGLETON);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
import com.facebook.presto.split.PageSourceManager;
import com.facebook.presto.split.SplitManager;
import com.facebook.presto.sql.analyzer.FeaturesConfig;
import com.facebook.presto.sql.expressions.ExpressionOptimizerManager;
import com.facebook.presto.sql.parser.SqlParser;
import com.facebook.presto.sql.parser.SqlParserOptions;
import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager;
Expand Down Expand Up @@ -168,6 +169,7 @@ public class TestingPrestoServer
private final TaskManager taskManager;
private final GracefulShutdownHandler gracefulShutdownHandler;
private final ShutdownAction shutdownAction;
private final ExpressionOptimizerManager expressionManager;
private final RequestBlocker requestBlocker;
private final boolean resourceManager;
private final boolean catalogServer;
Expand Down Expand Up @@ -368,6 +370,7 @@ public TestingPrestoServer(
procedureTester = injector.getInstance(ProcedureTester.class);
splitManager = injector.getInstance(SplitManager.class);
pageSourceManager = injector.getInstance(PageSourceManager.class);
expressionManager = injector.getInstance(ExpressionOptimizerManager.class);
if (coordinator) {
dispatchManager = injector.getInstance(DispatchManager.class);
queryManager = injector.getInstance(QueryManager.class);
Expand All @@ -385,6 +388,7 @@ public TestingPrestoServer(
eventListenerManager = ((TestingEventListenerManager) injector.getInstance(EventListenerManager.class));
clusterStateProvider = null;
planCheckerProviderManager = injector.getInstance(PlanCheckerProviderManager.class);
expressionManager.loadExpressionOptimizerFactory();
}
else if (resourceManager) {
dispatchManager = null;
Expand Down Expand Up @@ -704,6 +708,11 @@ public ShutdownAction getShutdownAction()
return shutdownAction;
}

public ExpressionOptimizerManager getExpressionManager()
{
return expressionManager;
}

public boolean isCoordinator()
{
return coordinator;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,8 @@ public class FeaturesConfig
private boolean generateDomainFilters;
private boolean printEstimatedStatsFromCache;
private boolean removeCrossJoinWithSingleConstantRow = true;
private boolean delegatingRowOptimizerEnabled;
private int delegatingRowOptimizerMaxIterations = 10;
private CreateView.Security defaultViewSecurityMode = DEFINER;
private boolean useHistograms;

Expand Down Expand Up @@ -2861,4 +2863,31 @@ public FeaturesConfig setPrestoSparkExecutionEnvironment(boolean prestoSparkExec
this.prestoSparkExecutionEnvironment = prestoSparkExecutionEnvironment;
return this;
}

public boolean isDelegatingRowExpressionOptimizerEnabled()
{
return delegatingRowOptimizerEnabled;
}

@Config("optimizer.delegating-row-expression-optimizer-enabled")
@ConfigDescription("Enable delegating row optimizer")
public FeaturesConfig setDelegatingRowExpressionOptimizerEnabled(boolean delegatingRowOptimizerEnabled)
{
this.delegatingRowOptimizerEnabled = delegatingRowOptimizerEnabled;
return this;
}

@Min(1)
public int getDelegatingRowExpressionOptimizerMaxIterations()
{
return delegatingRowOptimizerMaxIterations;
}

@Config("optimizer.delegating-row-expression-optimizer-max-iterations")
@ConfigDescription("Maximum number of iterations for delegating row optimizer")
public FeaturesConfig setDelegatingRowExpressionOptimizerMaxIterations(int delegatingRowOptimizerMaxIterations)
{
this.delegatingRowOptimizerMaxIterations = delegatingRowOptimizerMaxIterations;
return this;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.sql.expressions;

import com.facebook.presto.metadata.FunctionAndTypeManager;
import com.facebook.presto.nodeManager.PluginNodeManager;
import com.facebook.presto.spi.NodeManager;
import com.facebook.presto.spi.relation.ExpressionOptimizer;
import com.facebook.presto.spi.relation.ExpressionOptimizerProvider;
import com.facebook.presto.spi.sql.planner.ExpressionOptimizerContext;
import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory;
import com.facebook.presto.sql.relational.FunctionResolution;
import com.facebook.presto.sql.relational.RowExpressionOptimizer;

import javax.inject.Inject;

import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;

import static com.facebook.presto.util.PropertiesUtil.loadProperties;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Strings.isNullOrEmpty;
import static java.util.Objects.requireNonNull;

public class ExpressionOptimizerManager
implements ExpressionOptimizerProvider
{
private static final File EXPRESSION_MANAGER_CONFIGURATION = new File("etc/expression-manager.properties");
public static final String EXPRESSION_MANAGER_FACTORY_NAME = "expression-manager-factory.name";

private final Map<String, ExpressionOptimizerFactory> expressionOptimizerFactories = new ConcurrentHashMap<>();
private final AtomicReference<ExpressionOptimizer> rowExpressionInterpreter = new AtomicReference<>();
private final NodeManager nodeManager;
private final FunctionAndTypeManager functionAndTypeManager;
private final FunctionResolution functionResolution;
private final ExpressionOptimizer defaultExpressionOptimizer;

@Inject
public ExpressionOptimizerManager(PluginNodeManager nodeManager, FunctionAndTypeManager functionAndTypeManager)
{
requireNonNull(nodeManager, "nodeManager is null");
this.nodeManager = requireNonNull(nodeManager, "nodeManager is null");
this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null");
this.functionResolution = new FunctionResolution(functionAndTypeManager.getFunctionAndTypeResolver());
this.defaultExpressionOptimizer = new RowExpressionOptimizer(functionAndTypeManager);
rowExpressionInterpreter.set(defaultExpressionOptimizer);
}

public void loadExpressionOptimizerFactory()
{
try {
if (EXPRESSION_MANAGER_CONFIGURATION.exists()) {
Map<String, String> properties = new HashMap<>(loadProperties(EXPRESSION_MANAGER_CONFIGURATION));
loadExpressionOptimizerFactory(properties);
}
}
catch (IOException e) {
throw new UncheckedIOException("Failed to load expression manager configuration", e);
}
}

private void loadExpressionOptimizerFactory(Map<String, String> properties)
{
properties = new HashMap<>(properties);
String factoryName = properties.remove(EXPRESSION_MANAGER_FACTORY_NAME);
checkArgument(!isNullOrEmpty(factoryName), "%s does not contain %s", EXPRESSION_MANAGER_CONFIGURATION, EXPRESSION_MANAGER_FACTORY_NAME);
checkArgument(
rowExpressionInterpreter.compareAndSet(
defaultExpressionOptimizer,
expressionOptimizerFactories.get(factoryName).createOptimizer(properties, new ExpressionOptimizerContext(nodeManager, functionAndTypeManager, functionResolution))),
"ExpressionManager is already loaded");
}

public void addExpressionOptimizerFactory(ExpressionOptimizerFactory expressionOptimizerFactory)
{
String name = expressionOptimizerFactory.getName();
checkArgument(
this.expressionOptimizerFactories.putIfAbsent(name, expressionOptimizerFactory) == null,
"ExpressionOptimizerFactory %s is already registered", name);
}

@Override
public ExpressionOptimizer getExpressionOptimizer()
{
return rowExpressionInterpreter.get();
}
}
Loading

0 comments on commit 5a6ccb9

Please sign in to comment.