diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java index 3eb3ef011237..511899a2e03f 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcConnector.java @@ -109,7 +109,7 @@ public ConnectorPlanOptimizerProvider getConnectorPlanOptimizerProvider() functionManager, functionResolution, rowExpressionService.getDeterminismEvaluator(), - rowExpressionService.getExpressionOptimizer()); + rowExpressionService); } @Override diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java index e0ad9a3282f3..8d8db43d9765 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcComputePushdown.java @@ -29,7 +29,7 @@ import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.relation.DeterminismEvaluator; -import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.RowExpression; import java.util.Optional; @@ -44,7 +44,7 @@ public class JdbcComputePushdown implements ConnectorPlanOptimizer { - private final ExpressionOptimizer expressionOptimizer; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final JdbcFilterToSqlTranslator jdbcFilterToSqlTranslator; private final LogicalRowExpressions logicalRowExpressions; @@ -52,7 +52,7 @@ public JdbcComputePushdown( FunctionMetadataManager functionMetadataManager, StandardFunctionResolution functionResolution, DeterminismEvaluator determinismEvaluator, - ExpressionOptimizer expressionOptimizer, + ExpressionOptimizerProvider expressionOptimizerProvider, String identifierQuote, Set> functionTranslators) { @@ -62,7 +62,7 @@ public JdbcComputePushdown( requireNonNull(determinismEvaluator, "determinismEvaluator is null"); requireNonNull(functionResolution, "functionResolution is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.jdbcFilterToSqlTranslator = new JdbcFilterToSqlTranslator( functionMetadataManager, buildFunctionTranslator(functionTranslators), @@ -106,7 +106,7 @@ public PlanNode visitFilter(FilterNode node, RewriteContext context) TableHandle oldTableHandle = oldTableScanNode.getTable(); JdbcTableHandle oldConnectorTable = (JdbcTableHandle) oldTableHandle.getConnectorHandle(); - RowExpression predicate = expressionOptimizer.optimize(node.getPredicate(), OPTIMIZED, session); + RowExpression predicate = expressionOptimizerProvider.getExpressionOptimizer(session).optimize(node.getPredicate(), OPTIMIZED, session); predicate = logicalRowExpressions.convertToConjunctiveNormalForm(predicate); TranslatedExpression jdbcExpression = translateWith( predicate, diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java index 9dcd399b6647..5edbfd21eb7f 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/optimization/JdbcPlanOptimizerProvider.java @@ -20,7 +20,7 @@ import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.relation.DeterminismEvaluator; -import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; @@ -34,7 +34,7 @@ public class JdbcPlanOptimizerProvider private final FunctionMetadataManager functionManager; private final StandardFunctionResolution functionResolution; private final DeterminismEvaluator determinismEvaluator; - private final ExpressionOptimizer expressionOptimizer; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final String identifierQuote; @Inject @@ -43,12 +43,12 @@ public JdbcPlanOptimizerProvider( FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution, DeterminismEvaluator determinismEvaluator, - ExpressionOptimizer expressionOptimizer) + ExpressionOptimizerProvider expressionOptimizerProvider) { this.functionManager = requireNonNull(functionManager, "functionManager is null"); this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.identifierQuote = jdbcClient.getIdentifierQuote(); } @@ -65,7 +65,7 @@ public Set getPhysicalPlanOptimizers() functionManager, functionResolution, determinismEvaluator, - expressionOptimizer, + expressionOptimizerProvider, identifierQuote, getFunctionTranslators())); } diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java index 726f89f6631b..34669ff0f1fa 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/optimization/TestJdbcComputePushdown.java @@ -105,7 +105,7 @@ public TestJdbcComputePushdown() functionAndTypeManager, functionResolution, determinismEvaluator, - new RowExpressionOptimizer(METADATA), + (ConnectorSession session) -> new RowExpressionOptimizer(METADATA), "'", getFunctionTranslators()); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java b/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java index cde88e508d81..37b3b35139a2 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/HiveTestUtils.java @@ -151,7 +151,7 @@ public String formatRowExpression(ConnectorSession session, RowExpression expres }; public static final FilterStatsCalculatorService FILTER_STATS_CALCULATOR_SERVICE = new ConnectorFilterStatsCalculatorService( - new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA), new StatsNormalizer())); + new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA, ROW_EXPRESSION_SERVICE), new StatsNormalizer())); public static final HiveClientConfig HIVE_CLIENT_CONFIG = new HiveClientConfig(); public static final MetastoreClientConfig METASTORE_CLIENT_CONFIG = new MetastoreClientConfig(); diff --git a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java index 6ed7888b33d5..56f3e15d72aa 100644 --- a/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -62,12 +62,12 @@ import com.facebook.presto.split.RecordPageSourceProvider; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.planPrinter.RowExpressionFormatter; import com.facebook.presto.sql.relational.ConnectorRowExpressionService; import com.facebook.presto.sql.relational.FunctionResolution; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -116,6 +116,7 @@ public class ConnectorManager private final PageIndexerFactory pageIndexerFactory; private final NodeInfo nodeInfo; private final TransactionManager transactionManager; + private final ExpressionOptimizerManager expressionOptimizerManager; private final DomainTranslator domainTranslator; private final PredicateCompiler predicateCompiler; private final DeterminismEvaluator determinismEvaluator; @@ -151,6 +152,7 @@ public ConnectorManager( PageSorter pageSorter, PageIndexerFactory pageIndexerFactory, TransactionManager transactionManager, + ExpressionOptimizerManager expressionOptimizerManager, DomainTranslator domainTranslator, PredicateCompiler predicateCompiler, DeterminismEvaluator determinismEvaluator, @@ -176,6 +178,7 @@ public ConnectorManager( this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); this.nodeInfo = requireNonNull(nodeInfo, "nodeInfo is null"); this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); this.predicateCompiler = requireNonNull(predicateCompiler, "predicateCompiler is null"); this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); @@ -382,7 +385,7 @@ private Connector createConnector(ConnectorId connectorId, ConnectorFactory fact pageIndexerFactory, new ConnectorRowExpressionService( domainTranslator, - new RowExpressionOptimizer(metadataManager), + expressionOptimizerManager, predicateCompiler, determinismEvaluator, new RowExpressionFormatter(metadataManager.getFunctionAndTypeManager())), diff --git a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java index 4e9c96c31f9b..a25351ca7a88 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/ScalarStatsCalculator.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.InputReferenceExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -78,13 +79,18 @@ public class ScalarStatsCalculator { private final Metadata metadata; - private final ExpressionOptimizerManager expressionOptimizerManager; + private final ExpressionOptimizerProvider expressionOptimizerProvider; @Inject public ScalarStatsCalculator(Metadata metadata, ExpressionOptimizerManager expressionOptimizerManager) + { + this(metadata, (ExpressionOptimizerProvider) expressionOptimizerManager); + } + + public ScalarStatsCalculator(Metadata metadata, ExpressionOptimizerProvider expressionOptimizerProvider) { this.metadata = requireNonNull(metadata, "metadata can not be null"); - this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager can not be null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerManager can not be null"); } @Deprecated @@ -128,7 +134,7 @@ public VariableStatsEstimate visitCall(CallExpression call, Void context) return computeArithmeticBinaryStatistics(call, context); } - RowExpression value = expressionOptimizerManager.getExpressionOptimizer(session).optimize(call, OPTIMIZED, session); + RowExpression value = expressionOptimizerProvider.getExpressionOptimizer(session).optimize(call, OPTIMIZED, session); if (isNull(value)) { return nullStatsEstimate(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index ab4b87ed9ab0..859a2e1d2aa6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -350,7 +350,7 @@ public PlanOptimizers( estimatedExchangesCostCalculator, new RewriteConstantArrayContainsToInExpression(metadata.getFunctionAndTypeManager()).rules()); - PlanOptimizer predicatePushDown = new StatsRecordingPlanOptimizer(optimizerStats, new PredicatePushDown(metadata, sqlParser, featuresConfig.isNativeExecutionEnabled())); + PlanOptimizer predicatePushDown = new StatsRecordingPlanOptimizer(optimizerStats, new PredicatePushDown(metadata, sqlParser, expressionOptimizerManager, featuresConfig.isNativeExecutionEnabled())); PlanOptimizer prefilterForLimitingAggregation = new StatsRecordingPlanOptimizer(optimizerStats, new PrefilterForLimitingAggregation(metadata, statsCalculator)); builder.add( @@ -731,7 +731,7 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of(new RemoveRedundantIdentityProjections(), new PruneRedundantProjectionAssignments())), - new PushdownSubfields(metadata)); + new PushdownSubfields(metadata, expressionOptimizerManager)); builder.add(predicatePushDown); // Run predicate push down one more time in case we can leverage new information from layouts' effective predicate builder.add(simplifyRowExpressionOptimizer); // Should be always run after PredicatePushDown diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java index 087cab8bcc77..b5a4edaf0351 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PredicatePushDown.java @@ -45,6 +45,7 @@ import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.parser.SqlParser; @@ -65,7 +66,6 @@ import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableList; @@ -132,13 +132,15 @@ public class PredicatePushDown private final SqlParser sqlParser; private final RowExpressionDomainTranslator rowExpressionDomainTranslator; private final boolean nativeExecution; + private final ExpressionOptimizerProvider expressionOptimizerProvider; - public PredicatePushDown(Metadata metadata, SqlParser sqlParser, boolean nativeExecution) + public PredicatePushDown(Metadata metadata, SqlParser sqlParser, ExpressionOptimizerProvider expressionOptimizerProvider, boolean nativeExecution) { this.metadata = requireNonNull(metadata, "metadata is null"); rowExpressionDomainTranslator = new RowExpressionDomainTranslator(metadata); this.effectivePredicateExtractor = new EffectivePredicateExtractor(rowExpressionDomainTranslator, metadata.getFunctionAndTypeManager()); this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.nativeExecution = nativeExecution; } @@ -150,7 +152,7 @@ public PlanOptimizerResult optimize(PlanNode plan, Session session, TypeProvider requireNonNull(types, "types is null"); requireNonNull(idAllocator, "idAllocator is null"); - Rewriter rewriter = new Rewriter(variableAllocator, idAllocator, metadata, effectivePredicateExtractor, rowExpressionDomainTranslator, sqlParser, session, nativeExecution); + Rewriter rewriter = new Rewriter(variableAllocator, idAllocator, metadata, effectivePredicateExtractor, rowExpressionDomainTranslator, expressionOptimizerProvider, sqlParser, session, nativeExecution); PlanNode rewrittenPlan = SimplePlanRewriter.rewriteWith(rewriter, plan, TRUE_CONSTANT); return PlanOptimizerResult.optimizerResult(rewrittenPlan, rewriter.isPlanChanged()); } @@ -184,6 +186,7 @@ private static class Rewriter private final Metadata metadata; private final EffectivePredicateExtractor effectivePredicateExtractor; private final RowExpressionDomainTranslator rowExpressionDomainTranslator; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final Session session; private final boolean nativeExecution; private final ExpressionEquivalence expressionEquivalence; @@ -199,6 +202,7 @@ private Rewriter( Metadata metadata, EffectivePredicateExtractor effectivePredicateExtractor, RowExpressionDomainTranslator rowExpressionDomainTranslator, + ExpressionOptimizerProvider expressionOptimizerProvider, SqlParser sqlParser, Session session, boolean nativeExecution) @@ -208,6 +212,7 @@ private Rewriter( this.metadata = requireNonNull(metadata, "metadata is null"); this.effectivePredicateExtractor = requireNonNull(effectivePredicateExtractor, "effectivePredicateExtractor is null"); this.rowExpressionDomainTranslator = rowExpressionDomainTranslator; + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.session = requireNonNull(session, "session is null"); this.nativeExecution = nativeExecution; this.expressionEquivalence = new ExpressionEquivalence(metadata, sqlParser); @@ -1159,7 +1164,8 @@ private RowExpression getPostJoinPredicate() } } - private InnerJoinPushDownResult processInnerJoin(RowExpression inheritedPredicate, + private InnerJoinPushDownResult processInnerJoin( + RowExpression inheritedPredicate, RowExpression leftEffectivePredicate, RowExpression rightEffectivePredicate, RowExpression joinPredicate, @@ -1283,6 +1289,7 @@ private InnerJoinPushDownResult processInnerJoin(RowExpression inheritedPredicat joinConjuncts.addAll(allInference.generateEqualitiesPartitionedBy(in(leftVariables)::apply).getScopeStraddlingEqualities()); // scope straddling equalities get dropped in as part of the join predicate return new Rewriter.InnerJoinPushDownResult( + expressionOptimizerProvider, logicalRowExpressions.combineConjuncts(leftPushDownConjuncts.build()), logicalRowExpressions.combineConjuncts(rightPushDownConjuncts.build()), logicalRowExpressions.combineConjuncts(joinConjuncts.build()), TRUE_CONSTANT); @@ -1294,13 +1301,20 @@ private static class InnerJoinPushDownResult private final RowExpression rightPredicate; private final RowExpression joinPredicate; private final RowExpression postJoinPredicate; - - private InnerJoinPushDownResult(RowExpression leftPredicate, RowExpression rightPredicate, RowExpression joinPredicate, RowExpression postJoinPredicate) + private final ExpressionOptimizerProvider expressionOptimizerProvider; + + private InnerJoinPushDownResult( + ExpressionOptimizerProvider expressionOptimizerProvider, + RowExpression leftPredicate, + RowExpression rightPredicate, + RowExpression joinPredicate, + RowExpression postJoinPredicate) { - this.leftPredicate = leftPredicate; - this.rightPredicate = rightPredicate; - this.joinPredicate = joinPredicate; - this.postJoinPredicate = postJoinPredicate; + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); + this.leftPredicate = requireNonNull(leftPredicate, "leftPredicate is null"); + this.rightPredicate = requireNonNull(rightPredicate, "rightPredicate is null"); + this.joinPredicate = requireNonNull(joinPredicate, "joinPredicate is null"); + this.postJoinPredicate = requireNonNull(postJoinPredicate, "postJoinPredicate is null"); } private RowExpression getLeftPredicate() @@ -1425,7 +1439,7 @@ private boolean canConvertOuterToInner(List innerVa // Temporary implementation for joins because the SimplifyExpressions optimizers can not run properly on join clauses private RowExpression simplifyExpression(RowExpression expression) { - return new RowExpressionOptimizer(metadata).optimize(expression, ExpressionOptimizer.Level.SERIALIZABLE, session.toConnectorSession()); + return expressionOptimizerProvider.getExpressionOptimizer(session.toConnectorSession()).optimize(expression, ExpressionOptimizer.Level.SERIALIZABLE, session.toConnectorSession()); } private boolean areExpressionsEquivalent(RowExpression leftExpression, RowExpression rightExpression) @@ -1440,7 +1454,7 @@ private RowExpression nullInputEvaluator(final Collection constantNull(variable.getSourceLocation(), variable.getType())))); - return new RowExpressionOptimizer(metadata).optimize(expression, ExpressionOptimizer.Level.OPTIMIZED, session.toConnectorSession()); + return expressionOptimizerProvider.getExpressionOptimizer(session.toConnectorSession()).optimize(expression, ExpressionOptimizer.Level.OPTIMIZED, session.toConnectorSession()); } private Predicate joinEqualityExpression(final Collection leftVariables) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java index ada7285410e2..fd5d7087c39a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java @@ -57,6 +57,7 @@ import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; @@ -71,7 +72,6 @@ import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.relational.FunctionResolution; -import com.facebook.presto.sql.relational.RowExpressionOptimizer; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -111,11 +111,13 @@ public class PushdownSubfields public static final QualifiedObjectName ELEMENT_AT = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "element_at"); public static final QualifiedObjectName CAST = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "$operator$cast"); private final Metadata metadata; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private boolean isEnabledForTesting; - public PushdownSubfields(Metadata metadata) + public PushdownSubfields(Metadata metadata, ExpressionOptimizerProvider expressionOptimizerProvider) { this.metadata = requireNonNull(metadata, "metadata is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); } @Override @@ -141,7 +143,7 @@ public PlanOptimizerResult optimize(PlanNode plan, Session session, TypeProvider return PlanOptimizerResult.optimizerResult(plan, false); } - Rewriter rewriter = new Rewriter(session, metadata); + Rewriter rewriter = new Rewriter(session, metadata, expressionOptimizerProvider); PlanNode rewrittenPlan = SimplePlanRewriter.rewriteWith(rewriter, plan, new Rewriter.Context()); return PlanOptimizerResult.optimizerResult(rewrittenPlan, rewriter.isPlanChanged()); } @@ -157,12 +159,13 @@ private static class Rewriter private static final QualifiedObjectName ARBITRARY_AGGREGATE_FUNCTION = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "arbitrary"); private boolean planChanged; - public Rewriter(Session session, Metadata metadata) + public Rewriter(Session session, Metadata metadata, ExpressionOptimizerProvider expressionOptimizerProvider) { this.session = requireNonNull(session, "session is null"); this.metadata = requireNonNull(metadata, "metadata is null"); + requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.functionResolution = new FunctionResolution(metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver()); - this.expressionOptimizer = new RowExpressionOptimizer(metadata); + this.expressionOptimizer = expressionOptimizerProvider.getExpressionOptimizer(session.toConnectorSession()); this.subfieldExtractor = new SubfieldExtractor( functionResolution, expressionOptimizer, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java index 0679efdea3c9..ca423293bfb8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/ConnectorRowExpressionService.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.PredicateCompiler; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; @@ -28,20 +29,20 @@ public final class ConnectorRowExpressionService implements RowExpressionService { private final DomainTranslator domainTranslator; - private final ExpressionOptimizer expressionOptimizer; + private final ExpressionOptimizerProvider expressionOptimizerProvider; private final PredicateCompiler predicateCompiler; private final DeterminismEvaluator determinismEvaluator; private final RowExpressionFormatter rowExpressionFormatter; public ConnectorRowExpressionService( DomainTranslator domainTranslator, - ExpressionOptimizer expressionOptimizer, + ExpressionOptimizerProvider expressionOptimizerProvider, PredicateCompiler predicateCompiler, DeterminismEvaluator determinismEvaluator, RowExpressionFormatter rowExpressionFormatter) { this.domainTranslator = requireNonNull(domainTranslator, "domainTranslator is null"); - this.expressionOptimizer = requireNonNull(expressionOptimizer, "expressionOptimizer is null"); + this.expressionOptimizerProvider = requireNonNull(expressionOptimizerProvider, "expressionOptimizerProvider is null"); this.predicateCompiler = requireNonNull(predicateCompiler, "predicateCompiler is null"); this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); this.rowExpressionFormatter = requireNonNull(rowExpressionFormatter, "rowExpressionFormatter is null"); @@ -56,7 +57,7 @@ public DomainTranslator getDomainTranslator() @Override public ExpressionOptimizer getExpressionOptimizer(ConnectorSession session) { - return expressionOptimizer; + return expressionOptimizerProvider.getExpressionOptimizer(session); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 75f11537a99b..f38172e5e88c 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -495,6 +495,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, pageSorter, pageIndexerFactory, transactionManager, + expressionOptimizerManager, new RowExpressionDomainTranslator(metadata), new RowExpressionPredicateCompiler(metadata), new RowExpressionDeterminismEvaluator(metadata.getFunctionAndTypeManager()), diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java index 7cef5e6c1cd2..e7f69f06d1e7 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorContext.java @@ -41,6 +41,7 @@ import com.facebook.presto.spi.relation.DeterminismEvaluator; import com.facebook.presto.spi.relation.DomainTranslator; import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.PredicateCompiler; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.RowExpressionService; @@ -66,7 +67,8 @@ public class TestingConnectorContext private final DomainTranslator domainTranslator = new RowExpressionDomainTranslator(metadata); private final PredicateCompiler predicateCompiler = new RowExpressionPredicateCompiler(metadata); private final DeterminismEvaluator determinismEvaluator = new RowExpressionDeterminismEvaluator(functionAndTypeManager); - private final FilterStatsCalculatorService filterStatsCalculatorService = new ConnectorFilterStatsCalculatorService(new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer())); + private final ExpressionOptimizerProvider expressionOptimizerProvider = (ConnectorSession session) -> new RowExpressionOptimizer(metadata); + private final FilterStatsCalculatorService filterStatsCalculatorService = new ConnectorFilterStatsCalculatorService(new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata, expressionOptimizerProvider), new StatsNormalizer())); private final BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager(); @Override @@ -117,7 +119,7 @@ public DomainTranslator getDomainTranslator() } @Override - public ExpressionOptimizer getExpressionOptimizer() + public ExpressionOptimizer getExpressionOptimizer(ConnectorSession session) { return new RowExpressionOptimizer(metadata); } diff --git a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java index 66c2ceef405f..865cf0d018e8 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestComparisonStatsCalculator.java @@ -18,6 +18,7 @@ import com.facebook.presto.common.type.VarcharType; import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Cast; import com.facebook.presto.sql.tree.ComparisonExpression; @@ -82,7 +83,7 @@ public void setUp() throws Exception { MetadataManager metadata = MetadataManager.createTestMetadataManager(); - filterStatsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer()); + filterStatsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata, new InMemoryExpressionOptimizerProvider(metadata)), new StatsNormalizer()); uStats = VariableStatsEstimate.builder() .setAverageRowSize(8.0) diff --git a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java index c8f919326f92..aef0e39f5629 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Expression; @@ -146,7 +147,7 @@ public void setUp() .build()); MetadataManager metadata = MetadataManager.createTestMetadataManager(); - statsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer()); + statsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata, new InMemoryExpressionOptimizerProvider(metadata)), new StatsNormalizer()); translator = new TestingRowExpressionTranslator(MetadataManager.createTestMetadataManager()); } diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java b/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java index 584f55140e3a..8c8a9fe1acc0 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestConnectorFilterStatsCalculatorService.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.statistics.DoubleRange; import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.TableStatistics; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.tree.Expression; @@ -56,7 +57,12 @@ public void setUp() { session = testSessionBuilder().build(); MetadataManager metadata = MetadataManager.createTestMetadataManager(); - FilterStatsCalculator statsCalculator = new FilterStatsCalculator(metadata, new ScalarStatsCalculator(metadata), new StatsNormalizer()); + FilterStatsCalculator statsCalculator = new FilterStatsCalculator( + metadata, + new ScalarStatsCalculator( + metadata, + new InMemoryExpressionOptimizerProvider(metadata)), + new StatsNormalizer()); statsCalculatorService = new ConnectorFilterStatsCalculatorService(statsCalculator); xStats = ColumnStatistics.builder() .setDistinctValuesCount(Estimate.of(40)) diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java b/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java index 0f642ae51756..5d757b68d1e5 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestJoinStatsRule.java @@ -18,8 +18,10 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.JoinType; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.google.common.collect.ImmutableList; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -90,8 +92,10 @@ public class TestJoinStatsRule private static final MetadataManager METADATA = createTestMetadataManager(); private static final StatsNormalizer NORMALIZER = new StatsNormalizer(); + + private static final ExpressionOptimizerProvider EXPRESSION_OPTIMIZER_PROVIDER = new InMemoryExpressionOptimizerProvider(METADATA); private static final JoinStatsRule JOIN_STATS_RULE = new JoinStatsRule( - new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA), NORMALIZER), + new FilterStatsCalculator(METADATA, new ScalarStatsCalculator(METADATA, EXPRESSION_OPTIMIZER_PROVIDER), NORMALIZER), NORMALIZER, 1.0); diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java b/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java index c7951ffbeebb..afe69f74e8fe 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.TestingRowExpressionTranslator; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.LiteralEncoder; @@ -68,9 +69,10 @@ public class TestScalarStatsCalculator @BeforeClass public void setUp() { - calculator = new ScalarStatsCalculator(MetadataManager.createTestMetadataManager()); + MetadataManager metadata = createTestMetadataManager(); + calculator = new ScalarStatsCalculator(metadata, new InMemoryExpressionOptimizerProvider(metadata)); session = testSessionBuilder().build(); - translator = new TestingRowExpressionTranslator(MetadataManager.createTestMetadataManager()); + translator = new TestingRowExpressionTranslator(metadata); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java b/presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java new file mode 100644 index 000000000000..95b0d9a09d5b --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/InMemoryExpressionOptimizerProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.ExpressionOptimizerProvider; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; + +import static java.util.Objects.requireNonNull; + +public class InMemoryExpressionOptimizerProvider + implements ExpressionOptimizerProvider +{ + private final Metadata metadata; + + public InMemoryExpressionOptimizerProvider(Metadata metadata) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + } + + @Override + public ExpressionOptimizer getExpressionOptimizer(ConnectorSession session) + { + return new RowExpressionOptimizer(metadata); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java index df925d425bef..f3aaca8d2d19 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestPredicatePushdown.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.WindowNode; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; @@ -476,7 +477,7 @@ public void testNoPushdownWithTry() public void testPredicatePushDownCanReduceInnerToCrossJoin() { RuleTester tester = new RuleTester(); - tester.assertThat(new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), false)) + tester.assertThat(new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), new InMemoryExpressionOptimizerProvider(tester.getMetadata()), false)) .on(p -> p.join(INNER, p.filter(p.comparison(OperatorType.EQUAL, p.variable("a1"), constant(1L, INTEGER)), @@ -509,7 +510,7 @@ public void testPredicatePushDownCanReduceInnerToCrossJoin() public void testPredicatePushdownDoesNotAddProjectsBetweenJoinNodes() { RuleTester tester = new RuleTester(); - PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), false); + PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), new InMemoryExpressionOptimizerProvider(tester.getMetadata()), false); tester.assertThat(predicatePushDownOptimizer) .on("SELECT 1 " + @@ -589,7 +590,7 @@ public void testPredicatePushdownDoesNotAddProjectsBetweenJoinNodes() public void testDomainFiltersCanBeInferredForLargeDisjunctiveFilters() { RuleTester tester = new RuleTester(emptyList(), ImmutableMap.of(GENERATE_DOMAIN_FILTERS, "true")); - PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), false); + PredicatePushDown predicatePushDownOptimizer = new PredicatePushDown(tester.getMetadata(), tester.getSqlParser(), new InMemoryExpressionOptimizerProvider(tester.getMetadata()), false); // For Inner Join tester.assertThat(predicatePushDownOptimizer) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java index e35960c601fd..1ea58b5627d5 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReorderWindows.java @@ -15,6 +15,7 @@ import com.facebook.presto.common.block.SortOrder; import com.facebook.presto.spi.plan.WindowNode; +import com.facebook.presto.sql.InMemoryExpressionOptimizerProvider; import com.facebook.presto.sql.planner.RuleStatsRecorder; import com.facebook.presto.sql.planner.assertions.BasePlanTest; import com.facebook.presto.sql.planner.assertions.ExpectedValueProvider; @@ -322,7 +323,7 @@ private void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern patter { List optimizers = ImmutableList.of( new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), - new PredicatePushDown(getMetadata(), getQueryRunner().getSqlParser(), false), + new PredicatePushDown(getMetadata(), getQueryRunner().getSqlParser(), new InMemoryExpressionOptimizerProvider(getMetadata()), false), new IterativeOptimizer( getMetadata(), new RuleStatsRecorder(),