forked from apache/doris
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[feature](mtmv) Support agg state roll up and optimize the roll up co…
…de (apache#35026) agg_state is agg intermediate state, detail see state combinator: https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/combinators/state this support agg function roll up as following +---------------------+---------------------------------------------+---------------------+ | query | materialized view | roll up | | ------------------- | ------------------------------------------- | ------------------- | | agg_funtion() | agg_funtion_unoin() or agg_funtion_state() | agg_funtion_merge() | | agg_funtion_unoin() | agg_funtion_unoin() or agg_funtion_state() | agg_funtion_union() | | agg_funtion_merge() | agg_funtion_unoin() or agg_funtion_state() | agg_funtion_merge() | +---------------------+---------------------------------------------+---------------------+ for example which can be rewritten by mv sucessfully as following MV defination is ``` select o_orderstatus, l_partkey, l_suppkey, sum_union(sum_state(o_shippriority)), group_concat_union(group_concat_state(l_shipinstruct)), avg_union(avg_state(l_linenumber)), max_by_union(max_by_state(l_shipmode, l_suppkey)), count_union(count_state(l_orderkey)), multi_distinct_count_union(multi_distinct_count_state(l_shipmode)) from lineitem left join orders on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate group by o_orderstatus, l_partkey, l_suppkey; ``` Query is ``` select o_orderstatus, l_suppkey, sum(o_shippriority), group_concat(l_shipinstruct), avg(l_linenumber), max_by(l_shipmode,l_suppkey), count(l_orderkey), multi_distinct_count(l_shipmode) from lineitem left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate group by o_orderstatus, l_suppkey; ```
- Loading branch information
Showing
28 changed files
with
1,350 additions
and
244 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
78 changes: 78 additions & 0 deletions
78
...n/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| // Licensed to the Apache Software Foundation (ASF) under one | ||
| // or more contributor license agreements. See the NOTICE file | ||
| // distributed with this work for additional information | ||
| // regarding copyright ownership. The ASF licenses this file | ||
| // to you under the Apache License, Version 2.0 (the | ||
| // "License"); you may not use this file except in compliance | ||
| // with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| package org.apache.doris.nereids.rules.exploration.mv.rollup; | ||
|
|
||
| import org.apache.doris.common.Pair; | ||
| import org.apache.doris.nereids.trees.expressions.Any; | ||
| import org.apache.doris.nereids.trees.expressions.Expression; | ||
| import org.apache.doris.nereids.trees.expressions.functions.Function; | ||
| import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; | ||
| import org.apache.doris.nereids.trees.expressions.functions.agg.RollUpTrait; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * Aggregate function roll up handler | ||
| */ | ||
| public abstract class AggFunctionRollUpHandler { | ||
|
|
||
| /** | ||
| * Decide the query and view function can roll up or not | ||
| */ | ||
| public boolean canRollup(AggregateFunction queryAggregateFunction, | ||
| Expression queryAggregateFunctionShuttled, | ||
| Pair<Expression, Expression> mvExprToMvScanExprQueryBasedPair) { | ||
| Expression viewExpression = mvExprToMvScanExprQueryBasedPair.key(); | ||
| if (!(viewExpression instanceof RollUpTrait) || !((RollUpTrait) viewExpression).canRollUp()) { | ||
| return false; | ||
| } | ||
| AggregateFunction aggregateFunction = (AggregateFunction) viewExpression; | ||
| return !aggregateFunction.isDistinct(); | ||
| } | ||
|
|
||
| /** | ||
| * Do the aggregate function roll up | ||
| */ | ||
| public abstract Function doRollup( | ||
| AggregateFunction queryAggregateFunction, | ||
| Expression queryAggregateFunctionShuttled, | ||
| Pair<Expression, Expression> mvExprToMvScanExprQueryBasedPair); | ||
|
|
||
| /** | ||
| * Extract the function arguments by functionWithAny pattern | ||
| * Such as functionWithAny def is bitmap_union(to_bitmap(Any.INSTANCE)), | ||
| * actualFunction is bitmap_union(to_bitmap(case when a = 5 then 1 else 2 end)) | ||
| * after extracting, the return argument is: case when a = 5 then 1 else 2 end | ||
| */ | ||
| protected static List<Expression> extractArguments(Expression functionWithAny, Function actualFunction) { | ||
| Set<Object> exprSetToRemove = functionWithAny.collectToSet(expr -> !(expr instanceof Any)); | ||
| return actualFunction.collectFirst(expr -> | ||
| exprSetToRemove.stream().noneMatch(exprToRemove -> exprToRemove.equals(expr))); | ||
| } | ||
|
|
||
| /** | ||
| * Extract the target expression in actualFunction by targetClazz | ||
| * Such as actualFunction def is avg_merge(avg_union(c1)), target Clazz is Combinator | ||
| * after extracting, the return argument is avg_union(c1) | ||
| */ | ||
| protected static <T> T extractLastExpression(Expression actualFunction, Class<T> targetClazz) { | ||
| List<Expression> expressions = actualFunction.collectToList(targetClazz::isInstance); | ||
| return targetClazz.cast(expressions.get(expressions.size() - 1)); | ||
| } | ||
| } |
Oops, something went wrong.