Skip to content

Commit

Permalink
[CALCITE-4735] Enhance Aggregate-Materialization, when aggcall of que…
Browse files Browse the repository at this point in the history 
…ry could be expressed by target's grouping (xurenhe)
  • Loading branch information
@wojustme @yanlin-Lynn
wojustme authored and yanlin-Lynn committed Oct 15, 2021
1 parent 9c741e4 commit 46fb263
Show file tree
Hide file tree
Showing 2 changed files with 170 additions and 33 deletions.
124 changes: 91 additions & 33 deletions core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import org.apache.calcite.util.ControlFlowException;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Litmus;
import org.apache.calcite.util.Optionality;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.mapping.Mapping;
Expand Down Expand Up @@ -1906,18 +1907,39 @@ public static MutableAggregate permute(MutableAggregate aggregate,
int i = targetGroupByIndexList.indexOf(inputIndex);
projects.add(i);
}
final List<AggregateCall> targetGroupGenAggCalls = new ArrayList<>();
for (AggregateCall aggregateCall : query.aggCalls) {
int i = target.aggCalls.indexOf(aggregateCall);
if (i < 0) {
return null;
final AggregateCall newAggCall = genAggCallWithTargetGrouping(
aggregateCall, targetGroupByIndexList);
if (newAggCall == null) {
return null;
} else {
// Here, we create a new `MutableAggregate` to return.
// So, we record this new agg-call.
targetGroupGenAggCalls.add(newAggCall);
}
} else {
if (!targetGroupGenAggCalls.isEmpty()) {
// Here, we didn't build target's agg-call by ref of mv's agg-call,
// if some agg-call is generated by target's grouping.
// So, we return null to stop it.
return null;
}
projects.add(groupCount + i + projOffset);
}
projects.add(groupCount + i + projOffset);
}

List<RexNode> compenProjs = MutableRels.createProjectExprs(target, projects);
RexProgram compenRexProgram = RexProgram.create(
target.rowType, compenProjs, targetCond, query.rowType, rexBuilder);
result = MutableCalc.of(target, compenRexProgram);
if (targetGroupGenAggCalls.isEmpty()) {
List<RexNode> compenProjs = MutableRels.createProjectExprs(target, projects);
RexProgram compenRexProgram = RexProgram.create(
target.rowType, compenProjs, targetCond, query.rowType, rexBuilder);
result = MutableCalc.of(target, compenRexProgram);
} else {
result = MutableAggregate.of(target,
target.groupSet, target.groupSets, targetGroupGenAggCalls);
}
} else if (target.getGroupType() == Aggregate.Group.SIMPLE) {
// Query is coarser level of aggregation. Generate an aggregate.
final Map<Integer, Integer> map = new HashMap<>();
Expand All @@ -1935,38 +1957,34 @@ public static MutableAggregate permute(MutableAggregate aggregate,
}
final List<AggregateCall> aggregateCalls = new ArrayList<>();
for (AggregateCall aggregateCall : query.aggCalls) {
if (aggregateCall.isDistinct() && aggregateCall.getArgList().size() == 1) {
final int aggIndex = aggregateCall.getArgList().get(0);
final int newIndex = targetGroupByIndexList.indexOf(aggIndex);
if (newIndex >= 0) {
aggregateCalls.add(
AggregateCall.create(aggregateCall.getAggregation(),
aggregateCall.isDistinct(), aggregateCall.isApproximate(),
aggregateCall.ignoreNulls(),
ImmutableList.of(newIndex), -1, aggregateCall.distinctKeys,
aggregateCall.collation, aggregateCall.type,
aggregateCall.name));
continue;
AggregateCall newAggCall = null;
// 1. try to find rollup agg-call.
if (!aggregateCall.isDistinct()) {
int i = target.aggCalls.indexOf(aggregateCall);
if (i >= 0) {
// When an SqlAggFunction does not support roll up, it will return null,
// which means that it cannot do secondary aggregation
// and the materialization recognition will fail.
final SqlAggFunction aggFunction = aggregateCall.getAggregation().getRollup();
if (aggFunction != null) {
newAggCall = AggregateCall.create(aggFunction,
aggregateCall.isDistinct(), aggregateCall.isApproximate(),
aggregateCall.ignoreNulls(),
ImmutableList.of(target.groupSet.cardinality() + i), -1,
aggregateCall.distinctKeys, aggregateCall.collation,
aggregateCall.type, aggregateCall.name);
}
}
return null;
}
int i = target.aggCalls.indexOf(aggregateCall);
if (i < 0) {
return null;
// 2. try to build a new agg-cal by target's grouping.
if (newAggCall == null) {
newAggCall = genAggCallWithTargetGrouping(aggregateCall, targetGroupByIndexList);
}
// When an SqlAggFunction does not support roll up, it will return null, which means that
// it cannot do secondary aggregation and the materialization recognition will fail.
final SqlAggFunction aggFunction = aggregateCall.getAggregation().getRollup();
if (aggFunction == null) {
if (newAggCall == null) {
// gen agg call fail.
return null;
}
aggregateCalls.add(
AggregateCall.create(aggFunction,
aggregateCall.isDistinct(), aggregateCall.isApproximate(),
aggregateCall.ignoreNulls(),
ImmutableList.of(target.groupSet.cardinality() + i), -1,
aggregateCall.distinctKeys, aggregateCall.collation,
aggregateCall.type, aggregateCall.name));
aggregateCalls.add(newAggCall);
}
if (targetCond != null && !targetCond.isAlwaysTrue()) {
RexProgram compenRexProgram = RexProgram.create(
Expand All @@ -1986,6 +2004,46 @@ public static MutableAggregate permute(MutableAggregate aggregate,
return result;
}

/**
* Generate agg call by mv's grouping.
*/
private static @Nullable AggregateCall genAggCallWithTargetGrouping(AggregateCall queryAggCall,
List<Integer> targetGroupByIndexes) {
final SqlAggFunction aggregation = queryAggCall.getAggregation();
final List<Integer> argList = queryAggCall.getArgList();
final List<Integer> newArgList = new ArrayList<>();
for (Integer arg : argList) {
final int newArgIndex = targetGroupByIndexes.indexOf(arg);
if (newArgIndex < 0) {
return null;
}
newArgList.add(newArgIndex);
}
final boolean isAllowBuild;
if (newArgList.size() == 0) {
// Size of agg-call's args is empty, we stop to build a new agg-call,
// eg: count(1) or count(*).
isAllowBuild = false;
} else if (queryAggCall.isDistinct()) {
// Args of agg-call is distinct, we can build a new agg-call.
isAllowBuild = true;
} else if (aggregation.getDistinctOptionality() == Optionality.IGNORED) {
// If attribute of agg-call's distinct could be ignore, we can build a new agg-call.
isAllowBuild = true;
} else {
isAllowBuild = false;
}
if (!isAllowBuild) {
return null;
}
return AggregateCall.create(aggregation,
queryAggCall.isDistinct(), queryAggCall.isApproximate(),
queryAggCall.ignoreNulls(),
newArgList, -1, queryAggCall.distinctKeys,
queryAggCall.collation, queryAggCall.type,
queryAggCall.name);
}

@Deprecated // to be removed before 2.0
public static @Nullable SqlAggFunction getRollup(SqlAggFunction aggregation) {
if (aggregation == SqlStdOperatorTable.SUM
Expand Down
79 changes: 79 additions & 0 deletions core/src/test/java/org/apache/calcite/test/MaterializedViewSubstitutionVisitorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1135,6 +1135,85 @@ public class MaterializedViewSubstitutionVisitorTest extends AbstractMaterialize
sql(mv, query).ok();
}

/**
* It's match, distinct agg-call could be expressed by mv's grouping.
*/
@Test void testAggDistinctInMvGrouping() {
final String mv = ""
+ "select \"deptno\", \"name\""
+ "from \"emps\" group by \"deptno\", \"name\"";
final String query = ""
+ "select \"deptno\", \"name\", count(distinct \"name\")"
+ "from \"emps\" group by \"deptno\", \"name\"";
sql(mv, query).ok();
}

/**
* It's match, `Optionality.IGNORED` agg-call could be expressed by mv's grouping.
*/
@Test void testAggOptionalityInMvGrouping() {
final String mv = ""
+ "select \"deptno\", \"salary\""
+ "from \"emps\" group by \"deptno\", \"salary\"";
final String query = ""
+ "select \"deptno\", \"salary\", max(\"salary\")"
+ "from \"emps\" group by \"deptno\", \"salary\"";
sql(mv, query).ok();
}

/**
* It's not match, normal agg-call could be expressed by mv's grouping.
* Such as: sum, count
*/
@Test void testAggNormalInMvGrouping() {
final String mv = ""
+ "select \"deptno\", \"salary\""
+ "from \"emps\" group by \"deptno\", \"salary\"";
final String query = ""
+ "select \"deptno\", sum(\"salary\")"
+ "from \"emps\" group by \"deptno\"";
sql(mv, query).noMat();
}

/**
* It's not match, which is count(*) with same grouping.
*/
@Test void testGenerateQueryAggCallByMvGroupingForEmptyArg1() {
final String mv = ""
+ "select \"deptno\""
+ "from \"emps\" group by \"deptno\"";
final String query = ""
+ "select \"deptno\", count(*)"
+ "from \"emps\" group by \"deptno\"";
sql(mv, query).noMat();
}

/**
* It's not match, which is count(*) with rollup grouping.
*/
@Test void testGenerateQueryAggCallByMvGroupingForEmptyArg2() {
final String mv = ""
+ "select \"deptno\", \"commission\", \"salary\""
+ "from \"emps\" group by \"deptno\", \"commission\", \"salary\"";
final String query = ""
+ "select \"deptno\", \"commission\", count(*)"
+ "from \"emps\" group by \"deptno\", \"commission\"";
sql(mv, query).noMat();
}

/**
* It's match, when query's agg-calls could be both rollup and expressed by mv's grouping.
*/
@Test void testAggCallBothGenByMvGroupingAndRollupOk() {
final String mv = ""
+ "select \"name\", \"deptno\", \"empid\", min(\"commission\")"
+ "from \"emps\" group by \"name\", \"deptno\", \"empid\"";
final String query = ""
+ "select \"name\", max(\"deptno\"), count(distinct \"empid\"), min(\"commission\")"
+ "from \"emps\" group by \"name\"";
sql(mv, query).ok();
}

/** Unit test for logic functions
* {@link org.apache.calcite.plan.SubstitutionVisitor#mayBeSatisfiable} and
* {@link RexUtil#simplify}. */
Expand Down

0 comments on commit 46fb263

Please sign in to comment.