From 30f6991e0fa3af1db8c8232921060b7837577ec1 Mon Sep 17 00:00:00 2001 From: "xinqiu.hu" Date: Thu, 20 Apr 2023 16:40:47 +0800 Subject: [PATCH 1/2] [CALCITE-5665] Reducing ineffective matches for MaterializedViewRules --- .../materialize/MaterializedViewRule.java | 111 +++++++++++++++--- .../test/MaterializedViewRelOptRulesTest.java | 13 ++ 2 files changed, 107 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java index e2aa969c4ab0..cabe307adf8b 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java @@ -56,6 +56,9 @@ import org.apache.calcite.util.mapping.IntPair; import org.apache.calcite.util.mapping.Mapping; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; @@ -76,6 +79,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.apache.calcite.linq4j.Nullness.castNonNull; @@ -92,6 +96,16 @@ public abstract class MaterializedViewRule extends RelRule { + /** + * Cache of RelMetadataQuery to MaterializationMetadata. + */ + private final LoadingCache + materializationMetadataCache = + CacheBuilder.newBuilder() + .expireAfterAccess(5, TimeUnit.MINUTES) + .maximumSize(1_024) + .build(CacheLoader.from(MaterializationMetadata::new)); + //~ Constructors ----------------------------------------------------------- /** Creates a MaterializedViewRule. */ @@ -100,7 +114,20 @@ public abstract class MaterializedViewRule materializations = + call.getPlanner().getMaterializations(); + if (materializations.isEmpty()) { + return false; + } + RelMetadataQuery mq = call.getMetadataQuery(); + MaterializationMetadata materializationMetadata = + materializationMetadataCache.getUnchecked(mq); + for (RelOptMaterialization materialization : materializations) { + if (materializationMetadata.isValidMaterialization(mq, materialization)) { + return true; + } + } + return false; } /** @@ -197,7 +224,15 @@ protected void perform(RelOptRuleCall call, @Nullable Project topProject, RelNod // 3. We iterate through all applicable materializations trying to // rewrite the given query + MaterializationMetadata materializationMetadata = + materializationMetadataCache.getUnchecked(mq); for (RelOptMaterialization materialization : materializations) { + // 3.1. View checks before proceeding + if (!materializationMetadata.isValidMaterialization(mq, materialization)) { + // Skip it + continue; + } + RelNode view = materialization.tableRel; Project topViewProject; RelNode viewNode; @@ -210,11 +245,8 @@ protected void perform(RelOptRuleCall call, @Nullable Project topProject, RelNod } // Extract view table references - final Set viewTableRefs = mq.getTableReferences(viewNode); - if (viewTableRefs == null) { - // Skip it - continue; - } + final Set viewTableRefs = + materializationMetadata.viewTableRefsMap.get(materialization); // Filter relevant materializations. Currently, we only check whether // the materialization contains any table that is used by the query @@ -231,21 +263,11 @@ protected void perform(RelOptRuleCall call, @Nullable Project topProject, RelNod continue; } - // 3.1. View checks before proceeding - if (!isValidPlan(topViewProject, viewNode, mq)) { - // Skip it - continue; - } - // 3.2. Initialize all query related auxiliary data structures // that will be used throughout query rewriting process // Extract view predicates final RelOptPredicateList viewPredicateList = - mq.getAllPredicates(viewNode); - if (viewPredicateList == null) { - // Skip it - continue; - } + materializationMetadata.viewPredicateListMap.get(materialization); final RexNode viewPred = simplify.simplifyUnknownAsFalse( RexUtil.composeConjunction(rexBuilder, @@ -1426,6 +1448,61 @@ protected enum MatchModality { QUERY_PARTIAL } + /** + * Check if the materialization is valid for this rule. + * If it does, cache the materialization's RelTableRef and RelOptPredicateList. + */ + protected class MaterializationMetadata { + private final Set notValidMaterializations = new HashSet<>(); + private final Map> viewTableRefsMap = new HashMap<>(); + private final Map + viewPredicateListMap = new HashMap<>(); + + protected boolean isValidMaterialization( + RelMetadataQuery mq, RelOptMaterialization materialization) { + if (notValidMaterializations.contains(materialization)) { + return false; + } else if (viewTableRefsMap.containsKey(materialization)) { + return true; + } else { + Project topViewProject; + RelNode viewNode; + if (materialization.queryRel instanceof Project) { + topViewProject = (Project) materialization.queryRel; + viewNode = topViewProject.getInput(); + } else { + topViewProject = null; + viewNode = materialization.queryRel; + } + + final Set viewTableRefs = mq.getTableReferences(viewNode); + if (viewTableRefs == null) { + // Skip it + notValidMaterializations.add(materialization); + return false; + } + + if (!isValidPlan(topViewProject, viewNode, mq)) { + // Skip it + notValidMaterializations.add(materialization); + return false; + } + + final RelOptPredicateList viewPredicateList = + mq.getAllPredicates(viewNode); + if (viewPredicateList == null) { + // Skip it + notValidMaterializations.add(materialization); + return false; + } + + viewTableRefsMap.put(materialization, viewTableRefs); + viewPredicateListMap.put(materialization, viewPredicateList); + return true; + } + } + } + /** Rule configuration. */ public interface Config extends RelRule.Config { /** Whether to generate rewritings containing union if the query results diff --git a/core/src/test/java/org/apache/calcite/test/MaterializedViewRelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/MaterializedViewRelOptRulesTest.java index ccbb8514125b..73137f30a143 100644 --- a/core/src/test/java/org/apache/calcite/test/MaterializedViewRelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/MaterializedViewRelOptRulesTest.java @@ -157,6 +157,19 @@ protected final MaterializedViewFixture sql(String materialize, .noMat(); } + @Test void testMultiMaterializationsNoAggregateFuncs() { + fixture("select \"empid\", \"deptno\" from \"emps\" group by \"empid\", \"deptno\"") + .withMaterializations( + ImmutableList.of( + Pair.of( + "select \"empid\", \"deptno\" from \"emps\" group by \"empid\", \"deptno\"", + "MV0"), + Pair.of( + "select \"empid\" from \"emps\" group by \"empid\", \"deptno\"", + "MV1"))) + .ok(); + } + @Test void testAggregateMaterializationAggregateFuncs1() { sql("select \"empid\", \"deptno\", count(*) as c, sum(\"empid\") as s\n" + "from \"emps\" group by \"empid\", \"deptno\"", From 7dbfdb2b2cfaa3c4e4d800574aaeb238411f9aa0 Mon Sep 17 00:00:00 2001 From: "xinqiu.hu" Date: Tue, 25 Apr 2023 15:30:05 +0800 Subject: [PATCH 2/2] fix CheckerFramework error --- .../materialize/MaterializedViewRule.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java index cabe307adf8b..a302bda6f596 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java @@ -123,7 +123,8 @@ public abstract class MaterializedViewRule viewTableRefs = materializationMetadata.viewTableRefsMap.get(materialization); + assert viewTableRefs != null; // Filter relevant materializations. Currently, we only check whether // the materialization contains any table that is used by the query @@ -268,6 +271,7 @@ protected void perform(RelOptRuleCall call, @Nullable Project topProject, RelNod // Extract view predicates final RelOptPredicateList viewPredicateList = materializationMetadata.viewPredicateListMap.get(materialization); + assert viewPredicateList != null; final RexNode viewPred = simplify.simplifyUnknownAsFalse( RexUtil.composeConjunction(rexBuilder, @@ -1452,17 +1456,20 @@ protected enum MatchModality { * Check if the materialization is valid for this rule. * If it does, cache the materialization's RelTableRef and RelOptPredicateList. */ - protected class MaterializationMetadata { + protected static class MaterializationMetadata { private final Set notValidMaterializations = new HashSet<>(); private final Map> viewTableRefsMap = new HashMap<>(); private final Map viewPredicateListMap = new HashMap<>(); - protected boolean isValidMaterialization( - RelMetadataQuery mq, RelOptMaterialization materialization) { + protected boolean isValidMaterialization( + RelMetadataQuery mq, + RelOptMaterialization materialization, + MaterializedViewRule materializedViewRule) { if (notValidMaterializations.contains(materialization)) { return false; - } else if (viewTableRefsMap.containsKey(materialization)) { + } else if (viewTableRefsMap.containsKey(materialization) + && viewPredicateListMap.containsKey(materialization)) { return true; } else { Project topViewProject; @@ -1482,7 +1489,7 @@ protected boolean isValidMaterialization( return false; } - if (!isValidPlan(topViewProject, viewNode, mq)) { + if (!materializedViewRule.isValidPlan(topViewProject, viewNode, mq)) { // Skip it notValidMaterializations.add(materialization); return false;