HIVE-9037 : Improve explain plan to show joining keys for shuffle joi…

…n (Ashutosh Chauhan via John Pullokkaran) git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1644752 13f79535-47bb-0310-9956-ffa450edef68
akitanaka · Dec 11, 2014 · 7dde96f · 7dde96f
1 parent fdbe214
commit 7dde96f
Show file tree

Hide file tree

Showing 416 changed files with 2,195 additions and 6,682 deletions.
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -152,9 +152,9 @@ STAGE PLANS:
         Join Operator
           condition map:
                Inner Join 0 to 1
-          condition expressions:
-            0 
-            1 {VALUE._col0} {VALUE._col1}
+          keys:
+            0 UDFToDouble(_col0) (type: double)
+            1 UDFToDouble(_col0) (type: double)
           outputColumnNames: _col2, _col3
           Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
           Select Operator
@@ -302,9 +302,9 @@ STAGE PLANS:
         Join Operator
           condition map:
                Inner Join 0 to 1
-          condition expressions:
-            0 
-            1 {KEY.reducesinkkey0} {VALUE._col0}
+          keys:
+            0 _col0 (type: int)
+            1 _col0 (type: int)
           outputColumnNames: _col2, _col3
           Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
           Select Operator
@@ -551,9 +551,9 @@ STAGE PLANS:
         Join Operator
           condition map:
                Inner Join 0 to 1
-          condition expressions:
-            0 {VALUE._col0} {VALUE._col1}
-            1 {VALUE._col1}
+          keys:
+            0 UDFToDouble(_col0) (type: double)
+            1 UDFToDouble(_col0) (type: double)
           outputColumnNames: _col0, _col1, _col3
           Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
           Select Operator

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -7175,7 +7175,7 @@ private Operator genReduceSinkPlan(String dest, QB qb, Operator<?> input,
   }
 
   private Operator genJoinOperatorChildren(QBJoinTree join, Operator left,
-      Operator[] right, HashSet<Integer> omitOpts) throws SemanticException {
+      Operator[] right, HashSet<Integer> omitOpts, ExprNodeDesc[][] joinKeys) throws SemanticException {
 
     RowResolver outputRR = new RowResolver();
     ArrayList<String> outputColumnNames = new ArrayList<String>();
@@ -7269,7 +7269,7 @@ private Operator genJoinOperatorChildren(QBJoinTree join, Operator left,
     }
 
     JoinDesc desc = new JoinDesc(exprMap, outputColumnNames,
-        join.getNoOuterJoin(), joinCondns, filterMap);
+        join.getNoOuterJoin(), joinCondns, filterMap, joinKeys);
     desc.setReversedExprs(reversedExprs);
     desc.setFilterMap(join.getFilterMap());
 
@@ -7470,7 +7470,7 @@ private Operator genJoinOperator(QB qb, QBJoinTree joinTree,
     }
 
     JoinOperator joinOp = (JoinOperator) genJoinOperatorChildren(joinTree,
-      joinSrcOp, srcOps, omitOpts);
+      joinSrcOp, srcOps, omitOpts, joinKeys);
     joinContext.put(joinOp, joinTree);
 
     Operator op = joinOp;

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CommonMergeJoinDesc.java
@@ -20,7 +20,6 @@
 
 import java.io.Serializable;
 
-
 @Explain(displayName = "Merge Join Operator")
 public class CommonMergeJoinDesc extends MapJoinDesc implements Serializable {
   private static final long serialVersionUID = 1L;

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java
@@ -19,14 +19,10 @@
 package org.apache.hadoop.hive.ql.plan;
 
 import java.io.Serializable;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
 
 import org.apache.hadoop.fs.Path;
 
@@ -117,21 +113,6 @@ public HashTableSinkDesc(MapJoinDesc clone) {
     this.hashtableMemoryUsage = clone.getHashTableMemoryUsage();
   }
 
-
-  private void initRetainExprList() {
-    retainList = new HashMap<Byte, List<Integer>>();
-    Set<Entry<Byte, List<ExprNodeDesc>>> set = exprs.entrySet();
-    Iterator<Entry<Byte, List<ExprNodeDesc>>> setIter = set.iterator();
-    while (setIter.hasNext()) {
-      Entry<Byte, List<ExprNodeDesc>> current = setIter.next();
-      List<Integer> list = new ArrayList<Integer>();
-      for (int i = 0; i < current.getValue().size(); i++) {
-        list.add(i);
-      }
-      retainList.put(current.getKey(), list);
-    }
-  }
-
   public float getHashtableMemoryUsage() {
     return hashtableMemoryUsage;
   }

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
@@ -28,7 +28,6 @@
 
 import org.apache.hadoop.fs.Path;
 
-
 /**
  * Join operator Descriptor implementation.
  *
@@ -87,17 +86,20 @@ public class JoinDesc extends AbstractOperatorDesc {
   // it's resulted from RS-dedup optimization, which removes following RS under some condition
   private boolean fixedAsSorted;
 
+  // used only for explain.
+  private transient ExprNodeDesc [][] joinKeys;
   public JoinDesc() {
   }
 
   public JoinDesc(final Map<Byte, List<ExprNodeDesc>> exprs,
       List<String> outputColumnNames, final boolean noOuterJoin,
-      final JoinCondDesc[] conds, final Map<Byte, List<ExprNodeDesc>> filters) {
+      final JoinCondDesc[] conds, final Map<Byte, List<ExprNodeDesc>> filters, ExprNodeDesc[][] joinKeys) {
     this.exprs = exprs;
     this.outputColumnNames = outputColumnNames;
     this.noOuterJoin = noOuterJoin;
     this.conds = conds;
     this.filters = filters;
+    this.joinKeys = joinKeys;
 
     resetOrder();
   }
@@ -157,22 +159,6 @@ public Object clone() {
     return ret;
   }
 
-  public JoinDesc(final Map<Byte, List<ExprNodeDesc>> exprs,
-      List<String> outputColumnNames, final boolean noOuterJoin,
-      final JoinCondDesc[] conds) {
-    this(exprs, outputColumnNames, noOuterJoin, conds, null);
-  }
-
-  public JoinDesc(final Map<Byte, List<ExprNodeDesc>> exprs,
-      List<String> outputColumnNames) {
-    this(exprs, outputColumnNames, true, null);
-  }
-
-  public JoinDesc(final Map<Byte, List<ExprNodeDesc>> exprs,
-      List<String> outputColumnNames, final JoinCondDesc[] conds) {
-    this(exprs, outputColumnNames, true, conds, null);
-  }
-
   public JoinDesc(JoinDesc clone) {
     this.bigKeysDirMap = clone.bigKeysDirMap;
     this.conds = clone.conds;
@@ -204,33 +190,16 @@ public void setReversedExprs(Map<String, Byte> reversedExprs) {
     this.reversedExprs = reversedExprs;
   }
 
-  @Explain(displayName = "condition expressions")
-  public Map<Byte, String> getExprsStringMap() {
-    if (getExprs() == null) {
-      return null;
-    }
-
-    LinkedHashMap<Byte, String> ret = new LinkedHashMap<Byte, String>();
-
-    for (Map.Entry<Byte, List<ExprNodeDesc>> ent : getExprs().entrySet()) {
-      StringBuilder sb = new StringBuilder();
-      boolean first = true;
-      if (ent.getValue() != null) {
-        for (ExprNodeDesc expr : ent.getValue()) {
-          if (!first) {
-            sb.append(" ");
-          }
-
-          first = false;
-          sb.append("{");
-          sb.append(expr.getExprString());
-          sb.append("}");
-        }
-      }
-      ret.put(ent.getKey(), sb.toString());
+  /**
+   * @return the keys in string form
+   */
+  @Explain(displayName = "keys")
+  public Map<Byte, String> getKeysString() {
+    Map<Byte, String> keyMap = new LinkedHashMap<Byte, String>();
+    for (byte i = 0; i < joinKeys.length; i++) {
+      keyMap.put(i, PlanUtils.getExprListString(Arrays.asList(joinKeys[i])));
     }
-
-    return ret;
+    return keyMap;
   }
 
   public void setExprs(final Map<Byte, List<ExprNodeDesc>> exprs) {

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
@@ -98,7 +98,7 @@ public MapJoinDesc(final Map<Byte, List<ExprNodeDesc>> keys,
       final List<TableDesc> valueTblDescs,final List<TableDesc> valueFilteredTblDescs,  List<String> outputColumnNames,
       final int posBigTable, final JoinCondDesc[] conds,
       final Map<Byte, List<ExprNodeDesc>> filters, boolean noOuterJoin, String dumpFilePrefix) {
-    super(values, outputColumnNames, noOuterJoin, conds, filters);
+    super(values, outputColumnNames, noOuterJoin, conds, filters, null);
     this.keys = keys;
     this.keyTblDesc = keyTblDesc;
     this.valueTblDescs = valueTblDescs;
@@ -192,6 +192,7 @@ public void setDumpFilePrefix(String dumpFilePrefix) {
   /**
    * @return the keys in string form
    */
+  @Override
   @Explain(displayName = "keys")
   public Map<Byte, String> getKeysString() {
     Map<Byte, String> keyMap = new LinkedHashMap<Byte, String>();
@@ -200,7 +201,7 @@ public Map<Byte, String> getKeysString() {
     }
     return keyMap;
   }
-   
+
   /**
    * @return the keys
    */
@@ -325,11 +326,11 @@ public void setHashTableMemoryUsage(float hashtableMemoryUsage) {
   public float getHashTableMemoryUsage() {
     return hashtableMemoryUsage;
   }
-  
+
   public void setCustomBucketMapJoin(boolean customBucketMapJoin) {
     this.customBucketMapJoin = customBucketMapJoin;
   }
-  
+
   public boolean getCustomBucketMapJoin() {
     return this.customBucketMapJoin;
   }

diff --git a/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out b/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out
@@ -130,9 +130,9 @@ STAGE PLANS:
         Join Operator
           condition map:
                Inner Join 0 to 1
-          condition expressions:
-            0 {KEY.reducesinkkey0} {VALUE._col0}
-            1 {VALUE._col0}
+          keys:
+            0 key (type: int)
+            1 key (type: int)
           outputColumnNames: _col0, _col1, _col7
           Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE
           Select Operator
@@ -184,9 +184,6 @@ STAGE PLANS:
               predicate: key is not null (type: boolean)
               Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE
               HashTable Sink Operator
-                condition expressions:
-                  0 {key} {value}
-                  1 {value}
                 keys:
                   0 key (type: int)
                   1 key (type: int)
@@ -203,9 +200,6 @@ STAGE PLANS:
               Map Join Operator
                 condition map:
                      Inner Join 0 to 1
-                condition expressions:
-                  0 {key} {value}
-                  1 {value}
                 keys:
                   0 key (type: int)
                   1 key (type: int)

diff --git a/ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out b/ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out
@@ -79,9 +79,6 @@ STAGE PLANS:
               predicate: key is not null (type: boolean)
               Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
               HashTable Sink Operator
-                condition expressions:
-                  0 {value}
-                  1 {key} {value}
                 keys:
                   0 key (type: int)
                   1 key (type: int)
@@ -98,9 +95,6 @@ STAGE PLANS:
               Map Join Operator
                 condition map:
                      Inner Join 0 to 1
-                condition expressions:
-                  0 {key} {value}
-                  1 {key} {value}
                 keys:
                   0 key (type: int)
                   1 key (type: int)

diff --git a/ql/src/test/results/clientpositive/allcolref_in_udf.q.out b/ql/src/test/results/clientpositive/allcolref_in_udf.q.out
@@ -109,9 +109,9 @@ STAGE PLANS:
         Join Operator
           condition map:
                Inner Join 0 to 1
-          condition expressions:
-            0 {VALUE._col0} {VALUE._col1}
-            1 {VALUE._col0} {VALUE._col1}
+          keys:
+            0 (key + 1) (type: double)
+            1 UDFToDouble(key) (type: double)
           outputColumnNames: _col0, _col1, _col5, _col6
           Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
           Select Operator

diff --git a/ql/src/test/results/clientpositive/ambiguous_col.q.out b/ql/src/test/results/clientpositive/ambiguous_col.q.out
@@ -47,9 +47,9 @@ STAGE PLANS:
         Join Operator
           condition map:
                Inner Join 0 to 1
-          condition expressions:
-            0 {KEY.reducesinkkey0} {VALUE._col0}
-            1 
+          keys:
+            0 _col0 (type: string)
+            1 _col0 (type: string)
           outputColumnNames: _col0, _col1
           Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
           Select Operator
@@ -118,9 +118,9 @@ STAGE PLANS:
         Join Operator
           condition map:
                Inner Join 0 to 1
-          condition expressions:
-            0 {KEY.reducesinkkey0}
-            1 
+          keys:
+            0 _col0 (type: string)
+            1 _col0 (type: string)
           outputColumnNames: _col0
           Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
           Select Operator
@@ -189,9 +189,9 @@ STAGE PLANS:
         Join Operator
           condition map:
                Inner Join 0 to 1
-          condition expressions:
-            0 {KEY.reducesinkkey0}
-            1 
+          keys:
+            0 _col0 (type: string)
+            1 _col0 (type: string)
           outputColumnNames: _col0
           Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
           Select Operator