HIVE-9025 : join38.q (without map join) produces incorrect result whe…

…n testing with multiple reducers (Ted Xu via Ashutosh Chauhan) git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1644717 13f79535-47bb-0310-9956-ffa450edef68
akitanaka · Dec 11, 2014 · fdbe214 · fdbe214
1 parent 0350c9f
commit fdbe214
Show file tree

Hide file tree

Showing 19 changed files with 168 additions and 7 deletions.
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
@@ -8,6 +8,7 @@ minimr.query.files=auto_sortmerge_join_16.q,\
   bucketizedhiveinputformat.q,\
   bucketmapjoin6.q,\
   bucketmapjoin7.q,\
+  constprog_partitioner.q,\
   disable_merge_for_bucketing.q,\
   empty_dir_in_table.q,\
   external_table_with_space_in_location_path.q,\

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
@@ -889,15 +889,22 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object..
       rsDesc.setKeyCols(newKeyEpxrs);
 
       // partition columns
-      ArrayList<ExprNodeDesc> newPartExprs = new ArrayList<ExprNodeDesc>();
-      for (ExprNodeDesc desc : rsDesc.getPartitionCols()) {
-        ExprNodeDesc expr = foldExpr(desc, constants, cppCtx, op, 0, false);
-        if (expr instanceof ExprNodeConstantDesc || expr instanceof ExprNodeNullDesc) {
-          continue;
+      if (!rsDesc.getPartitionCols().isEmpty()) {
+        ArrayList<ExprNodeDesc> newPartExprs = new ArrayList<ExprNodeDesc>();
+        for (ExprNodeDesc desc : rsDesc.getPartitionCols()) {
+          ExprNodeDesc expr = foldExpr(desc, constants, cppCtx, op, 0, false);
+          if (expr instanceof ExprNodeConstantDesc || expr instanceof ExprNodeNullDesc) {
+            continue;
+          }
+          newPartExprs.add(expr);
+        }
+        if (newPartExprs.isEmpty()) {
+          // If all partition columns are removed because of constant, insert an extra column to avoid
+          // random partitioning.
+          newPartExprs.add(new ExprNodeConstantDesc(""));
         }
-        newPartExprs.add(expr);
+        rsDesc.setPartitionCols(newPartExprs);
       }
-      rsDesc.setPartitionCols(newPartExprs);
 
       // value columns
       ArrayList<ExprNodeDesc> newValExprs = new ArrayList<ExprNodeDesc>();

diff --git a/ql/src/test/queries/clientpositive/constprog_partitioner.q b/ql/src/test/queries/clientpositive/constprog_partitioner.q
@@ -0,0 +1,11 @@
+set hive.fetch.task.conversion=more;
+set hive.optimize.constant.propagation=true;
+
+set mapred.reduce.tasks=4;
+
+EXPLAIN
+SELECT src1.key, src1.key + 1, src2.value
+       FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100;
+
+SELECT src1.key, src1.key + 1, src2.value
+       FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100;
diff --git a/ql/src/test/results/clientpositive/cluster.q.out b/ql/src/test/results/clientpositive/cluster.q.out
@@ -25,6 +25,7 @@ STAGE PLANS:
                 Reduce Output Operator
                   key expressions: '10' (type: string)
                   sort order: +
+                  Map-reduce partition columns: '' (type: string)
                   Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col1 (type: string)
       Reduce Operator Tree:
@@ -82,6 +83,7 @@ STAGE PLANS:
                 Reduce Output Operator
                   key expressions: '20' (type: string)
                   sort order: +
+                  Map-reduce partition columns: '' (type: string)
                   Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col1 (type: string)
       Reduce Operator Tree:
@@ -139,6 +141,7 @@ STAGE PLANS:
                 Reduce Output Operator
                   key expressions: '20' (type: string)
                   sort order: +
+                  Map-reduce partition columns: '' (type: string)
                   Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col1 (type: string)
       Reduce Operator Tree:
@@ -196,6 +199,7 @@ STAGE PLANS:
                 Reduce Output Operator
                   key expressions: '20' (type: string)
                   sort order: +
+                  Map-reduce partition columns: '' (type: string)
                   Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col1 (type: string)
       Reduce Operator Tree:
@@ -253,6 +257,7 @@ STAGE PLANS:
                 Reduce Output Operator
                   key expressions: '20' (type: string)
                   sort order: +
+                  Map-reduce partition columns: '' (type: string)
                   Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col1 (type: string)
       Reduce Operator Tree:
@@ -310,6 +315,7 @@ STAGE PLANS:
                 Reduce Output Operator
                   key expressions: '20' (type: string)
                   sort order: +
+                  Map-reduce partition columns: '' (type: string)
                   Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col1 (type: string)
       Reduce Operator Tree:
@@ -424,6 +430,7 @@ STAGE PLANS:
                 Reduce Output Operator
                   key expressions: '20' (type: string)
                   sort order: +
+                  Map-reduce partition columns: '' (type: string)
                   Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col1 (type: string)
       Reduce Operator Tree:
@@ -478,6 +485,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
           TableScan
@@ -489,6 +497,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator
@@ -567,6 +576,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
           TableScan
@@ -578,6 +588,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
       Reduce Operator Tree:
@@ -662,6 +673,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
           TableScan
@@ -673,6 +685,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
       Reduce Operator Tree:
@@ -702,6 +715,7 @@ STAGE PLANS:
             Reduce Output Operator
               key expressions: '20' (type: string)
               sort order: +
+              Map-reduce partition columns: '' (type: string)
               Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col1 (type: string), _col3 (type: string)
       Reduce Operator Tree:
@@ -756,6 +770,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
           TableScan
@@ -767,6 +782,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator
@@ -795,6 +811,7 @@ STAGE PLANS:
             Reduce Output Operator
               key expressions: '20' (type: string)
               sort order: +
+              Map-reduce partition columns: '' (type: string)
               Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col1 (type: string), _col2 (type: string)
       Reduce Operator Tree:

diff --git a/ql/src/test/results/clientpositive/constprog2.q.out b/ql/src/test/results/clientpositive/constprog2.q.out
@@ -23,6 +23,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '86' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
           TableScan
             alias: src2
@@ -33,6 +34,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '86' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
       Reduce Operator Tree:
@@ -98,6 +100,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '86' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
           TableScan
             alias: src2
@@ -108,6 +111,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '86' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
       Reduce Operator Tree:

diff --git a/ql/src/test/results/clientpositive/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/constprog_partitioner.q.out
@@ -0,0 +1,80 @@
+PREHOOK: query: EXPLAIN
+SELECT src1.key, src1.key + 1, src2.value
+       FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT src1.key, src1.key + 1, src2.value
+       FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: src1
+            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: (key = 100) (type: boolean)
+              Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+              Reduce Output Operator
+                key expressions: '100' (type: string)
+                sort order: +
+                Map-reduce partition columns: '' (type: string)
+                Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+          TableScan
+            alias: src2
+            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: (key = 100) (type: boolean)
+              Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+              Reduce Output Operator
+                key expressions: '100' (type: string)
+                sort order: +
+                Map-reduce partition columns: '' (type: string)
+                Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+                value expressions: value (type: string)
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 
+            1 {VALUE._col0}
+          outputColumnNames: _col6
+          Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+          Select Operator
+            expressions: '100' (type: string), 101.0 (type: double), _col6 (type: string)
+            outputColumnNames: _col0, _col1, _col2
+            Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+            File Output Operator
+              compressed: false
+              Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: SELECT src1.key, src1.key + 1, src2.value
+       FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT src1.key, src1.key + 1, src2.value
+       FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+100	101.0	val_100
+100	101.0	val_100
+100	101.0	val_100
+100	101.0	val_100
diff --git a/ql/src/test/results/clientpositive/join_nullsafe.q.out b/ql/src/test/results/clientpositive/join_nullsafe.q.out
@@ -1525,6 +1525,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: null (type: void)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: int)
           TableScan
@@ -1536,6 +1537,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: null (type: void)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
                 value expressions: key (type: int)
       Reduce Operator Tree:

diff --git a/ql/src/test/results/clientpositive/ppd2.q.out b/ql/src/test/results/clientpositive/ppd2.q.out
@@ -351,6 +351,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
           TableScan
@@ -362,6 +363,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator

diff --git a/ql/src/test/results/clientpositive/ppd_clusterby.q.out b/ql/src/test/results/clientpositive/ppd_clusterby.q.out
@@ -25,6 +25,7 @@ STAGE PLANS:
                 Reduce Output Operator
                   key expressions: '10' (type: string)
                   sort order: +
+                  Map-reduce partition columns: '' (type: string)
                   Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col1 (type: string)
       Reduce Operator Tree:
@@ -79,6 +80,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
           TableScan
@@ -90,6 +92,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator
@@ -171,6 +174,7 @@ STAGE PLANS:
                 Reduce Output Operator
                   key expressions: '10' (type: string)
                   sort order: +
+                  Map-reduce partition columns: '' (type: string)
                   Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col1 (type: string)
       Reduce Operator Tree:
@@ -225,6 +229,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                 value expressions: value (type: string)
           TableScan
@@ -236,6 +241,7 @@ STAGE PLANS:
               Reduce Output Operator
                 key expressions: '20' (type: string)
                 sort order: +
+                Map-reduce partition columns: '' (type: string)
                 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator