Skip to content

Commit

Permalink
HIVE-8869 : RowSchema not updated for some ops when columns are prune…
Browse files Browse the repository at this point in the history 
…d (Jesús Camacho Rodríguez via Ashutosh Chauhan)

git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1640227 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
@ashutoshc
ashutoshc committed Nov 17, 2014
1 parent d202ca0 commit 6ad69d9
Show file tree
Hide file tree
Showing 7 changed files with 87 additions and 67 deletions.
14 changes: 11 additions & 3 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.List;
import java.util.Map;

import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
Expand Down Expand Up @@ -128,9 +129,16 @@ public List<String> genColLists(Operator<? extends OperatorDesc> curOp)
public List<String> getColsFromSelectExpr(SelectOperator op) {
List<String> cols = new ArrayList<String>();
SelectDesc conf = op.getConf();
List<ExprNodeDesc> exprList = conf.getColList();
for (ExprNodeDesc expr : exprList) {
cols = Utilities.mergeUniqElems(cols, expr.getCols());
if(conf.isSelStarNoCompute()) {
for (ColumnInfo colInfo : op.getSchema().getSignature()) {
cols.add(colInfo.getInternalName());
}
}
else {
List<ExprNodeDesc> exprList = conf.getColList();
for (ExprNodeDesc expr : exprList) {
cols = Utilities.mergeUniqElems(cols, expr.getCols());
}
}
return cols;
}
Expand Down
20 changes: 16 additions & 4 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,12 +386,14 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
scanOp.setNeededColumnIDs(null);
return null;
}

cols = cols == null ? new ArrayList<String>() : cols;

cppCtx.getPrunedColLists().put((Operator<? extends OperatorDesc>) nd,
cols);
RowResolver inputRR = cppCtx.getOpToParseCtxMap().get(scanOp).getRowResolver();
setupNeededColumns(scanOp, inputRR, cols);

return null;
}
}
Expand Down Expand Up @@ -569,7 +571,7 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
// following SEL will do CP for columns from UDTF, not adding SEL in here
newColNames.addAll(outputCols.subList(numSelColumns, outputCols.size()));
op.getConf().setOutputInternalColNames(newColNames);

pruneOperator(ctx, op, newColNames);
cppCtx.getPrunedColLists().put(op, colsAfterReplacement);
return null;
}
Expand Down Expand Up @@ -607,6 +609,12 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
((SelectDesc)select.getConf()).setSelStarNoCompute(false);
((SelectDesc)select.getConf()).setColList(colList);
((SelectDesc)select.getConf()).setOutputColumnNames(outputColNames);
pruneOperator(ctx, select, outputColNames);

Operator<?> udtfPath = op.getChildOperators().get(LateralViewJoinOperator.UDTF_TAG);
List<String> lvFCols = new ArrayList<String>(cppCtx.getPrunedColLists().get(udtfPath));
lvFCols = Utilities.mergeUniqElems(lvFCols, outputColNames);
pruneOperator(ctx, op, lvFCols);
}
return null;
}
Expand Down Expand Up @@ -874,12 +882,16 @@ private static void pruneOperator(NodeProcessorCtx ctx,
RowSchema inputSchema = op.getSchema();
if (inputSchema != null) {
ArrayList<ColumnInfo> rs = new ArrayList<ColumnInfo>();
ArrayList<ColumnInfo> inputCols = inputSchema.getSignature();
for (ColumnInfo i: inputCols) {
if (cols.contains(i.getInternalName())) {
RowResolver oldRR = ((ColumnPrunerProcCtx)ctx).getOpToParseCtxMap().get(op).getRowResolver();
RowResolver newRR = new RowResolver();
for(ColumnInfo i : oldRR.getRowSchema().getSignature()) {
if ( cols.contains(i.getInternalName())) {
String[] nm = oldRR.reverseLookup(i.getInternalName());
newRR.put(nm[0], nm[1], i);
rs.add(i);
}
}
((ColumnPrunerProcCtx)ctx).getOpToParseCtxMap().get(op).setRowResolver(newRR);
op.getSchema().setSignature(rs);
}
}
Expand Down
12 changes: 6 additions & 6 deletions ql/src/test/results/clientnegative/udf_assert_true.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ STAGE PLANS:
Lateral View Forward
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 134000 Basic stats: COMPLETE Column stats: COMPLETE
Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Lateral View Join Operator
outputColumnNames: _col5
Statistics: Num rows: 1000 Data size: 158000 Basic stats: COMPLETE Column stats: COMPLETE
Statistics: Num rows: 1000 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: assert_true((_col5 > 0)) (type: void)
outputColumnNames: _col0
Expand All @@ -48,7 +48,7 @@ STAGE PLANS:
function name: explode
Lateral View Join Operator
outputColumnNames: _col5
Statistics: Num rows: 1000 Data size: 158000 Basic stats: COMPLETE Column stats: COMPLETE
Statistics: Num rows: 1000 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: assert_true((_col5 > 0)) (type: void)
outputColumnNames: _col0
Expand Down Expand Up @@ -98,10 +98,10 @@ STAGE PLANS:
Lateral View Forward
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 134000 Basic stats: COMPLETE Column stats: COMPLETE
Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Lateral View Join Operator
outputColumnNames: _col5
Statistics: Num rows: 1000 Data size: 158000 Basic stats: COMPLETE Column stats: COMPLETE
Statistics: Num rows: 1000 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: assert_true((_col5 < 2)) (type: void)
outputColumnNames: _col0
Expand All @@ -125,7 +125,7 @@ STAGE PLANS:
function name: explode
Lateral View Join Operator
outputColumnNames: _col5
Statistics: Num rows: 1000 Data size: 158000 Basic stats: COMPLETE Column stats: COMPLETE
Statistics: Num rows: 1000 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: assert_true((_col5 < 2)) (type: void)
outputColumnNames: _col0
Expand Down
6 changes: 3 additions & 3 deletions ql/src/test/results/clientnegative/udf_assert_true2.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ STAGE PLANS:
Lateral View Forward
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 134000 Basic stats: COMPLETE Column stats: COMPLETE
Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Lateral View Join Operator
outputColumnNames: _col5
Statistics: Num rows: 1000 Data size: 158000 Basic stats: COMPLETE Column stats: COMPLETE
Statistics: Num rows: 1000 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: (1 + assert_true((_col5 < 2))) (type: double)
outputColumnNames: _col0
Expand All @@ -43,7 +43,7 @@ STAGE PLANS:
function name: explode
Lateral View Join Operator
outputColumnNames: _col5
Statistics: Num rows: 1000 Data size: 158000 Basic stats: COMPLETE Column stats: COMPLETE
Statistics: Num rows: 1000 Data size: 24000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: (1 + assert_true((_col5 < 2))) (type: double)
outputColumnNames: _col0
Expand Down
Loading

0 comments on commit 6ad69d9

Please sign in to comment.