*: optimization, do not read not referenced columns.

ast/ast.go

@@ -120,6 +120,9 @@ type ResultField struct {
 	// will be set for every retrieved row.
 	Expr      ExprNode
 	TableName *TableName
+	// Whether this result field has been referenced.
+	// If not, we don't need to get the values.
+	Referenced bool
 }
 
 // Row represents a single row from Recordset.

executor/executor.go

@@ -287,13 +287,22 @@ func (e *TableScanExec) seekRange(handle int64) (inRange bool) {
 func (e *TableScanExec) getRow(handle int64) (*Row, error) {
 	row := &Row{}
 	var err error
-	row.Data, err = e.t.Row(e.ctx, handle)
+
+	columns := make([]*column.Col, len(e.fields))
+	for i, v := range e.fields {
+		if v.Referenced {
+			columns[i] = e.t.Cols()[i]
+		}
+	}
+	row.Data, err = e.t.RowWithCols(e.ctx, handle, columns)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
 	// Set result fields value.
 	for i, v := range e.fields {
-		v.Expr.SetValue(row.Data[i].GetValue())
+		if v.Referenced {
+			v.Expr.SetValue(row.Data[i].GetValue())
+		}
 	}
 
 	// Put rowKey to the tail of record row
@@ -415,7 +424,13 @@ func indexCompare(idxKey []types.Datum, boundVals []types.Datum) (int, error) {
 func (e *IndexRangeExec) lookupRow(h int64) (*Row, error) {
 	row := &Row{}
 	var err error
-	row.Data, err = e.scan.tbl.Row(e.scan.ctx, h)
+	columns := make([]*column.Col, len(e.scan.fields))
+	for i, v := range e.scan.fields {
+		if v.Referenced {
+			columns[i] = e.scan.tbl.Cols()[i]
+		}
+	}
+	row.Data, err = e.scan.tbl.RowWithCols(e.scan.ctx, h, columns)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}

executor/executor_xapi.go

@@ -67,10 +67,16 @@ func (e *XSelectTableExec) Next() (*Row, error) {
 			e.subResult = nil
 			continue
 		}
+		fullRowData := make([]types.Datum, len(e.tablePlan.Fields()))
+		var j int
 		for i, field := range e.tablePlan.Fields() {
-			field.Expr.SetDatum(rowData[i])
+			if field.Referenced {
+				fullRowData[i] = rowData[j]
+				field.Expr.SetDatum(rowData[j])
+				j++
+			}
 		}
-		return resultRowToRow(e.table, h, rowData), nil
+		return resultRowToRow(e.table, h, fullRowData), nil
 	}
 }
 
@@ -104,7 +110,14 @@ func (e *XSelectTableExec) doRequest() error {
 	selReq.Fields = resultFieldsToPBExpression(e.tablePlan.Fields())
 	selReq.Where = conditionsToPBExpression(e.tablePlan.FilterConditions...)
 	selReq.Ranges = tableRangesToPBRanges(e.tablePlan.Ranges)
-	selReq.TableInfo = tablecodec.TableToProto(e.tablePlan.Table)
+
+	referenced := make([]bool, len(e.tablePlan.Fields()))
+	for i, v := range e.tablePlan.Fields() {
+		if v.Referenced {
+			referenced[i] = true
+		}
+	}
+	selReq.TableInfo = tablecodec.TableToProto(e.tablePlan.Table, referenced)
 	e.result, err = xapi.Select(txn.GetClient(), selReq, 1)
 	if err != nil {
 		return errors.Trace(err)
@@ -179,7 +192,10 @@ func (e *XSelectIndexExec) doRequest() error {
 
 	sort.Sort(int64Slice(handles))
 	tblResult, err := e.doTableRequest(txn, handles)
-	rows, err := extractRowsFromTableResult(e.table, tblResult)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	rows, err := e.extractRowsFromTableResult(e.table, tblResult)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -222,7 +238,13 @@ func (e *XSelectIndexExec) doTableRequest(txn kv.Transaction, handles []int64) (
 	selTableReq := new(tipb.SelectRequest)
 	startTs := txn.StartTS()
 	selTableReq.StartTs = &startTs
-	selTableReq.TableInfo = tablecodec.TableToProto(e.indexPlan.Table)
+	referenced := make([]bool, len(e.indexPlan.Fields()))
+	for i, v := range e.indexPlan.Fields() {
+		if v.Referenced {
+			referenced[i] = true
+		}
+	}
+	selTableReq.TableInfo = tablecodec.TableToProto(e.indexPlan.Table, referenced)
 	selTableReq.Fields = resultFieldsToPBExpression(e.indexPlan.Fields())
 	for _, h := range handles {
 		if h == math.MaxInt64 {
@@ -381,7 +403,7 @@ func extractHandlesFromIndexSubResult(subResult *xapi.SubResult) ([]int64, error
 	return handles, nil
 }
 
-func extractRowsFromTableResult(t table.Table, tblResult *xapi.SelectResult) ([]*Row, error) {
+func (e *XSelectIndexExec) extractRowsFromTableResult(t table.Table, tblResult *xapi.SelectResult) ([]*Row, error) {
 	var rows []*Row
 	for {
 		subResult, err := tblResult.Next()
@@ -391,7 +413,7 @@ func extractRowsFromTableResult(t table.Table, tblResult *xapi.SelectResult) ([]
 		if subResult == nil {
 			break
 		}
-		subRows, err := extractRowsFromSubResult(t, subResult)
+		subRows, err := e.extractRowsFromSubResult(t, subResult)
 		if err != nil {
 			return nil, errors.Trace(err)
 		}
@@ -400,7 +422,7 @@ func extractRowsFromTableResult(t table.Table, tblResult *xapi.SelectResult) ([]
 	return rows, nil
 }
 
-func extractRowsFromSubResult(t table.Table, subResult *xapi.SubResult) ([]*Row, error) {
+func (e *XSelectIndexExec) extractRowsFromSubResult(t table.Table, subResult *xapi.SubResult) ([]*Row, error) {
 	var rows []*Row
 	for {
 		h, rowData, err := subResult.Next()
@@ -410,7 +432,16 @@ func extractRowsFromSubResult(t table.Table, subResult *xapi.SubResult) ([]*Row,
 		if rowData == nil {
 			break
 		}
-		row := resultRowToRow(t, h, rowData)
+		fullRowData := make([]types.Datum, len(e.indexPlan.Fields()))
+		var j int
+		for i, field := range e.indexPlan.Fields() {
+			if field.Referenced {
+				fullRowData[i] = rowData[j]
+				field.Expr.SetDatum(fullRowData[i])
+				j++
+			}
+		}
+		row := resultRowToRow(t, h, fullRowData)
 		rows = append(rows, row)
 	}
 	return rows, nil

optimizer/plan/planbuilder.go

@@ -737,6 +737,9 @@ func (b *planBuilder) buildDistinct(src Plan) Plan {
 func (b *planBuilder) buildUpdate(update *ast.UpdateStmt) Plan {
 	sel := &ast.SelectStmt{From: update.TableRefs, Where: update.Where, OrderBy: update.Order, Limit: update.Limit}
 	p := b.buildFrom(sel)
+	for _, v := range p.Fields() {
+		v.Referenced = true
+	}
 	if sel.OrderBy != nil && !matchOrder(p, sel.OrderBy.Items) {
 		p = b.buildSort(p, sel.OrderBy.Items)
 		if b.err != nil {
@@ -772,6 +775,9 @@ func (b *planBuilder) buildUpdateLists(list []*ast.Assignment, fields []*ast.Res
 func (b *planBuilder) buildDelete(del *ast.DeleteStmt) Plan {
 	sel := &ast.SelectStmt{From: del.TableRefs, Where: del.Where, OrderBy: del.Order, Limit: del.Limit}
 	p := b.buildFrom(sel)
+	for _, v := range p.Fields() {
+		v.Referenced = true
+	}
 	if sel.OrderBy != nil && !matchOrder(p, sel.OrderBy.Items) {
 		p = b.buildSort(p, sel.OrderBy.Items)
 		if b.err != nil {

optimizer/resolver.go

@@ -609,6 +609,7 @@ func (nr *nameResolver) resolveColumnInTableSources(cn *ast.ColumnNameExpr, tabl
 	if matchedResultField != nil {
 		// Bind column.
 		cn.Refer = matchedResultField
+		matchedResultField.Referenced = true
 		return true
 	}
 	return false
@@ -638,6 +639,7 @@ func (nr *nameResolver) resolveColumnInResultFields(ctx *resolverContext, cn *as
 			if rf.Column.Name.L == "" {
 				// This is not a real table column, resolve it directly.
 				cn.Refer = rf
+				rf.Referenced = true
 				return true
 			}
 			if matched == nil {
@@ -664,6 +666,7 @@ func (nr *nameResolver) resolveColumnInResultFields(ctx *resolverContext, cn *as
 		}
 		// Bind column.
 		cn.Refer = matched
+		matched.Referenced = true
 		return true
 	}
 	return false
@@ -714,6 +717,7 @@ func (nr *nameResolver) createResultFields(field *ast.SelectField) (rfs []*ast.R
 
 		}
 		for _, trf := range tableRfs {
+			trf.Referenced = true
 			// Convert it to ColumnNameExpr
 			cn := &ast.ColumnName{
 				Schema: trf.DBName,
@@ -799,6 +803,7 @@ func (nr *nameResolver) handlePosition(pos *ast.PositionExpr) {
 	}
 	nf.Expr = expr
 	pos.Refer = &nf
+	pos.Refer.Referenced = true
 	if nr.currentContext().inGroupBy {
 		// make sure item is not aggregate function
 		if ast.HasAggFlag(pos.Refer.Expr) {

table/tables/memory_tables.go

@@ -216,6 +216,9 @@ func (t *MemoryTable) RowWithCols(ctx context.Context, h int64, cols []*column.C
 	row := item.(*itemPair).data
 	v := make([]types.Datum, len(cols))
 	for i, col := range cols {
+		if col == nil {
+			continue
+		}
 		v[i] = row[col.Offset]
 	}
 	return v, nil

table/tables/tables.go

@@ -437,6 +437,9 @@ func (t *Table) RowWithCols(ctx context.Context, h int64, cols []*column.Col) ([
 	}
 	v := make([]types.Datum, len(cols))
 	for i, col := range cols {
+		if col == nil {
+			continue
+		}
 		if col.State != model.StatePublic {
 			return nil, errors.Errorf("Cannot use none public column - %v", cols)
 		}

xapi/tablecodec/tablecodec.go

@@ -85,6 +85,9 @@ func DecodeRowKey(key kv.Key) (handle int64, err error) {
 
 // DecodeValues decodes a byte slice into datums with column types.
 func DecodeValues(data []byte, fts []*types.FieldType, inIndex bool) ([]types.Datum, error) {
+	if data == nil {
+		return nil, nil
+	}
 	values, err := codec.Decode(data)
 	if err != nil {
 		return nil, errors.Trace(err)
@@ -216,12 +219,15 @@ func collationToProto(c string) int32 {
 }
 
 // TableToProto converts a model.TableInfo to a tipb.TableInfo.
-func TableToProto(t *model.TableInfo) *tipb.TableInfo {
+func TableToProto(t *model.TableInfo, referenced []bool) *tipb.TableInfo {
 	pt := &tipb.TableInfo{
 		TableId: proto.Int64(t.ID),
 	}
 	cols := make([]*tipb.ColumnInfo, 0, len(t.Columns))
-	for _, c := range t.Columns {
+	for i, c := range t.Columns {
+		if !referenced[i] {
+			continue
+		}
 		if c.State != model.StatePublic {
 			continue
 		}

xapi/xapi.go

@@ -92,6 +92,12 @@ func (r *SubResult) Next() (handle int64, data []types.Datum, err error) {
 	if err != nil {
 		return 0, nil, errors.Trace(err)
 	}
+	if data == nil {
+		// When no column is referenced, the data may be nil, like 'select count(*) from t'.
+		// In this case, we need to create a zero length datum slice,
+		// as caller will check if data is nil to finish iteration.
+		data = make([]types.Datum, 0)
+	}
 	handleBytes := row.GetHandle()
 	datums, err := codec.Decode(handleBytes)
 	if err != nil {