planner: initialize charset, collation and repertoire correctly for p…

…arameter values (pingcap#42645)

ref pingcap#36598

expression/builtin_other_test.go

@@ -141,7 +141,7 @@ func TestGetVar(t *testing.T) {
 		} else {
 			tp = types.NewFieldType(mysql.TypeVarString)
 		}
-		types.DefaultParamTypeForValue(kv.val, tp)
+		types.InferParamTypeFromUnderlyingValue(kv.val, tp)
 		ctx.GetSessionVars().SetUserVarType(kv.key, tp)
 	}
 

expression/constant.go

@@ -132,7 +132,7 @@ func (c *Constant) GetType() *types.FieldType {
 		// so it should avoid data race. We achieve this by returning different FieldType pointer for each call.
 		tp := types.NewFieldType(mysql.TypeUnspecified)
 		dt := c.ParamMarker.GetUserVar()
-		types.DefaultParamTypeForValue(dt.GetValue(), tp)
+		types.InferParamTypeFromDatum(&dt, tp)
 		return tp
 	}
 	return c.RetType

expression/util.go

@@ -1002,7 +1002,7 @@ func ParamMarkerExpression(ctx sessionctx.Context, v *driver.ParamMarkerExpr, ne
 	useCache := ctx.GetSessionVars().StmtCtx.UseCache
 	isPointExec := ctx.GetSessionVars().StmtCtx.PointExec
 	tp := types.NewFieldType(mysql.TypeUnspecified)
-	types.DefaultParamTypeForValue(v.GetValue(), tp)
+	types.InferParamTypeFromDatum(&v.Datum, tp)
 	value := &Constant{Value: v.Datum, RetType: tp}
 	if useCache || isPointExec || needParam {
 		value.ParamMarker = &ParamMarker{

planner/core/expression_rewriter.go

@@ -1111,6 +1111,19 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, v *ast.S
 	return v, true
 }
 
+func initConstantRepertoire(c *expression.Constant) {
+	c.SetRepertoire(expression.ASCII)
+	if c.GetType().EvalType() == types.ETString {
+		for _, b := range c.Value.GetBytes() {
+			// if any character in constant is not ascii, set the repertoire to UNICODE.
+			if b >= 0x80 {
+				c.SetRepertoire(expression.UNICODE)
+				break
+			}
+		}
+	}
+}
+
 // Leave implements Visitor interface.
 func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok bool) {
 	if er.err != nil {
@@ -1134,23 +1147,15 @@ func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok
 		}
 		v.Datum.SetValue(v.Datum.GetValue(), retType)
 		value := &expression.Constant{Value: v.Datum, RetType: retType}
-		value.SetRepertoire(expression.ASCII)
-		if retType.EvalType() == types.ETString {
-			for _, b := range v.Datum.GetBytes() {
-				// if any character in constant is not ascii, set the repertoire to UNICODE.
-				if b >= 0x80 {
-					value.SetRepertoire(expression.UNICODE)
-					break
-				}
-			}
-		}
+		initConstantRepertoire(value)
 		er.ctxStackAppend(value, types.EmptyName)
 	case *driver.ParamMarkerExpr:
-		var value expression.Expression
+		var value *expression.Constant
 		value, er.err = expression.ParamMarkerExpression(er.sctx, v, false)
 		if er.err != nil {
 			return retNode, false
 		}
+		initConstantRepertoire(value)
 		er.ctxStackAppend(value, types.EmptyName)
 	case *ast.VariableExpr:
 		er.rewriteVariable(v)

planner/core/plan_cache_param.go

@@ -136,7 +136,7 @@ func Params2Expressions(params []*driver.ValueExpr) []expression.Expression {
 	exprs := make([]expression.Expression, 0, len(params))
 	for _, p := range params {
 		tp := new(types.FieldType)
-		types.DefaultParamTypeForValue(p.Datum.GetValue(), tp)
+		types.InferParamTypeFromDatum(&p.Datum, tp)
 		exprs = append(exprs, &expression.Constant{
 			Value:   p.Datum,
 			RetType: tp,

planner/core/plan_cache_test.go

@@ -1684,6 +1684,19 @@ func TestNonPreparedPlanCachePanic(t *testing.T) {
 	}
 }
 
+func TestNonPreparedPlanCacheUnicode(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+
+	tk.MustExec("use test")
+	tk.MustExec(`set tidb_enable_non_prepared_plan_cache=1`)
+	tk.MustExec(`create table t1 (a varchar(10) character set latin1, b int)`)
+	tk.MustExec(`insert into t1 values ('a',1)`)
+	tk.MustQuery(`select concat(a, if(b>10, N'x', N'y')) from t1`).Check(testkit.Rows("ay")) // no error
+	tk.MustQuery(`select concat(a, if(b>10, N'x', N'y')) from t1`).Check(testkit.Rows("ay"))
+	tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1"))
+}
+
 func TestNonPreparedPlanCacheBuiltinFuncs(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)

server/conn_stmt.go

@@ -676,7 +676,7 @@ func parseExecArgs(sc *stmtctx.StatementContext, params []expression.Expression,
 
 	for i := range params {
 		ft := new(types.FieldType)
-		types.DefaultParamTypeForValue(args[i].GetValue(), ft)
+		types.InferParamTypeFromUnderlyingValue(args[i].GetValue(), ft)
 		params[i] = &expression.Constant{Value: args[i], RetType: ft}
 	}
 	return

types/etc.go

@@ -133,6 +133,11 @@ func IsString(tp byte) bool {
 	return IsTypeChar(tp) || IsTypeBlob(tp) || IsTypeVarchar(tp) || IsTypeUnspecified(tp)
 }
 
+// IsStringKind returns a boolean indicating whether the tp is a string type.
+func IsStringKind(kind byte) bool {
+	return kind == KindString || kind == KindBytes
+}
+
 var kind2Str = map[byte]string{
 	KindNull:          "null",
 	KindInt64:         "bigint",

types/field_type.go

@@ -176,8 +176,22 @@ func SetTypeFlag(flag *uint, flagItem uint, on bool) {
 	}
 }
 
-// DefaultParamTypeForValue returns the default FieldType for the parameterized value.
-func DefaultParamTypeForValue(value interface{}, tp *FieldType) {
+// InferParamTypeFromDatum is used for plan cache to infer the type of a parameter from its datum.
+func InferParamTypeFromDatum(d *Datum, tp *FieldType) {
+	InferParamTypeFromUnderlyingValue(d.GetValue(), tp)
+	if IsStringKind(d.k) {
+		// consider charset and collation here
+		c, err := collate.GetCollationByName(d.collation)
+		if err != nil || c == nil {
+			return // use default charset and collation
+		}
+		tp.SetCharset(c.CharsetName)
+		tp.SetCollate(d.collation)
+	}
+}
+
+// InferParamTypeFromUnderlyingValue is used for plan cache to infer the type of a parameter from its underlying value.
+func InferParamTypeFromUnderlyingValue(value interface{}, tp *FieldType) {
 	switch value.(type) {
 	case nil:
 		tp.SetType(mysql.TypeVarString)