mime: derestrict value backslash unescaping for all encodings

Previously consumeValue performed consumption of "unnecessary backslashes"
strictly for non-ASCII and non-token runes. Thus if it encountered a
backslash before a rune that is out of the ASCII range, it would
erroneously skip that backslash. This change now derestricts
"unnecessary backslash" unescaping for all character encodings,
using "isTSpecial" instead of "!isTokenChar".
This change is a follow-up of CL 32175.

Fixes golang#25888

Change-Id: I5e02bbf9c42f753a6eb31399b8d20315af991490
Reviewed-on: https://go-review.googlesource.com/119795
Reviewed-by: Brad Fitzpatrick <[email protected]>

src/mime/mediatype.go

@@ -280,7 +280,7 @@ func consumeValue(v string) (value, rest string) {
 		// and intended as a literal backslash. This makes Go servers deal better
 		// with MSIE without affecting the way they handle conforming MIME
 		// generators.
-		if r == '\\' && i+1 < len(v) && !isTokenChar(rune(v[i+1])) {
+		if r == '\\' && i+1 < len(v) && isTSpecial(rune(v[i+1])) {
 			buffer.WriteByte(v[i+1])
 			i++
 			continue

src/mime/mediatype_test.go

@@ -40,6 +40,8 @@ func TestConsumeValue(t *testing.T) {
 		{`"\\" rest`, "\\", " rest"},
 		{`"My \" value"end`, "My \" value", "end"},
 		{`"\" rest`, "", `"\" rest`},
+		{`"C:\dev\go\robots.txt"`, `C:\dev\go\robots.txt`, ""},
+		{`"C:\新建文件件\中文第二次测试.mp4"`, `C:\新建文件件\中文第二次测试.mp4`, ""},
 	}
 	for _, test := range tests {
 		value, rest := consumeValue(test[0])
@@ -393,6 +395,7 @@ func TestParseMediaType(t *testing.T) {
 
 		// Microsoft browers in intranet mode do not think they need to escape \ in file name.
 		{`form-data; name="file"; filename="C:\dev\go\robots.txt"`, "form-data", m("name", "file", "filename", `C:\dev\go\robots.txt`)},
+		{`form-data; name="file"; filename="C:\新建文件件\中文第二次测试.mp4"`, "form-data", m("name", "file", "filename", `C:\新建文件件\中文第二次测试.mp4`)},
 	}
 	for _, test := range tests {
 		mt, params, err := ParseMediaType(test.in)