Merge branch 'master' into master

YQGL · Aug 11, 2016 · a261ca5 · a261ca5
2 parents 4f6168e + dd654d9
commit a261ca5
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/parser/lexer.go b/parser/lexer.go
@@ -317,9 +317,13 @@ func (mb *lazyBuf) setUseBuf(str string) {
 	}
 }
 
-func (mb *lazyBuf) writeRune(r rune) {
+func (mb *lazyBuf) writeRune(r rune, w int) {
 	if mb.useBuf {
-		mb.b.WriteRune(r)
+		if w > 1 {
+			mb.b.WriteRune(r)
+		} else {
+			mb.b.WriteByte(byte(r))
+		}
 	}
 }
 
@@ -352,7 +356,7 @@ func (s *Scanner) scanString() (tok int, pos Pos, lit string) {
 			mb.setUseBuf(mb.r.data(&pos)[1:])
 			ch0 = handleEscape(s)
 		}
-		mb.writeRune(ch0)
+		mb.writeRune(ch0, s.r.w)
 		s.r.inc()
 		ch0 = s.r.peek()
 	}
@@ -499,6 +503,9 @@ func (r *reader) peek() rune {
 		return unicode.ReplacementChar
 	case v >= 0x80:
 		v, w = utf8.DecodeRuneInString(r.s[r.p.Offset:])
+		if v == utf8.RuneError && w == 1 {
+			v = rune(r.s[r.p.Offset]) // illegal UTF-8 encoding
+		}
 	}
 	r.w = w
 	return v

diff --git a/parser/scanner_test.go b/parser/scanner_test.go
@@ -181,6 +181,8 @@ func (s *testLexerSuite) TestscanString(c *C) {
 		{`"\"hello"`, `"hello`},
 		{`'disappearing\ backslash'`, "disappearing backslash"},
 		{"'한국의中文UTF8およびテキストトラック'", "한국의中文UTF8およびテキストトラック"},
+		{"'\\a\x90'", "a\x90"},
+		{`"\aèàø»"`, `aèàø»`},
 	}
 
 	for _, v := range table {