parser: fix wrong offset for token inner mysql comment (pingcap#2141)

parser/lexer.go

@@ -39,7 +39,12 @@ type Scanner struct {
 	stmtStartPos int
 
 	// for scanning such kind of comment: /*! MySQL-specific code */
-	specialComment *Scanner
+	specialComment *specialCommentScanner
+}
+
+type specialCommentScanner struct {
+	*Scanner
+	Pos
 }
 
 // Errors returns the errors during a scan.
@@ -139,8 +144,9 @@ func (s *Scanner) scan() (tok int, pos Pos, lit string) {
 		tok, pos, lit = specialComment.scan()
 		if tok != 0 {
 			// return the specialComment scan result as the result
-			pos.Line += s.r.p.Line
-			pos.Offset += s.r.p.Col
+			pos.Line += s.specialComment.Line
+			pos.Col += s.specialComment.Col
+			pos.Offset += s.specialComment.Offset
 			return
 		}
 		// leave specialComment scan mode after all stream consumed.
@@ -262,7 +268,14 @@ func startWithSlash(s *Scanner) (tok int, pos Pos, lit string) {
 		comment := s.r.data(&pos)
 		if strings.HasPrefix(comment, "/*!") {
 			sql := specCodePattern.ReplaceAllStringFunc(comment, trimComment)
-			s.specialComment = NewScanner(sql)
+			s.specialComment = &specialCommentScanner{
+				Scanner: NewScanner(sql),
+				Pos: Pos{
+					pos.Line,
+					pos.Col,
+					pos.Offset + sqlOffsetInComment(comment),
+				},
+			}
 		}
 
 		return s.scan()
@@ -271,6 +284,24 @@ func startWithSlash(s *Scanner) (tok int, pos Pos, lit string) {
 	return
 }
 
+func sqlOffsetInComment(comment string) int {
+	// find the first SQL token offset in pattern like "/*!40101 mysql specific code */"
+	offset := 0
+	for i := 0; i < len(comment); i++ {
+		if unicode.IsSpace(rune(comment[i])) {
+			offset = i
+			break
+		}
+	}
+	for offset < len(comment) {
+		offset++
+		if !unicode.IsSpace(rune(comment[offset])) {
+			break
+		}
+	}
+	return offset
+}
+
 func startWithAt(s *Scanner) (tok int, pos Pos, lit string) {
 	pos = s.r.pos()
 	s.r.inc()

parser/lexer_test.go

@@ -204,3 +204,18 @@ func (s *testLexerSuite) TestIdentifier(c *C) {
 		c.Assert(v.ident, Equals, item[1])
 	}
 }
+
+func (s *testLexerSuite) TestSpecialComment(c *C) {
+	l := NewScanner("/*!40101 select\n5*/")
+	tok, pos, lit := l.scan()
+	fmt.Println(tok, pos, lit)
+	c.Assert(tok, Equals, identifier)
+	c.Assert(lit, Equals, "select")
+	c.Assert(pos, Equals, Pos{0, 0, 9})
+
+	tok, pos, lit = l.scan()
+	fmt.Println(tok, pos, lit)
+	c.Assert(tok, Equals, intLit)
+	c.Assert(lit, Equals, "5")
+	c.Assert(pos, Equals, Pos{1, 1, 16})
+}