Skip to content

Commit

Permalink
Fix to distinguish %Q!...! (string) and %Q (mod and identifier).
Browse files Browse the repository at this point in the history
  • Loading branch information
harukasan committed Sep 30, 2016
1 parent a53c1e7 commit 5384d0b
Showing 3 changed files with 25 additions and 9 deletions.
21 changes: 12 additions & 9 deletions scanner/scanner.go
Original file line number Diff line number Diff line change
@@ -456,25 +456,23 @@ func closeBracket(c byte) byte {
case '<':
return '>'
}
return 0
return c
}

func scanPercent(s *Scanner) (token.Token, []byte) {
switch s.char {
case '{', '(', '[', '<':
term := closeBracket(s.char)
s.next()
return scanDoubleQuotedString(s, term, 2)
case 'Q':
s.next()
if s.char == '{' || s.char == '(' || s.char == '[' || s.char == '<' {
p := s.peek(2)
if p != nil && !token.IsAlnum(p[1]) { // %Q!...!
s.next()
term := closeBracket(s.char)
s.next()
return scanDoubleQuotedString(s, term, 3)
}
case 'q':
s.next()
if s.char == '{' || s.char == '(' || s.char == '[' || s.char == '<' {
p := s.peek(2)
if p != nil && !token.IsAlnum(p[1]) { // %q!...!
s.next()
term := closeBracket(s.char)
s.next()
return scanSingleQuotedString(s, term, 3)
@@ -483,6 +481,11 @@ func scanPercent(s *Scanner) (token.Token, []byte) {
s.next()
return token.AssignMod, nil
}
if s.err == nil && !token.IsAlnum(s.char) { // %!...!
term := closeBracket(s.char)
s.next()
return scanDoubleQuotedString(s, term, 2)
}
return token.Mod, nil
}

8 changes: 8 additions & 0 deletions scanner/scanner_test.go
Original file line number Diff line number Diff line change
@@ -226,12 +226,20 @@ var rules = map[string][]struct {
`"\ca"`: {{0, token.String, []byte{'\a'}}},
`"\C-a"`: {{0, token.String, []byte{'\a'}}},
"\"\\\n\"": {{0, token.String, []byte(``)}},
`%!\n!`: {{0, token.String, []byte{0x0a}}},
`%{\n}`: {{0, token.String, []byte{0x0a}}},
`%(\n)`: {{0, token.String, []byte{0x0a}}},
`%[\n]`: {{0, token.String, []byte{0x0a}}},
`%<\n>`: {{0, token.String, []byte{0x0a}}},
`%Q!\n!`: {{0, token.String, []byte{0x0a}}},
`%Q{\n}`: {{0, token.String, []byte{0x0a}}},

`1%Q`: {
{0, token.DecimalInteger, []byte(`1`)},
{1, token.Mod, nil},
{2, token.IdentConst, []byte(`Q`)},
},

`'a'`: {{0, token.String, []byte(`a`)}},
`'\''`: {{0, token.String, []byte(`'`)}},
`'\\'`: {{0, token.String, []byte(`\`)}},
5 changes: 5 additions & 0 deletions token/class.go
Original file line number Diff line number Diff line change
@@ -53,3 +53,8 @@ func IsIdentStart(c byte) bool {
func IsIdent(c byte) bool {
return IsLetter(c) || IsDecimal(c) || c == '_'
}

// IsAlnum returns whether the character is a letter or a number.
func IsAlnum(c byte) bool {
return IsDecimal(c) || IsLetter(c)
}

0 comments on commit 5384d0b

Please sign in to comment.