fixes issues reported by mpvl emicklei#107, emicklei#109 (emicklei#110)

* fixed issue emicklei#107

* add unquote tests

* add test for issue emicklei#109

* add comment about validation and unquote

* SourceRepresentation prints the quote rune that was used to delimit the string literal

* test for returned quote rune

README.md

@@ -41,6 +41,11 @@ Package in Go for parsing Google Protocol Buffers [.proto files version 2 + 3] (
 		fmt.Println(m.Name)
 	}
 
+### validation
+
+Current parser implementation is not completely validating `.proto` definitions.
+In many but not all cases, the parser will report syntax errors when reading unexpected charaters or tokens.
+Use some linting tools (e.g. https://github.com/uber/prototool) or `protoc` for full validation.
 
 ### contributions
 

import.go

@@ -47,7 +47,7 @@ func (i *Import) parse(p *Parser) error {
 		i.Kind = lit
 		return i.parse(p)
 	case tIDENT:
-		i.Filename = unQuote(lit)
+		i.Filename, _ = unQuote(lit)
 	default:
 		return p.unexpected(lit, "import classifier weak|public|quoted", i)
 	}

option.go

@@ -125,11 +125,17 @@ type Literal struct {
 	Position scanner.Position
 	Source   string
 	IsString bool
+
+	// The rune use to delimit the string value (only valid iff IsString)
+	QuoteRune rune
+
 	// literal value can be an array literal value (even nested)
 	Array []*Literal
+
 	// literal value can be a map of literals (even nested)
 	// DEPRECATED: use OrderedMap instead
 	Map map[string]*Literal
+
 	// literal value can be a map of literals (even nested)
 	// this is done as pairs of name keys and literal values so the original ordering is preserved
 	OrderedMap LiteralMap
@@ -150,15 +156,15 @@ func (m LiteralMap) Get(key string) (*Literal, bool) {
 	return new(Literal), false
 }
 
-// SourceRepresentation returns the source (if quoted then use double quote).
+// SourceRepresentation returns the source (use the same rune that was used to delimit the string).
 func (l Literal) SourceRepresentation() string {
 	var buf bytes.Buffer
 	if l.IsString {
-		buf.WriteRune('"')
+		buf.WriteRune(l.QuoteRune)
 	}
 	buf.WriteString(l.Source)
 	if l.IsString {
-		buf.WriteRune('"')
+		buf.WriteRune(l.QuoteRune)
 	}
 	return buf.String()
 }
@@ -222,15 +228,16 @@ func (l *Literal) parse(p *Parser) error {
 	source := lit
 	iss := isString(lit)
 	if iss {
-		source = unQuote(source)
+		source, l.QuoteRune = unQuote(source)
 	}
 	l.Position, l.Source, l.IsString = pos, source, iss
 
 	// peek for multiline strings
 	for {
 		pos, tok, lit := p.next()
 		if isString(lit) {
-			l.Source += unQuote(lit)
+			line, _ := unQuote(lit)
+			l.Source += line
 		} else {
 			p.nextPut(pos, tok, lit)
 			break
@@ -331,7 +338,8 @@ func parseAggregateConstants(p *Parser, container interface{}) (list []*NamedLit
 		// workaround issue #59 TODO
 		if isString(lit) && len(list) > 0 {
 			// concatenate with previous constant
-			list[len(list)-1].Source += unQuote(lit)
+			s, _ := unQuote(lit)
+			list[len(list)-1].Source += s
 			continue
 		}
 		key := lit

option_test.go

@@ -695,3 +695,28 @@ func TestEmptyArrayInOptionStructure(t *testing.T) {
 		t.Fatalf("got [%s] want [%s]", got, want)
 	}
 }
+
+// https://github.com/emicklei/proto/issues/107
+func TestQuoteNotDroppedInOption(t *testing.T) {
+	src := `string name = 1 [ quote = '<="foo"' ];`
+	f := newNormalField()
+	if err := f.parse(newParserOn(src)); err != nil {
+		t.Fatal(err)
+	}
+	sr := f.Options[0].Constant.SourceRepresentation()
+	if got, want := sr, `'<="foo"'`; got != want {
+		t.Errorf("got [%s] want [%s]", got, want)
+	}
+}
+
+func TestWhatYouTypeIsWhatYouGetOptionValue(t *testing.T) {
+	src := `string n = 1 [ quote = 'm"\"/"' ];`
+	f := newNormalField()
+	if err := f.parse(newParserOn(src)); err != nil {
+		t.Fatal(err)
+	}
+	sr := f.Options[0].Constant.SourceRepresentation()
+	if got, want := sr, `'m"\"/"'`; got != want {
+		t.Errorf("got [%s] want [%s]", got, want)
+	}
+}

reserved.go

@@ -64,7 +64,8 @@ func (r *Reserved) parse(p *Parser) error {
 			continue
 		}
 		if isString(lit) {
-			r.FieldNames = append(r.FieldNames, unQuote(lit))
+			s, _ := unQuote(lit)
+			r.FieldNames = append(r.FieldNames, s)
 			continue
 		}
 		if tSEMICOLON == tok {

syntax.go

@@ -44,7 +44,7 @@ func (s *Syntax) parse(p *Parser) error {
 	if !isString(lit) {
 		return p.unexpected(lit, "syntax string constant", s)
 	}
-	s.Value = unQuote(lit)
+	s.Value, _ = unQuote(lit)
 	return nil
 }
 

token.go

@@ -119,25 +119,25 @@ func isComment(lit string) bool {
 	return strings.HasPrefix(lit, "//") || strings.HasPrefix(lit, "/*")
 }
 
-func unQuote(lit string) string {
-	lit = strings.TrimLeft(lit, "\"'")
-	array := []rune(lit)
-	// https://github.com/emicklei/proto/issues/103
-	// cannot use strconv.Unquote as this unescapes quotes
-	for len(array) > 0 {
-		last := array[len(array)-1]
-		if last != '\'' && last != '"' {
-			break
-		}
-		if len(array) > 1 {
-			secondLast := array[len(array)-2]
-			if secondLast == '\\' {
-				break
-			}
-		}
-		array = array[:len(array)-1]
+const doubleQuoteRune = rune('"')
+
+// unQuote removes one matching leading and trailing single or double quote.
+//
+// https://github.com/emicklei/proto/issues/103
+// cannot use strconv.Unquote as this unescapes quotes.
+func unQuote(lit string) (string, rune) {
+	if len(lit) < 2 {
+		return lit, doubleQuoteRune
+	}
+	chars := []rune(lit)
+	first, last := chars[0], chars[len(chars)-1]
+	if first != last {
+		return lit, doubleQuoteRune
+	}
+	if s := string(chars[0]); s == "\"" || s == stringWithSingleQuote {
+		return string(chars[1 : len(chars)-1]), chars[0]
 	}
-	return string(array)
+	return lit, doubleQuoteRune
 }
 
 func asToken(literal string) token {

token_test.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2019 Ernest Micklei
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+package proto
+
+import "testing"
+
+func TestUnQuoteCases(t *testing.T) {
+	singleQuoteRune := rune('\'')
+	for i, each := range []struct {
+		input, output string
+		quoteRune     rune
+	}{
+		{"thanos", "thanos", doubleQuoteRune},
+		{"`bucky`", "`bucky`", doubleQuoteRune},
+		{"'nat", "'nat", doubleQuoteRune},
+		{"'bruce'", "bruce", singleQuoteRune},
+		{"\"tony\"", "tony", doubleQuoteRune},
+		{"\"'\"\"'  -> \"\"\"\"\"\"", `'""'  -> """""`, doubleQuoteRune},
+		{`"''"`, "''", doubleQuoteRune},
+		{"''", "", singleQuoteRune},
+		{"", "", doubleQuoteRune},
+	} {
+		got, gotRune := unQuote(each.input)
+		if gotRune != each.quoteRune {
+			t.Errorf("[%d] got [%v] want [%v]", i, gotRune, each.quoteRune)
+		}
+		want := each.output
+		if got != want {
+			t.Errorf("[%d] got [%s] want [%s]", i, got, want)
+		}
+	}
+}