Merge pull request influxdata#17 from influxdb/nc-issue-8

add single quoted strings and allow for escaping quotes in strings

tick/lex.go

@@ -192,10 +192,10 @@ func lexToken(l *lexer) stateFn {
 		case unicode.IsLetter(r):
 			return lexIdent
 		case r == '"':
-			return lexString
+			return lexDoubleString
 		case r == '\'':
 			l.backup()
-			return lexTripleString
+			return lexSingleOrTripleString
 		case isSpace(r):
 			l.ignore()
 		case r == '/':
@@ -308,9 +308,13 @@ func lexNumberOrDuration(l *lexer) stateFn {
 	}
 }
 
-func lexString(l *lexer) stateFn {
+func lexDoubleString(l *lexer) stateFn {
 	for {
 		switch r := l.next(); r {
+		case '\\':
+			if l.peek() == '"' {
+				l.next()
+			}
 		case '"':
 			l.emit(tokenString)
 			return lexToken
@@ -320,25 +324,44 @@ func lexString(l *lexer) stateFn {
 	}
 }
 
-func lexTripleString(l *lexer) stateFn {
+func lexSingleOrTripleString(l *lexer) stateFn {
 	count := 0
+	total := 0
 	for {
 		switch r := l.next(); r {
 		case '\'':
 			count++
-			if count == 3 {
+			if count == 1 && l.peek() == '\'' {
+				// check for empty '' string
+				count++
+				l.next()
+				if l.peek() == '\'' {
+					count++
+					l.next()
+				} else {
+					l.emit(tokenString)
+					return lexToken
+				}
+			}
+			if (count == 1 && l.peek() != '\'') || count == 3 {
+				total = count
 				for {
-					switch r := l.next(); r {
-					case '\'':
+					switch r := l.next(); {
+					//escape single quotes if single quoted
+					case r == '\\' && count == 1:
+						if l.peek() == '\'' {
+							l.next()
+						}
+					case r == '\'':
 						count--
 						if count == 0 {
 							l.emit(tokenString)
 							return lexToken
 						}
-					case eof:
+					case r == eof:
 						return l.errorf("unterminated string")
 					default:
-						count = 3
+						count = total
 					}
 				}
 			}

tick/lex_test.go

@@ -215,6 +215,77 @@ func TestLexer(t *testing.T) {
 				token{tokenEOF, 7, ""},
 			},
 		},
+		//Strings
+		testCase{
+			in: `''`,
+			tokens: []token{
+				token{tokenString, 0, `''`},
+				token{tokenEOF, 2, ""},
+			},
+		},
+		testCase{
+			in: `""`,
+			tokens: []token{
+				token{tokenString, 0, `""`},
+				token{tokenEOF, 2, ""},
+			},
+		},
+		testCase{
+			in: `''''''`,
+			tokens: []token{
+				token{tokenString, 0, `''''''`},
+				token{tokenEOF, 6, ""},
+			},
+		},
+		testCase{
+			in: `'str'`,
+			tokens: []token{
+				token{tokenString, 0, `'str'`},
+				token{tokenEOF, 5, ""},
+			},
+		},
+		testCase{
+			in: `"str"`,
+			tokens: []token{
+				token{tokenString, 0, `"str"`},
+				token{tokenEOF, 5, ""},
+			},
+		},
+		testCase{
+			in: `'str\''`,
+			tokens: []token{
+				token{tokenString, 0, `'str\''`},
+				token{tokenEOF, 7, ""},
+			},
+		},
+		testCase{
+			in: `"str\""`,
+			tokens: []token{
+				token{tokenString, 0, `"str\""`},
+				token{tokenEOF, 7, ""},
+			},
+		},
+		testCase{
+			in: `'''s'tr'''`,
+			tokens: []token{
+				token{tokenString, 0, `'''s'tr'''`},
+				token{tokenEOF, 10, ""},
+			},
+		},
+		testCase{
+			in: `'''s\'tr'''`,
+			tokens: []token{
+				token{tokenString, 0, `'''s\'tr'''`},
+				token{tokenEOF, 11, ""},
+			},
+		},
+		testCase{
+			in: `'''str'''`,
+			tokens: []token{
+				token{tokenString, 0, `'''str'''`},
+				token{tokenEOF, 9, ""},
+			},
+		},
 		//Space
 		testCase{
 			in: " ",

tick/node.go

@@ -1,6 +1,7 @@
 package tick
 
 import (
+	"bytes"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -363,11 +364,26 @@ type stringNode struct {
 
 func newString(p int, txt string) *stringNode {
 
+	// Remove leading and trailing quotes
 	var literal string
-	if txt[0] == '\'' {
+	if len(txt) >= 6 && txt[0:3] == "'''" {
 		literal = txt[3 : len(txt)-3]
 	} else {
 		literal = txt[1 : len(txt)-1]
+		quote := txt[0]
+		// Unescape quotes
+		var buf bytes.Buffer
+		buf.Grow(len(literal))
+		last := 0
+		for i := 0; i < len(literal)-1; i++ {
+			if literal[i] == '\\' && literal[i+1] == quote {
+				buf.Write([]byte(literal[last:i]))
+				i++
+				last = i
+			}
+		}
+		buf.Write([]byte(literal[last:]))
+		literal = buf.String()
 	}
 
 	return &stringNode{

tick/parser_test.go

@@ -59,6 +59,70 @@ func TestParseErrors(t *testing.T) {
 		test(tc)
 	}
 }
+func TestParseStrings(t *testing.T) {
+	assert := assert.New(t)
+
+	testCases := []struct {
+		script  string
+		literal string
+	}{
+		{
+			script:  `f('a')`,
+			literal: "a",
+		},
+		{
+			script:  `f("a")`,
+			literal: "a",
+		},
+		{
+			script:  `f('''a''')`,
+			literal: "a",
+		},
+		{
+			script:  `f('a\'')`,
+			literal: "a'",
+		},
+		{
+			script:  `f("a\"")`,
+			literal: "a\"",
+		},
+		{
+			script:  `f('''\'a''')`,
+			literal: `\'a`,
+		},
+	}
+
+	for _, tc := range testCases {
+		tree, err := parse(tc.script)
+		assert.Nil(err)
+
+		//Assert we have a list of one statement
+		l, ok := tree.Root.(*listNode)
+		if !assert.True(ok, "tree.Root is not a list node") {
+			t.FailNow()
+		}
+		if !assert.Equal(1, len(l.Nodes), "Did not get exactly one node in statement list") {
+			t.FailNow()
+		}
+
+		//Assert the first statement is a function with one stringNode argument
+		f, ok := l.Nodes[0].(*funcNode)
+		if !assert.True(ok, "first statement is not a func node %q", l.Nodes[0]) {
+			t.FailNow()
+		}
+		if !assert.Equal(1, len(f.Args), "unexpected number of args got %d exp %d", len(f.Args), 1) {
+			t.FailNow()
+		}
+		str, ok := f.Args[0].(*stringNode)
+		if !assert.True(ok, "first argument is not a string node %q", f.Args[0]) {
+			t.FailNow()
+		}
+
+		// Assert strings literals are equal
+		assert.Equal(tc.literal, str.Literal)
+	}
+
+}
 
 func TestParseSingleStmt(t *testing.T) {
 	assert := assert.New(t)