Lexer: Handle triple quoted raw strings (e.g. r"""abc""").

-- MOS_MIGRATED_REVID=105956734
vanwaals · Oct 21, 2015 · 4d7fae3 · 4d7fae3
1 parent 5d9b8a0
commit 4d7fae3
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 9 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
@@ -312,7 +312,7 @@ private boolean skipTripleQuote(char quot) {
    *
    * @return the string-literal token.
    */
-  private Token escapedStringLiteral(char quot) {
+  private Token escapedStringLiteral(char quot, boolean isRaw) {
     boolean inTriplequote = skipTripleQuote(quot);
 
     int oldPos = pos - 1;
@@ -336,6 +336,14 @@ private Token escapedStringLiteral(char quot) {
             error("unterminated string literal at eof", oldPos, pos);
             return new Token(TokenKind.STRING, oldPos, pos, literal.toString());
           }
+          if (isRaw) {
+            // Insert \ and the following character.
+            // As in Python, it means that a raw string can never end with a single \.
+            literal.append('\\');
+            literal.append(buffer[pos]);
+            pos++;
+            break;
+          }
           c = buffer[pos];
           pos++;
           switch (c) {
@@ -428,7 +436,7 @@ private Token stringLiteral(char quot, boolean isRaw) {
     // Don't even attempt to parse triple-quotes here.
     if (skipTripleQuote(quot)) {
       pos -= 2;
-      return escapedStringLiteral(quot);
+      return escapedStringLiteral(quot, isRaw);
     }
 
     // first quick optimistic scan for a simple non-escaped string
@@ -446,11 +454,10 @@ private Token stringLiteral(char quot, boolean isRaw) {
             // skip the next character
             pos++;
             break;
-          } else {
-            // oops, hit an escape, need to start over & build a new string buffer
-            pos = oldPos + 1;
-            return escapedStringLiteral(quot);
           }
+          // oops, hit an escape, need to start over & build a new string buffer
+          pos = oldPos + 1;
+          return escapedStringLiteral(quot, false);
         case '\'':
         case '"':
           if (c == quot) {

diff --git a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
@@ -247,7 +247,14 @@ public void testStringEscapes() throws Exception {
     assertEquals("STRING(\\$$) NEWLINE EOF", values(tokens("'\\$$'")));
     assertEquals("STRING(ab) NEWLINE EOF",
                  values(tokens("'a\\\nb'"))); // escape end of line
+    assertEquals("STRING(abcd) NEWLINE EOF",
+                 values(tokens("\"ab\\ucd\"")));
+    assertEquals("/some/path.txt:1: escape sequence not implemented: \\u",
+                 lastError.toString());
+  }
 
+  @Test
+  public void testRawString() throws Exception {
     assertEquals("STRING(abcd) NEWLINE EOF",
                  values(tokens("r'abcd'")));
     assertEquals("STRING(abcd) NEWLINE EOF",
@@ -261,9 +268,26 @@ public void testStringEscapes() throws Exception {
     assertEquals("STRING(ab) IDENTIFIER(r) NEWLINE EOF",
                  values(tokens("r'ab'r")));
 
-    assertEquals("STRING(abcd) NEWLINE EOF",
-                 values(tokens("\"ab\\ucd\"")));
-    assertEquals("/some/path.txt:1: escape sequence not implemented: \\u",
+    // Unterminated raw string
+    values(tokens("r'\\'")); // r'\'
+    assertEquals("/some/path.txt:1: unterminated string literal at eof",
+                 lastError.toString());
+  }
+
+  @Test
+  public void testTripleRawString() throws Exception {
+    // r'''a\ncd'''
+    assertEquals("STRING(ab\\ncd) NEWLINE EOF",
+                 values(tokens("r'''ab\\ncd'''")));
+    // r"""ab
+    // cd"""
+    assertEquals(
+        "STRING(ab\ncd) NEWLINE EOF",
+        values(tokens("\"\"\"ab\ncd\"\"\"")));
+
+    // Unterminated raw string
+    values(tokens("r'''\\'''")); // r'''\'''
+    assertEquals("/some/path.txt:1: unterminated string literal at eof",
                  lastError.toString());
   }