Get path of preprocessor include.

quom/tokenizer/iterator.py

@@ -1,8 +1,5 @@
-import copy
 from typing import Union
 
-from .tokenize_error import TokenizeError
-
 
 class Iterator:
     pass
@@ -31,6 +28,9 @@ def __next__(self):
         self.it.next()
         return tmp
 
+    def __str__(self):
+        return ''.join(self)
+
 
 class Iterable:
     def __init__(self, src):

quom/tokenizer/preprocessor_tokenizer.py

@@ -1,7 +1,7 @@
 from typing import List
 
 from .comment_tokenizer import scan_for_comment
-from .iterator import LineWrapIterator
+from .iterator import LineWrapIterator, Span
 from .number_tokenizer import scan_for_number
 from .quote_tokenizer import scan_for_quote
 from .remaining_tokenizer import scan_for_remaining
@@ -17,8 +17,9 @@ def __init__(self, start, end, tokens: List[Token]):
 
 
 class PreprocessorIncludeToken(PreprocessorToken):
-    def __init__(self, start, end, tokens: List[Token]):
+    def __init__(self, start, end, tokens: List[Token], path_start, path_end):
         super().__init__(start, end, tokens)
+        self.path = Span(path_start, path_end)
 
 
 class PreprocessorPragmaToken(PreprocessorToken):
@@ -71,6 +72,9 @@ def scan_for_preprocessor_include(start: LineWrapIterator, it: LineWrapIterator,
     it = LineWrapIterator(it)
 
     if it.curr == '"':
+        it.next()
+        path_start = it.copy()
+
         # Parse until non escaped ".
         backslashes = 0
         while it.next() and it.curr != '\n' and (it.curr != '"' or backslashes % 2 != 0):
@@ -82,20 +86,25 @@ def scan_for_preprocessor_include(start: LineWrapIterator, it: LineWrapIterator,
         # Check if end of line is reached.
         if it.curr != '"':
             raise TokenizeError("Character sequence not terminated!", it)
+        path_end = it.copy()
         it.next()
 
     elif it.curr == '<':
+        it.next()
+        path_start = it.copy()
+
         # Scan until terminating >.
         while it.next() and it.curr != '\n' and it.curr != '>':
             pass
 
         # Check if end of line is reached.
         if it.curr != '>':
             raise TokenizeError("Character sequence not terminated!", it)
+        path_end = it.copy()
         it.next()
 
     scan_for_line_end(it, tokens)
-    return PreprocessorIncludeToken(start, it, tokens)
+    return PreprocessorIncludeToken(start, it, tokens, path_start, path_end)
 
 
 def scan_for_preprocessor_pragma(start: LineWrapIterator, it: LineWrapIterator, tokens: List[Token]):

quom/tokenizer/token.py

@@ -6,4 +6,4 @@ def __init__(self, start, end):
         self.span = Span(start, end) if start and end else None
 
     def __str__(self):
-        return ''.join(self.span)
+        return str(self.span)

tests/test_tokenizer.py

@@ -351,12 +351,15 @@ def test_preprocessor():
     tokens = tokenize('#include "abc" ')
     check_tokens(tokens, [PreprocessorToken])
     check_tokens(tokens, [PreprocessorIncludeToken])
+    assert str(tokens[1].path) == 'abc'
 
     tokens = tokenize('#include "abc\\"" ')
     check_tokens(tokens, [PreprocessorToken])
+    assert str(tokens[1].path) == 'abc\\"'
 
     tokens = tokenize('#include /*123*/ <abc> ')
     check_tokens(tokens, [PreprocessorToken])
+    assert str(tokens[1].path) == 'abc'
 
     with pytest.raises(TokenizeError):
         tokenize('#include "abc\\" ')