fix #74 (parser cannot parse no lex tokens)

src/nimly/lexer.nim

@@ -106,6 +106,9 @@ proc lex*[T](nl: var NimlLexer[T]): T =
     except:
       discard
 
+proc isEmpty*[T](nl: NimlLexer[T]): bool =
+  nl.buf[nl.bufpos] == EndOfFile
+
 proc lexNext*[T](nl: var NimlLexer[T]): T =
   while nl.buf[nl.bufpos] != EndOfFile:
     result = nl.lex

src/nimly/parser.nim

@@ -110,8 +110,13 @@ proc top[T](parser: Parser[T]): State =
 proc parseImpl*[T, S](parser: var Parser[S],
                       lexer: var NimlLexer[T]): ParseTree[T, S] =
   var tree: seq[ParseTree[T, S]] = @[]
-  var token = lexer.lexNext
-  var symbol = TermS[S](token.kind)
+  var token: T
+  var symbol: Symbol[S]
+  if lexer.isEmpty:
+    symbol = End[S]()
+  else:
+    token  = lexer.lexNext
+    symbol = TermS[S](token.kind)
   while true:
     when defined(nimydebug):
       echo "parser stack:" & $parser.stack

tests/test_empty_str_does_not_cause_error.nim

@@ -0,0 +1,34 @@
+import unittest
+import patty
+
+import nimly
+
+variant Token:
+  CHARS(val: string)
+  IGNORE
+
+niml testLex[Token]:
+  r"\w+":
+    return CHARS(token.token)
+  r"\s":
+    return IGNORE()
+
+nimy testPar[Token]:
+  top[seq[string]]:
+    word{}:
+      return $1
+  word[string]:
+    CHARS:
+      return ($1).val
+
+test "parser works":
+  var testLexer = testLex.newWithString("This is a test")
+  testLexer.ignoreIf = proc(r: Token): bool = r.kind == TokenKind.IGNORE
+  var parser = testPar.newParser()
+  check parser.parse(testLexer) == @["This", "is", "a", "test"]
+
+test "empty string does not cause error":
+  var testLexer = testLex.newWithString("")
+  testLexer.ignoreIf = proc(r: Token): bool = r.kind == TokenKind.IGNORE
+  var parser = testPar.newParser()
+  check parser.parse(testLexer).len == 0