diff --git a/src/nimly/lexer.nim b/src/nimly/lexer.nim index e4aa01c..590022b 100644 --- a/src/nimly/lexer.nim +++ b/src/nimly/lexer.nim @@ -106,6 +106,9 @@ proc lex*[T](nl: var NimlLexer[T]): T = except: discard +proc isEmpty*[T](nl: NimlLexer[T]): bool = + nl.buf[nl.bufpos] == EndOfFile + proc lexNext*[T](nl: var NimlLexer[T]): T = while nl.buf[nl.bufpos] != EndOfFile: result = nl.lex diff --git a/src/nimly/parser.nim b/src/nimly/parser.nim index 253bd71..47edb44 100644 --- a/src/nimly/parser.nim +++ b/src/nimly/parser.nim @@ -110,8 +110,13 @@ proc top[T](parser: Parser[T]): State = proc parseImpl*[T, S](parser: var Parser[S], lexer: var NimlLexer[T]): ParseTree[T, S] = var tree: seq[ParseTree[T, S]] = @[] - var token = lexer.lexNext - var symbol = TermS[S](token.kind) + var token: T + var symbol: Symbol[S] + if lexer.isEmpty: + symbol = End[S]() + else: + token = lexer.lexNext + symbol = TermS[S](token.kind) while true: when defined(nimydebug): echo "parser stack:" & $parser.stack diff --git a/tests/test_empty_str_does_not_cause_error.nim b/tests/test_empty_str_does_not_cause_error.nim new file mode 100644 index 0000000..aa3645c --- /dev/null +++ b/tests/test_empty_str_does_not_cause_error.nim @@ -0,0 +1,34 @@ +import unittest +import patty + +import nimly + +variant Token: + CHARS(val: string) + IGNORE + +niml testLex[Token]: + r"\w+": + return CHARS(token.token) + r"\s": + return IGNORE() + +nimy testPar[Token]: + top[seq[string]]: + word{}: + return $1 + word[string]: + CHARS: + return ($1).val + +test "parser works": + var testLexer = testLex.newWithString("This is a test") + testLexer.ignoreIf = proc(r: Token): bool = r.kind == TokenKind.IGNORE + var parser = testPar.newParser() + check parser.parse(testLexer) == @["This", "is", "a", "test"] + +test "empty string does not cause error": + var testLexer = testLex.newWithString("") + testLexer.ignoreIf = proc(r: Token): bool = r.kind == TokenKind.IGNORE + var parser = testPar.newParser() + check parser.parse(testLexer).len == 0