Skip to content

Commit

Permalink
All test passing, with hand patched tiger.lex.sml.
Browse files Browse the repository at this point in the history
  • Loading branch information
@nixpulvis
nixpulvis committed Jan 29, 2016
1 parent ac2c5b9 commit bffb5af
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 52 deletions.
90 changes: 46 additions & 44 deletions src/tiger.lex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,58 +19,60 @@ fun decComment() =
updateDepth ~1

%%

%s ESCAPE COMMENT STRING;
%posarg

digit = [0-9];
id = [a-zA-Z][a-zA-Z0-9_]*;
alpha = [a-zA-Z];
whitespace = [\n\t\r ];

%%
<INITIAL>type => (Tokens.TYPE(yypos, size yytext));
<INITIAL>var => (Tokens.VAR(yypos, size yytext));
<INITIAL>function => (Tokens.FUNCTION(yypos, size yytext));
<INITIAL>break => (Tokens.BREAK(yypos, size yytext));
<INITIAL>of => (Tokens.OF(yypos, size yytext));
<INITIAL>end => (Tokens.END(yypos, size yytext));
<INITIAL>in => (Tokens.IN(yypos, size yytext));
<INITIAL>nil => (Tokens.NIL(yypos, size yytext));
<INITIAL>let => (Tokens.LET(yypos, size yytext));
<INITIAL>do => (Tokens.DO(yypos, size yytext));
<INITIAL>to => (Tokens.TO(yypos, size yytext));
<INITIAL>for => (Tokens.FOR(yypos, size yytext));
<INITIAL>while => (Tokens.WHILE(yypos, size yytext));
<INITIAL>else => (Tokens.ELSE(yypos, size yytext));
<INITIAL>then => (Tokens.THEN(yypos, size yytext));
<INITIAL>if => (Tokens.IF(yypos, size yytext));
<INITIAL>array => (Tokens.ARRAY(yypos, size yytext));

<INITIAL>:\= => (Tokens.ASSIGN(yypos, size yytext));
<INITIAL>\| => (Tokens.OR(yypos, size yytext));
<INITIAL>& => (Tokens.AND(yypos, size yytext));
<INITIAL>\>\= => (Tokens.GE(yypos, size yytext));
<INITIAL>\> => (Tokens.GT(yypos, size yytext));
<INITIAL>\<\= => (Tokens.LE(yypos, size yytext));
<INITIAL>\< => (Tokens.LT(yypos, size yytext));
<INITIAL>\<\> => (Tokens.NEQ(yypos, size yytext));
<INITIAL>\= => (Tokens.EQ(yypos, size yytext));
<INITIAL>\/ => (Tokens.DIVIDE(yypos, size yytext));
<INITIAL>\* => (Tokens.TIMES(yypos, size yytext));
<INITIAL>- => (Tokens.MINUS(yypos, size yytext));
<INITIAL>\+ => (Tokens.PLUS(yypos, size yytext));
<INITIAL>\. => (Tokens.DOT(yypos, size yytext));
<INITIAL>\} => (Tokens.RBRACE(yypos, size yytext));
<INITIAL>\{ => (Tokens.LBRACE(yypos, size yytext));
<INITIAL>\] => (Tokens.RBRACK(yypos, size yytext));
<INITIAL>"\[" => (Tokens.LBRACK(yypos, size yytext));
<INITIAL>\) => (Tokens.RPAREN(yypos, size yytext));
<INITIAL>\( => (Tokens.LPAREN(yypos, size yytext));
<INITIAL>\; => (Tokens.SEMICOLON(yypos, size yytext));
<INITIAL>: => (Tokens.COLON(yypos, size yytext));
<INITIAL>, => (Tokens.COMMA(yypos, size yytext));

<INITIAL>{digit}+ => (Tokens.INT(atoi yytext, yypos, size yytext));
<INITIAL>{id} => (Tokens.ID(yytext, yypos, size yytext));
<INITIAL>type => (Tokens.TYPE(yypos - 1, yypos + size yytext - 2));
<INITIAL>var => (Tokens.VAR(yypos - 1, yypos + size yytext - 2));
<INITIAL>function => (Tokens.FUNCTION(yypos - 1, yypos + size yytext - 2));
<INITIAL>break => (Tokens.BREAK(yypos - 1, yypos + size yytext - 2));
<INITIAL>of => (Tokens.OF(yypos - 1, yypos + size yytext - 2));
<INITIAL>end => (Tokens.END(yypos - 1, yypos + size yytext - 2));
<INITIAL>in => (Tokens.IN(yypos - 1, yypos + size yytext - 2));
<INITIAL>nil => (Tokens.NIL(yypos - 1, yypos + size yytext - 2));
<INITIAL>let => (Tokens.LET(yypos - 1, yypos + size yytext - 2));
<INITIAL>do => (Tokens.DO(yypos - 1, yypos + size yytext - 2));
<INITIAL>to => (Tokens.TO(yypos - 1, yypos + size yytext - 2));
<INITIAL>for => (Tokens.FOR(yypos - 1, yypos + size yytext - 2));
<INITIAL>while => (Tokens.WHILE(yypos - 1, yypos + size yytext - 2));
<INITIAL>else => (Tokens.ELSE(yypos - 1, yypos + size yytext - 2));
<INITIAL>then => (Tokens.THEN(yypos - 1, yypos + size yytext - 2));
<INITIAL>if => (Tokens.IF(yypos - 1, yypos + size yytext - 2));
<INITIAL>array => (Tokens.ARRAY(yypos - 1, yypos + size yytext - 2));

<INITIAL>:\= => (Tokens.ASSIGN(yypos - 1, yypos + size yytext - 2));
<INITIAL>\| => (Tokens.OR(yypos - 1, yypos + size yytext - 2));
<INITIAL>& => (Tokens.AND(yypos - 1, yypos + size yytext - 2));
<INITIAL>\>\= => (Tokens.GE(yypos - 1, yypos + size yytext - 2));
<INITIAL>\> => (Tokens.GT(yypos - 1, yypos + size yytext - 2));
<INITIAL>\<\= => (Tokens.LE(yypos - 1, yypos + size yytext - 2));
<INITIAL>\< => (Tokens.LT(yypos - 1, yypos + size yytext - 2));
<INITIAL>\<\> => (Tokens.NEQ(yypos - 1, yypos + size yytext - 2));
<INITIAL>\= => (Tokens.EQ(yypos - 1, yypos + size yytext - 2));
<INITIAL>\/ => (Tokens.DIVIDE(yypos - 1, yypos + size yytext - 2));
<INITIAL>\* => (Tokens.TIMES(yypos - 1, yypos + size yytext - 2));
<INITIAL>- => (Tokens.MINUS(yypos - 1, yypos + size yytext - 2));
<INITIAL>\+ => (Tokens.PLUS(yypos - 1, yypos + size yytext - 2));
<INITIAL>\. => (Tokens.DOT(yypos - 1, yypos + size yytext - 2));
<INITIAL>\} => (Tokens.RBRACE(yypos - 1, yypos + size yytext - 2));
<INITIAL>\{ => (Tokens.LBRACE(yypos - 1, yypos + size yytext - 2));
<INITIAL>\] => (Tokens.RBRACK(yypos - 1, yypos + size yytext - 2));
<INITIAL>"\[" => (Tokens.LBRACK(yypos - 1, yypos + size yytext - 2));
<INITIAL>\) => (Tokens.RPAREN(yypos - 1, yypos + size yytext - 2));
<INITIAL>\( => (Tokens.LPAREN(yypos - 1, yypos + size yytext - 2));
<INITIAL>\; => (Tokens.SEMICOLON(yypos - 1, yypos + size yytext - 2));
<INITIAL>: => (Tokens.COLON(yypos - 1, yypos + size yytext - 2));
<INITIAL>, => (Tokens.COMMA(yypos - 1, yypos + size yytext - 2));

<INITIAL>{digit}+ => (Tokens.INT(atoi yytext, yypos, yypos + size yytext - 2));
<INITIAL>{id} => (Tokens.ID(yytext, yypos, yypos + size yytext - 2));
<INITIAL>[ \t]* => (continue());

<INITIAL>\" => (YYBEGIN STRING; SrcString.new(yypos); continue());
Expand Down
3 changes: 2 additions & 1 deletion test/lexer/comments.sml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Test.test(fn () =>
val s = "hello world"
in
Test.assertEq(Tokens.STRING(s, 36, 48), List.hd(tokens), Tokens.toString);
Test.assertEq(4, Newline.getLine(36), Int.toString)
Test.assertEq(4, Newline.getLine(36), Int.toString);
Test.assertEq(1, Newline.getPos(36), Int.toString)
end
);
14 changes: 7 additions & 7 deletions test/lexer/strings.sml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,54 +2,54 @@ Test.test(fn () =>
let val tokens = Parse.parseFile "fixtures/lexer/strings/one.tig"
val string = "hello world"
in
Test.assertEq(Tokens.STRING(string,1,11), List.hd(tokens), Tokens.toString)
Test.assertEq(Tokens.STRING(string,1,13), List.hd(tokens), Tokens.toString)
end
);

Test.test(fn () =>
let val tokens = Parse.parseFile "fixtures/lexer/strings/two.tig"
val string = "hello newline\n"
in
Test.assertEq(Tokens.STRING(string,1,15), List.hd(tokens), Tokens.toString)
Test.assertEq(Tokens.STRING(string,1,17), List.hd(tokens), Tokens.toString)
end
);

Test.test(fn () =>
let val tokens = Parse.parseFile "fixtures/lexer/strings/three.tig"
val string = "this one has \120 ascii"
in
Test.assertEq(Tokens.STRING(string,1,23), List.hd(tokens), Tokens.toString)
Test.assertEq(Tokens.STRING(string,1,25), List.hd(tokens), Tokens.toString)
end
);

Test.test(fn () =>
let val tokens = Parse.parseFile "fixtures/lexer/strings/four.tig"
val string = "hello plus @#^! weird chars!"
in
Test.assertEq(Tokens.STRING(string,1,28), List.hd(tokens), Tokens.toString)
Test.assertEq(Tokens.STRING(string,1,30), List.hd(tokens), Tokens.toString)
end
);

Test.test(fn () =>
let val tokens = Parse.parseFile "fixtures/lexer/strings/five.tig"
val string = "this has \\ and \n newline and !@$^ curse words!"
in
Test.assertEq(Tokens.STRING(string,1,48), List.hd(tokens), Tokens.toString)
Test.assertEq(Tokens.STRING(string,1,50), List.hd(tokens), Tokens.toString)
end
);

Test.test(fn () =>
let val tokens = Parse.parseFile "fixtures/lexer/strings/six.tig"
val string = "this has control sequence \^C for end of text"
in
Test.assertEq(Tokens.STRING(string,1,45), List.hd(tokens), Tokens.toString)
Test.assertEq(Tokens.STRING(string,1,47), List.hd(tokens), Tokens.toString)
end
);

Test.test(fn () =>
let val tokens = Parse.parseFile "fixtures/lexer/strings/seven.tig"
val string = "this is a multiline\ \ string!"
in
Test.assertEq(Tokens.STRING(string,1,33), List.hd(tokens), Tokens.toString)
Test.assertEq(Tokens.STRING(string,1,35), List.hd(tokens), Tokens.toString)
end
);

0 comments on commit bffb5af

Please sign in to comment.