Skip to content

Commit

Permalink
Remove line number from token.
Browse files Browse the repository at this point in the history
  • Loading branch information
@nixpulvis
nixpulvis committed Jan 28, 2016
1 parent e72ac75 commit 1998632
Show file tree
Hide file tree
Showing 4 changed files with 137 additions and 145 deletions.
2 changes: 1 addition & 1 deletion src/src_string.sml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,6 @@ structure SrcString :> SRC_STRING = struct
ErrorMsg.error("unrecognized control sequence: " ^ text)

fun emit (yypos) =
Tokens.STRING(!innerString, !startPos, yypos, 1)
Tokens.STRING(!innerString, !startPos, yypos)

end
88 changes: 44 additions & 44 deletions src/tokens.sig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,48 +4,48 @@ sig
type linenum
type token

val TYPE: int * int * int -> token
val VAR: int * int * int -> token
val FUNCTION: int * int * int -> token
val BREAK: int * int * int -> token
val OF: int * int * int -> token
val END: int * int * int -> token
val IN: int * int * int -> token
val NIL: int * int * int -> token
val LET: int * int * int -> token
val DO: int * int * int -> token
val TO: int * int * int -> token
val FOR: int * int * int -> token
val WHILE: int * int * int -> token
val ELSE: int * int * int -> token
val THEN: int * int * int -> token
val IF: int * int * int -> token
val ARRAY: int * int * int -> token
val ASSIGN: int * int * int -> token
val OR: int * int * int -> token
val AND: int * int * int -> token
val GE: int * int * int -> token
val GT: int * int * int -> token
val LE: int * int * int -> token
val LT: int * int * int -> token
val NEQ: int * int * int -> token
val EQ: int * int * int -> token
val DIVIDE: int * int * int -> token
val TIMES: int * int * int -> token
val MINUS: int * int * int -> token
val PLUS: int * int * int -> token
val DOT: int * int * int -> token
val RBRACE: int * int * int -> token
val LBRACE: int * int * int -> token
val RBRACK: int * int * int -> token
val LBRACK: int * int * int -> token
val RPAREN: int * int * int -> token
val LPAREN: int * int * int -> token
val SEMICOLON: int * int * int -> token
val COLON: int * int * int -> token
val COMMA: int * int * int -> token
val STRING: string * int * int * int -> token
val INT: int * int * int * int -> token
val ID: string * int * int * int -> token
val EOF: int * int * int -> token
val TYPE: int * int -> token
val VAR: int * int -> token
val FUNCTION: int * int -> token
val BREAK: int * int -> token
val OF: int * int -> token
val END: int * int -> token
val IN: int * int -> token
val NIL: int * int -> token
val LET: int * int -> token
val DO: int * int -> token
val TO: int * int -> token
val FOR: int * int -> token
val WHILE: int * int -> token
val ELSE: int * int -> token
val THEN: int * int -> token
val IF: int * int -> token
val ARRAY: int * int -> token
val ASSIGN: int * int -> token
val OR: int * int -> token
val AND: int * int -> token
val GE: int * int -> token
val GT: int * int -> token
val LE: int * int -> token
val LT: int * int -> token
val NEQ: int * int -> token
val EQ: int * int -> token
val DIVIDE: int * int -> token
val TIMES: int * int -> token
val MINUS: int * int -> token
val PLUS: int * int -> token
val DOT: int * int -> token
val RBRACE: int * int -> token
val LBRACE: int * int -> token
val RBRACK: int * int -> token
val LBRACK: int * int -> token
val RPAREN: int * int -> token
val LPAREN: int * int -> token
val SEMICOLON: int * int -> token
val COLON: int * int -> token
val COMMA: int * int -> token
val STRING: string * int * int -> token
val INT: int * int * int -> token
val ID: string * int * int -> token
val EOF: int * int -> token
end
176 changes: 88 additions & 88 deletions src/tokens.sml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,95 +2,95 @@ structure Tokens =
struct
type linenum = int
datatype token =
TYPE of int * int * int |
VAR of int * int * int |
FUNCTION of int * int * int |
BREAK of int * int * int |
OF of int * int * int |
END of int * int * int |
IN of int * int * int |
NIL of int * int * int |
LET of int * int * int |
DO of int * int * int |
TO of int * int * int |
FOR of int * int * int |
WHILE of int * int * int |
ELSE of int * int * int |
THEN of int * int * int |
IF of int * int * int |
ARRAY of int * int * int |
ASSIGN of int * int * int |
OR of int * int * int |
AND of int * int * int |
GE of int * int * int |
GT of int * int * int |
LE of int * int * int |
LT of int * int * int |
NEQ of int * int * int |
EQ of int * int * int |
DIVIDE of int * int * int |
TIMES of int * int * int |
MINUS of int * int * int |
PLUS of int * int * int |
DOT of int * int * int |
RBRACE of int * int * int |
LBRACE of int * int * int |
RBRACK of int * int * int |
LBRACK of int * int * int |
RPAREN of int * int * int |
LPAREN of int * int * int |
SEMICOLON of int * int * int |
COLON of int * int * int |
COMMA of int * int * int |
STRING of string * int * int * int |
INT of int * int * int * int |
ID of string * int * int * int |
EOF of int * int * int;
TYPE of int * int |
VAR of int * int |
FUNCTION of int * int |
BREAK of int * int |
OF of int * int |
END of int * int |
IN of int * int |
NIL of int * int |
LET of int * int |
DO of int * int |
TO of int * int |
FOR of int * int |
WHILE of int * int |
ELSE of int * int |
THEN of int * int |
IF of int * int |
ARRAY of int * int |
ASSIGN of int * int |
OR of int * int |
AND of int * int |
GE of int * int |
GT of int * int |
LE of int * int |
LT of int * int |
NEQ of int * int |
EQ of int * int |
DIVIDE of int * int |
TIMES of int * int |
MINUS of int * int |
PLUS of int * int |
DOT of int * int |
RBRACE of int * int |
LBRACE of int * int |
RBRACK of int * int |
LBRACK of int * int |
RPAREN of int * int |
LPAREN of int * int |
SEMICOLON of int * int |
COLON of int * int |
COMMA of int * int |
STRING of string * int * int |
INT of int * int * int |
ID of string * int * int |
EOF of int * int;

fun toString token =
case token of
TYPE(i,j,l) => "TYPE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
VAR(i,j,l) => "VAR(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
FUNCTION(i,j,l) => "FUNCTION(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
BREAK(i,j,l) => "BREAK(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
OF(i,j,l) => "OF(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
END(i,j,l) => "END(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
IN(i,j,l) => "IN(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
NIL(i,j,l) => "NIL(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
LET(i,j,l) => "LET(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
DO(i,j,l) => "DO(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
TO(i,j,l) => "TO(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
FOR(i,j,l) => "FOR(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
WHILE(i,j,l) => "WHILE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
ELSE(i,j,l) => "ELSE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
THEN(i,j,l) => "THEN(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
IF(i,j,l) => "IF(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
ARRAY(i,j,l) => "ARRAY(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
ASSIGN(i,j,l) => "ASSIGN(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
OR(i,j,l) => "OR(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
AND(i,j,l) => "AND(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
GE(i,j,l) => "GE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
GT(i,j,l) => "GT(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
LE(i,j,l) => "LE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
LT(i,j,l) => "LT(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
NEQ(i,j,l) => "NEQ(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
EQ(i,j,l) => "EQ(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
DIVIDE(i,j,l) => "DIVIDE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
TIMES(i,j,l) => "TIMES(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
MINUS(i,j,l) => "MINUS(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
PLUS(i,j,l) => "PLUS(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
DOT(i,j,l) => "DOT(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
RBRACE(i,j,l) => "RBRACE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
LBRACE(i,j,l) => "LBRACE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
RBRACK(i,j,l) => "RBRACK(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
LBRACK(i,j,l) => "LBRACK(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
RPAREN(i,j,l) => "RPAREN(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
LPAREN(i,j,l) => "LPAREN(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
SEMICOLON(i,j,l) => "SEMICOLON(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
COLON(i,j,l) => "COLON(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
COMMA(i,j,l) => "COMMA(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
STRING(s,i,j,l) => "STRING(" ^ s ^ "," ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
INT(n,i,j,l) => "INT(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
ID(s,i,j,l) => "ID(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")" |
EOF(i,j,l) => "EOF(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ "," ^ Int.toString(l) ^ ")";
TYPE(i,j) => "TYPE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| VAR(i,j) => "VAR(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| FUNCTION(i,j) => "FUNCTION(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| BREAK(i,j) => "BREAK(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| OF(i,j) => "OF(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| END(i,j) => "END(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| IN(i,j) => "IN(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| NIL(i,j) => "NIL(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| LET(i,j) => "LET(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| DO(i,j) => "DO(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| TO(i,j) => "TO(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| FOR(i,j) => "FOR(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| WHILE(i,j) => "WHILE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| ELSE(i,j) => "ELSE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| THEN(i,j) => "THEN(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| IF(i,j) => "IF(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| ARRAY(i,j) => "ARRAY(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| ASSIGN(i,j) => "ASSIGN(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| OR(i,j) => "OR(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| AND(i,j) => "AND(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| GE(i,j) => "GE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| GT(i,j) => "GT(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| LE(i,j) => "LE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| LT(i,j) => "LT(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| NEQ(i,j) => "NEQ(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| EQ(i,j) => "EQ(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| DIVIDE(i,j) => "DIVIDE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| TIMES(i,j) => "TIMES(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| MINUS(i,j) => "MINUS(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| PLUS(i,j) => "PLUS(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| DOT(i,j) => "DOT(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| RBRACE(i,j) => "RBRACE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| LBRACE(i,j) => "LBRACE(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| RBRACK(i,j) => "RBRACK(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| LBRACK(i,j) => "LBRACK(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| RPAREN(i,j) => "RPAREN(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| LPAREN(i,j) => "LPAREN(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| SEMICOLON(i,j) => "SEMICOLON(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| COLON(i,j) => "COLON(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| COMMA(i,j) => "COMMA(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| STRING(s,i,j) => "STRING(" ^ s ^ "," ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| INT(n,i,j) => "INT(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| ID(s,i,j) => "ID(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")"
| EOF(i,j) => "EOF(" ^ Int.toString(i) ^ "," ^ Int.toString(j) ^ ")";
end
16 changes: 4 additions & 12 deletions test/src_string.sml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ Test.test (fn () =>
(Newline.reset();
SrcString.new 1;
SrcString.push("s", 10);
Test.assertEq(Tokens.STRING("s",1,10,1),
Test.assertEq(Tokens.STRING("s",1,10),
SrcString.emit(10),
Tokens.toString))
);
Expand All @@ -11,7 +11,7 @@ Test.test (fn () =>
(Newline.reset();
SrcString.new 1;
SrcString.pushString("hello");
Test.assertEq(Tokens.STRING("hello",1,10,1),
Test.assertEq(Tokens.STRING("hello",1,10),
SrcString.emit(10),
Tokens.toString))
);
Expand All @@ -20,11 +20,7 @@ Test.test (fn () =>
(Newline.reset();
SrcString.new 1;
SrcString.pushAscii("067");
(*
"\067"
123456
*)
Test.assertEq(Tokens.STRING("C",1,10,1),
Test.assertEq(Tokens.STRING("C",1,10),
SrcString.emit(10),
Tokens.toString))
);
Expand All @@ -34,11 +30,7 @@ Test.test (fn () =>
SrcString.new 1;
SrcString.pushString("hi ");
SrcString.pushControl("^C");
(*
"hi \^C"
12345678
*)
Test.assertEq(Tokens.STRING("hi \^C",1,10,1),
Test.assertEq(Tokens.STRING("hi \^C",1,10),
SrcString.emit(10),
Tokens.toString))
);

0 comments on commit 1998632

Please sign in to comment.