Comment tokens order doesn't matter..

rshk · Nov 11, 2013 · 1220dd9 · 1220dd9
1 parent 080e2f4
commit 1220dd9
Showing 1 changed file with 15 additions and 8 deletions.
diff --git a/mongosql/lexer.py b/mongosql/lexer.py
@@ -50,7 +50,10 @@
 ]
 
 
-tokens.extend(x[0] for x in token_symbols)
+globs = globals()
+for tk, tkval in token_symbols:
+    tokens.append(tk)
+    globs['t_' + tk] = re.escape(tkval)
 
 
 reserved = [
@@ -107,6 +110,17 @@ def t_error(t):
     raise LexerError("Unknown text {0!r}".format(t.value,))
 
 
+##------------------------------------------------------------
+## Comments:
+##
+##   // C++ style
+##   #  Python style
+##   -- SQL style
+##   /* C style */
+##------------------------------------------------------------
+
+## Note: maybe we shouldn't waste that many tokens for comments..
+
 comment_cpp = r'//.*'
 comment_python = r'\#.*'
 comment_sql = r'--.*'
@@ -119,13 +133,6 @@ def t_COMMENT(t):
     return None  # Just skip this token
 
 
-## We need to do this *after* declaring t_COMMENT
-## order matters in the lexer module...
-globs = globals()
-for tk, tkval in token_symbols:
-    globs['t_' + tk] = re.escape(tkval)
-
-
 ##------------------------------------------------------------
 ## Simple tokens
 ##------------------------------------------------------------