New token-based version of the minifier, more effective.

pico8/lua/lua.py

@@ -813,24 +813,45 @@ def to_lines(self):
             yield parts[-1]
 
 
-class LuaMinifyWriter(LuaASTEchoWriter):
-    """Writes the Lua code to use a minimal number of characters.
-    """
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._name_map = {}
-        self._next_name_id = 0
-
+class MinifyNameFactory():
+    """Maps code names to generated short names."""
     NAME_CHARS = 'abcdefghijklmnopqrstuvwxyz'
     PRESERVED_NAMES = lexer.LUA_KEYWORDS | PICO8_BUILTINS
 
+    def __init__(self):
+        self._name_map = {}
+        self._next_name_id = 0
+
     @classmethod
     def _name_for_id(cls, id):
         first = ''
-        if id >= len(LuaMinifyWriter.NAME_CHARS):
-            first = cls._name_for_id(int(id / len(LuaMinifyWriter.NAME_CHARS)))
-        return first + (LuaMinifyWriter.NAME_CHARS[id % len(LuaMinifyWriter.NAME_CHARS)])
-
+        if id >= len(MinifyNameFactory.NAME_CHARS):
+            first = cls._name_for_id(int(id / len(MinifyNameFactory.NAME_CHARS)))
+        return first + (MinifyNameFactory.NAME_CHARS[id % len(MinifyNameFactory.NAME_CHARS)])
+
+    def get_short_name(self, name):
+        if name in MinifyNameFactory.PRESERVED_NAMES:
+            return name
+        if name not in self._name_map:
+            new_name = None
+            while True:
+                new_name = self._name_for_id(self._next_name_id)
+                self._next_name_id += 1
+                if not new_name in MinifyNameFactory.PRESERVED_NAMES:
+                    break
+            self._name_map[name] = new_name
+            util.debug('- minifying name "{}" to "{}"\n'.format(
+                name, new_name))
+        return self._name_map[name]
+
+
+class LuaMinifyWriter(LuaASTEchoWriter):
+    """Writes the Lua code to use a minimal number of characters.
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._name_factory = MinifyNameFactory()
+
     def _get_name(self, node, tok):
         """Gets the minified name for a TokName.
 
@@ -844,22 +865,7 @@ def _get_name(self, node, tok):
         spaces = self._get_code_for_spaces(node)
         assert tok.matches(lexer.TokName)
         self._pos += 1
-
-        if tok.code in LuaMinifyWriter.PRESERVED_NAMES:
-            return spaces + tok.code
-
-        if tok.code not in self._name_map:
-            new_name = None
-            while True:
-                new_name = self._name_for_id(self._next_name_id)
-                self._next_name_id += 1
-                if not new_name in LuaMinifyWriter.PRESERVED_NAMES:
-                    break
-            self._name_map[tok.code] = new_name
-            util.debug('- minifying name "{}" to "{}"\n'.format(
-                tok.code, new_name))
-
-        return spaces + self._name_map[tok.code]
+        return spaces + self._name_factory.get_short_name(tok.code)
 
     def _get_code_for_spaces(self, node):
         """Calculates the minified text for the space and comment tokens that
@@ -1003,3 +1009,52 @@ def _get_code_for_spaces(self, node):
         # - block starts on a new line; 'end' always on its own line
 
         return spaces
+
+
+class LuaMinifyTokenWriter(BaseLuaWriter):
+    """Another minify writer.
+
+    Unlike LuaMinifyWriter, this implementation just runs across the token stream and ignores the parser.
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._name_factory = MinifyNameFactory()
+        self._last_was_name_keyword_number = False
+        self._saw_if = False
+
+    def to_lines(self):
+        """
+        Yields:
+          Chunks of Lua code.
+        """
+        for token in self._tokens:
+            if (token.matches(lexer.TokComment) or
+                token.matches(lexer.TokSpace)):
+                continue
+            elif token.matches(lexer.TokNewline):
+                # Hack for short-if: after seeing "if" (even if not short-if), keep the next newline.
+                if self._saw_if:
+                    self._saw_if = False
+                    self._last_was_name_keyword_number = False
+                    yield '\n'
+                continue
+            elif token.matches(lexer.TokName):
+                if self._last_was_name_keyword_number:
+                    yield ' '
+                self._last_was_name_keyword_number = True
+                yield self._name_factory.get_short_name(token.code)
+            elif token.matches(lexer.TokKeyword):
+                if token.code == 'if':
+                    self._saw_if = True
+                if self._last_was_name_keyword_number:
+                    yield ' '
+                self._last_was_name_keyword_number = True
+                yield token.code
+            elif token.matches(lexer.TokNumber):
+                if self._last_was_name_keyword_number:
+                    yield ' '
+                self._last_was_name_keyword_number = True
+                yield token.code
+            else:
+                self._last_was_name_keyword_number = False
+                yield token.code

pico8/tool.py

@@ -293,7 +293,7 @@ def luamin(g, outfh, out_fname, args=None):
       args: The argparse parsed args object, or None.
     """
     g.to_p8_file(outfh, filename=out_fname,
-                 lua_writer_cls=lua.LuaMinifyWriter)
+                 lua_writer_cls=lua.LuaMinifyTokenWriter)
 
 
 def luafmt(g, outfh, out_fname, args=None):

tests/pico8/lua/lua_test.py

@@ -203,6 +203,15 @@ def testMinifiesSpacesEveryNode(self):
 ''', txt)
         self.assertIn('f=1  n=2  o=3', txt)
 
+    def testMinifyTokenWriterMinifiesSpacesEveryNode(self):
+        result = lua.Lua.from_lines(VALID_LUA_EVERY_NODE, 4)
+        lines = list(result.to_lines(writer_cls=lua.LuaMinifyTokenWriter))
+        txt = ''.join(lines)
+        self.assertNotIn('-- the code with the nodes', txt)
+        self.assertIn('while f<10 do f+=1 if f%2==0 then\na(f)elseif f>5 then a(f,5)else a(f,1)g*=2 end end', txt)
+        self.assertIn('for g in i()do a(g)end, txt)
+        self.assertIn('f=1;n=2;o=3', txt)
+
 
 class TestLuaFormatterWriter(unittest.TestCase):
     def testNormalizesSpaceCharacters(self):