tests: Add main and filecheck tests for Toy (xdslproject#867)

I'm having trouble getting filecheck to cooperate. It refuses to run tests outside `tests/filecheck`, as far as I can tell, do you know what might be causing this? ``` (venv) (base) sasha@SL-MBP-16 ~/D/x/xdsl> lit -v docs/Toy/examples/ast.toy sasha/toy/filecheck! lit: /Users/sasha/Developer/xdslproject/xdsl/venv/lib/python3.11/site-packages/lit/discovery.py:132: warning: unable to find test suite for 'docs/Toy/examples/ast.toy' lit: /Users/sasha/Developer/xdslproject/xdsl/venv/lib/python3.11/site-packages/lit/discovery.py:281: warning: input 'docs/Toy/examples/ast.toy' contained no tests error: did not discover any tests for provided path(s) ``` Running the copy in `filecheck/frontend` works fine
emmau678 · May 4, 2023 · 3bff937 · 3bff937
1 parent 87a78e1
commit 3bff937
Show file tree

Hide file tree

Showing 11 changed files with 147 additions and 45 deletions.
diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml
@@ -44,3 +44,4 @@ jobs:
       run: |
         export PYTHONPATH=$(pwd)
         lit -v tests/filecheck/
+        lit -v docs/Toy/examples/
diff --git a/.gitignore b/.gitignore
@@ -143,7 +143,7 @@ cython_debug/
 
 
 # lit tests results
-tests/filecheck/.lit_test_times.txt
+**/.lit_test_times.txt
 tests/**/Output/*
 *.out
 

diff --git a/docs/Toy/Toy_Ch1.ipynb b/docs/Toy/Toy_Ch1.ipynb
@@ -30,7 +30,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "e07ae44f",
    "metadata": {},
    "outputs": [],
@@ -68,7 +68,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "853ca52d",
    "metadata": {},
    "outputs": [],
@@ -174,10 +174,63 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "cf9b107d",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Module:\n",
+      "  Function \n",
+      "    Proto 'multiply_transpose' @examples/ast.toy:4:1\n",
+      "    Params: [a, b]\n",
+      "    Block {\n",
+      "      Return\n",
+      "        BinOp: * @examples/ast.toy:5:25\n",
+      "          Call 'transpose' [ @examples/ast.toy:5:10\n",
+      "            var: a @examples/ast.toy:5:20\n",
+      "          ]\n",
+      "          Call 'transpose' [ @examples/ast.toy:5:25\n",
+      "            var: b @examples/ast.toy:5:35\n",
+      "          ]\n",
+      "    } // Block\n",
+      "  Function \n",
+      "    Proto 'main' @examples/ast.toy:8:1\n",
+      "    Params: []\n",
+      "    Block {\n",
+      "      VarDecl a<> @examples/ast.toy:11:3\n",
+      "        Literal: <2, 3>[ <3>[ 1.000000e+00, 2.000000e+00, 3.000000e+00], <3>[ 4.000000e+00, 5.000000e+00, 6.000000e+00]] @examples/ast.toy:11:11\n",
+      "      VarDecl b<2, 3> @examples/ast.toy:15:3\n",
+      "        Literal: <6>[ 1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 5.000000e+00, 6.000000e+00] @examples/ast.toy:15:17\n",
+      "      VarDecl c<> @examples/ast.toy:19:3\n",
+      "        Call 'multiply_transpose' [ @examples/ast.toy:19:11\n",
+      "          var: a @examples/ast.toy:19:30\n",
+      "          var: b @examples/ast.toy:19:33\n",
+      "        ]\n",
+      "      VarDecl d<> @examples/ast.toy:22:3\n",
+      "        Call 'multiply_transpose' [ @examples/ast.toy:22:11\n",
+      "          var: b @examples/ast.toy:22:30\n",
+      "          var: a @examples/ast.toy:22:33\n",
+      "        ]\n",
+      "      VarDecl e<> @examples/ast.toy:25:3\n",
+      "        Call 'multiply_transpose' [ @examples/ast.toy:25:11\n",
+      "          var: b @examples/ast.toy:25:30\n",
+      "          var: c @examples/ast.toy:25:33\n",
+      "        ]\n",
+      "      VarDecl f<> @examples/ast.toy:28:3\n",
+      "        Call 'multiply_transpose' [ @examples/ast.toy:28:11\n",
+      "          Call 'transpose' [ @examples/ast.toy:28:30\n",
+      "            var: a @examples/ast.toy:28:40\n",
+      "          ]\n",
+      "          var: c @examples/ast.toy:28:44\n",
+      "        ]\n",
+      "    } // Block\n",
+      "\n"
+     ]
+    }
+   ],
    "source": [
     "from pathlib import Path\n",
     "from toy.frontend.parser import Parser\n",

diff --git a/docs/Toy/examples/ast.toy b/docs/Toy/examples/ast.toy
@@ -1,4 +1,4 @@
-# RUN: toyc-ch1 %s -emit=ast 2>&1 | FileCheck %s
+# RUN: python -m toy %s --emit=ast | filecheck %s
 
 # User defined generic function that operates on unknown shaped arguments.
 def multiply_transpose(a, b) {

diff --git a/docs/Toy/examples/codegen.toy b/docs/Toy/examples/codegen.toy
@@ -1,4 +1,4 @@
-# RUN: toyc-ch2 %s -emit=mlir 2>&1 | FileCheck %s
+# RUN: python -m toy %s --emit=ir | filecheck %s
 
 # User defined generic function that operates on unknown shaped arguments
 def multiply_transpose(a, b) {
@@ -13,19 +13,22 @@ def main() {
   print(d);
 }
 
-# CHECK-LABEL: toy.func @multiply_transpose(
-# CHECK-SAME:                               [[VAL_0:%.*]]: tensor<*xf64>, [[VAL_1:%.*]]: tensor<*xf64>) -> tensor<*xf64>
-# CHECK:         [[VAL_2:%.*]] = toy.transpose([[VAL_0]] : tensor<*xf64>) to tensor<*xf64>
-# CHECK-NEXT:    [[VAL_3:%.*]] = toy.transpose([[VAL_1]] : tensor<*xf64>) to tensor<*xf64>
-# CHECK-NEXT:    [[VAL_4:%.*]] = toy.mul [[VAL_2]], [[VAL_3]] :  tensor<*xf64>
-# CHECK-NEXT:    toy.return [[VAL_4]] : tensor<*xf64>
-
-# CHECK-LABEL: toy.func @main()
-# CHECK-NEXT:    [[VAL_5:%.*]] = toy.constant dense<{{\[\[}}1.000000e+00, 2.000000e+00, 3.000000e+00], [4.000000e+00, 5.000000e+00, 6.000000e+00]]> : tensor<2x3xf64>
-# CHECK-NEXT:    [[VAL_6:%.*]] = toy.reshape([[VAL_5]] : tensor<2x3xf64>) to tensor<2x3xf64>
-# CHECK-NEXT:    [[VAL_7:%.*]] = toy.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 5.000000e+00, 6.000000e+00]> : tensor<6xf64>
-# CHECK-NEXT:    [[VAL_8:%.*]] = toy.reshape([[VAL_7]] : tensor<6xf64>) to tensor<2x3xf64>
-# CHECK-NEXT:    [[VAL_9:%.*]] = toy.generic_call @multiply_transpose([[VAL_6]], [[VAL_8]]) : (tensor<2x3xf64>, tensor<2x3xf64>) -> tensor<*xf64>
-# CHECK-NEXT:    [[VAL_10:%.*]] = toy.generic_call @multiply_transpose([[VAL_8]], [[VAL_6]]) : (tensor<2x3xf64>, tensor<2x3xf64>) -> tensor<*xf64>
-# CHECK-NEXT:    toy.print [[VAL_10]] : tensor<*xf64>
-# CHECK-NEXT:    toy.return
+# CHECK:       "builtin.module"() ({
+# CHECK-NEXT:    "toy.func"() ({
+# CHECK-NEXT:    ^0(%{{.*}} : tensor<*xf64>, %{{.*}} : tensor<*xf64>):
+# CHECK-NEXT:      %{{.*}} = "toy.transpose"(%{{.*}}) : (tensor<*xf64>) -> tensor<*xf64>
+# CHECK-NEXT:      %{{.*}} = "toy.transpose"(%{{.*}}) : (tensor<*xf64>) -> tensor<*xf64>
+# CHECK-NEXT:      %{{.*}} = "toy.mul"(%{{.*}}, %{{.*}}) : (tensor<*xf64>, tensor<*xf64>) -> tensor<*xf64>
+# CHECK-NEXT:      "toy.return"(%{{.*}}) : (tensor<*xf64>) -> ()
+# CHECK-NEXT:    }) {"sym_name" = "multiply_transpose", "function_type" = (tensor<*xf64>, tensor<*xf64>) -> tensor<*xf64>, "sym_visibility" = "private"} : () -> ()
+# CHECK-NEXT:    "toy.func"() ({
+# CHECK-NEXT:      %{{.*}} = "toy.constant"() {"value" = dense<[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]> : tensor<2x3xf64>} : () -> tensor<2x3xf64>
+# CHECK-NEXT:      %{{.*}} = "toy.reshape"(%{{.*}}) : (tensor<2x3xf64>) -> tensor<2x3xf64>
+# CHECK-NEXT:      %{{.*}} = "toy.constant"() {"value" = dense<[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]> : tensor<6xf64>} : () -> tensor<6xf64>
+# CHECK-NEXT:      %{{.*}} = "toy.reshape"(%{{.*}}) : (tensor<6xf64>) -> tensor<2x3xf64>
+# CHECK-NEXT:      %{{.*}} = "toy.generic_call"(%{{.*}}, %{{.*}}) {"callee" = @multiply_transpose} : (tensor<2x3xf64>, tensor<2x3xf64>) -> tensor<*xf64>
+# CHECK-NEXT:      %{{.*}} = "toy.generic_call"(%{{.*}}, %{{.*}}) {"callee" = @multiply_transpose} : (tensor<2x3xf64>, tensor<2x3xf64>) -> tensor<*xf64>
+# CHECK-NEXT:      "toy.print"(%{{.*}}) : (tensor<*xf64>) -> ()
+# CHECK-NEXT:      "toy.return"() : () -> ()
+# CHECK-NEXT:    }) {"sym_name" = "main", "function_type" = () -> ()} : () -> ()
+# CHECK-NEXT:  }) : () -> ()
diff --git a/docs/Toy/examples/empty.toy b/docs/Toy/examples/empty.toy
diff --git a/docs/Toy/examples/lit.cfg b/docs/Toy/examples/lit.cfg
@@ -0,0 +1,9 @@
+import lit.formats
+import os
+
+config.test_source_root = os.path.dirname(__file__)
+toy_src = os.path.dirname(config.test_source_root)
+
+config.name = "Toy"
+config.test_format = lit.formats.ShTest(preamble_commands=[f"cd {toy_src}"])
+config.suffixes = ['.mlir', '.toy']
diff --git a/docs/Toy/examples/scalar.toy b/docs/Toy/examples/scalar.toy
diff --git a/docs/Toy/toy/__main__.py b/docs/Toy/toy/__main__.py
@@ -0,0 +1,45 @@
+import argparse
+from pathlib import Path
+
+from xdsl.printer import Printer
+
+from .frontend.ir_gen import IRGen
+from .frontend.parser import Parser
+
+
+parser = argparse.ArgumentParser(description="Process Toy file")
+parser.add_argument("source", type=Path, help="toy source file")
+parser.add_argument(
+    "--emit",
+    dest="emit",
+    choices=["ast", "ir"],
+    default="ast",
+    help="Action to perform on source file (default: interpret file)",
+)
+
+
+def main(path: Path, emit: str):
+    path = args.source
+
+    with open(path, "r") as f:
+        parser = Parser(path, f.read())
+        ast = parser.parseModule()
+
+    if emit == "ast":
+        print(ast.dump())
+        return
+
+    ir_gen = IRGen()
+
+    module_op = ir_gen.ir_gen_module(ast)
+
+    if emit == "ir":
+        printer = Printer()
+        printer.print_op(module_op)
+
+    print(f"Unknown option {emit}")
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    main(args.source, args.emit)
diff --git a/docs/Toy/toy/frontend/lexer.py b/docs/Toy/toy/frontend/lexer.py
@@ -99,6 +99,13 @@ def flush():
                 text += char
                 continue
 
+            if char == ".":
+                # parse floating point
+                if not text or (text[0].isnumeric() and "." not in text):
+                    # allow `.5` and `5.5` but not `5.5.5`
+                    text += char
+                    continue
+
             flush()
 
             if char == " ":

diff --git a/docs/Toy/toy/frontend/toy_ast.py b/docs/Toy/toy/frontend/toy_ast.py
@@ -90,7 +90,7 @@ def kind(self):
 
     def inner_dump(self, prefix: str, dumper: Dumper):
         dims_str = ", ".join(f"{int(dim)}" for dim in self.varType.shape)
-        dumper.append("VarDecl ", f"{self.name}<{dims_str}> {self.loc}")
+        dumper.append("VarDecl ", f"{self.name}<{dims_str}> @{self.loc}")
         child = dumper.child()
         self.expr.inner_dump("", child)
 
@@ -143,7 +143,7 @@ def __dump(self) -> str:
         return f" <{dims_str}>[{vals_str}]"
 
     def inner_dump(self, prefix: str, dumper: Dumper):
-        dumper.append("Literal:", self.__dump() + f" {self.loc}")
+        dumper.append("Literal:", self.__dump() + f" @{self.loc}")
 
     def iter_flattened_values(self) -> Generator[float, None, None]:
         for value in self.values:
@@ -166,7 +166,7 @@ def kind(self):
         return ExprASTKind.Expr_Var
 
     def inner_dump(self, prefix: str, dumper: Dumper):
-        dumper.append("var: ", f"{self.name} {self.loc}")
+        dumper.append("var: ", f"{self.name} @{self.loc}")
 
 
 @dataclass
@@ -181,7 +181,7 @@ def kind(self):
         return ExprASTKind.Expr_BinOp
 
     def inner_dump(self, prefix: str, dumper: Dumper):
-        dumper.append(prefix, f"BinOp: {self.op} {self.loc}")
+        dumper.append(prefix, f"BinOp: {self.op} @{self.loc}")
         child = dumper.child()
         self.lhs.inner_dump("", child)
         self.rhs.inner_dump("", child)
@@ -200,7 +200,7 @@ def kind(self):
     def inner_dump(self, prefix: str, dumper: Dumper):
         dumper.append_list(
             prefix,
-            f"Call '{self.callee}' [ {self.loc}",
+            f"Call '{self.callee}' [ @{self.loc}",
             self.args,
             "]",
             lambda dd, arg: arg.inner_dump("", dd),
@@ -240,7 +240,7 @@ def dump(self):
         return dumper.message
 
     def inner_dump(self, prefix: str, dumper: Dumper):
-        dumper.append("", f"Proto '{self.name}' {self.loc}'")
+        dumper.append("", f"Proto '{self.name}' @{self.loc}")
         dumper.append("Params: ", f'[{", ".join(arg.name for arg in self.args)}]')