Huge Refactor - Use @runno/wasi-motor from runtime (#221)

* Fix fd_filestat_get with STDIO for Python compat

* Add some standard languages to the website

* WIP refactor to use wasi-motor instead of wasmer

* Remove dead @wasmer forks

* Sort out dependencies

* WIP python working locally

* Fix playground and clang-fs

* Handle termination vs crash errors better

* WIP: Various minor fixes to WASI implementation

* Improve debugging and fix some more minor bugs

* Improve debugging and fix some more minor bugs

* Fix up debug data passthrough

* Get ruby working with stdout config

* Fix passing crash through to host, add some more debugging

* Get SQLite and quickjs working

* Get QuickJS working - now SQL doesn't?

* Improve error handling

* Fix filestat difference in unstable vs preview1

* Performance and memory improvements

* Performance improvements + fix whence seeking in unstable

* Minor fixes

* Fix off-by-one error in write, C++ now works yiew

* Fix up commands for running C in Runno

* Remove dead build commands

* Fix errors building

* Switch lerna for npm workspaces

* Fix up tests

* Switch to production URL

README.md

@@ -85,8 +85,7 @@ This repo is broken down into a few packages using [lerna](https://lerna.js.org/
 - `client` - a static website that exposes the runno runtime to be embedded as an iframe
 - `host` - helpers for running code on the client from another website
 - `runtime` - a library that provides web components and helpers that can be bundled into your own project for using runno without an iframe
-- `terminal` - an internal package forked from `@wasmer/wasm-terminal` used to construct parts of the runtime - in the future this will be rolled into `runtime`
-- `wasi` - an internal package forked from `@wasmer/wasm-wasi` fixes some bugs - in the future this will be rolled into `runtime`
+- `wasi-motor` - a library for running WASI binaries in the browser
 
 ## Running locally
 

examples/src/index.ts

@@ -1,3 +1 @@
-import { defineElements } from "@runno/runtime";
-
-defineElements();
+import "@runno/runtime";

package.json

@@ -7,19 +7,20 @@
   "engines": {
     "node": ">=12"
   },
+  "workspaces": [
+    "packages/*"
+  ],
   "scripts": {
     "prepare": "npx husky install",
-    "bootstrap": "npx lerna bootstrap",
+    "bootstrap": "npm install",
     "dev": "echo TODO dev command, for now just use dev in client and website",
     "publish": "npm run build && npx lerna publish",
     "build:host": "cd packages/host && npm run build",
     "build:client": "cd packages/client && npm run build",
-    "build:wasi": "cd packages/wasi && npm run build",
     "build:wasi-motor": "cd packages/wasi-motor && npm run build",
-    "build:terminal": "cd packages/terminal && npm run build",
     "build:runtime": "cd packages/runtime && npm run build",
     "build:website": "cd packages/website && npm run build",
-    "build": "npm run build:host && npm run build:wasi && npm run build:wasi-motor && npm run build:terminal && npm run build:runtime && npm run build:client && npm run build:website",
+    "build": "npm run build:host && npm run build:wasi-motor && npm run build:runtime && npm run build:client && npm run build:website",
     "build:deploy": "npm run bootstrap && npm run build",
     "test:client": "cd packages/client && npm run test",
     "test:wasi-motor": "cd packages/wasi-motor && npm run test:prepare && npm run test",

packages/client/package.json

@@ -26,7 +26,7 @@
     "test": "npm run test:playwright"
   },
   "devDependencies": {
-    "@playwright/test": "^1.26.1",
+    "@playwright/test": "^1.35.0",
     "@types/url-safe-base64": "^1.1.0",
     "eslint": "^8.2.0",
     "typescript": "^4.4.4",
@@ -35,7 +35,6 @@
   "dependencies": {
     "@runno/host": "^0.3.0",
     "@runno/runtime": "^0.3.1",
-    "@runno/terminal": "^0.3.0",
     "normalize.css": "^8.0.1",
     "post-me": "^0.4.5",
     "url-safe-base64": "^1.1.1"

packages/client/src/main.ts

@@ -1,8 +1,7 @@
 import { createConnection } from "./messaging";
 import { handleParams } from "./params";
-import { defineElements, RunElement } from "@runno/runtime";
-
-defineElements();
+import "@runno/runtime";
+import type { RunElement } from "@runno/runtime";
 
 const runtime = document.querySelector<RunElement>("runno-run")!;
 

packages/client/src/params.ts

@@ -25,7 +25,6 @@ export function handleParams(provider: RuntimeMethods) {
   const params = new URLSearchParams(urlParams);
 
   const code = params.get("code") ? atob(decode(params.get("code")!)) : "";
-  const command = params.get("command");
   const runtimeName = params.get("runtime");
   const showEditor = isTruthy(params.get("editor"));
   const autorun = isTruthy(params.get("autorun"));
@@ -54,13 +53,4 @@ export function handleParams(provider: RuntimeMethods) {
   if (autorun && runtimeName) {
     provider.interactiveRunCode(runtimeName as Runtime, code);
   }
-
-  if (command) {
-    provider.interactiveUnsafeCommand(command, {
-      code: {
-        name: "code",
-        content: code,
-      },
-    });
-  }
 }

packages/client/tests/smoke.spec.ts

@@ -31,16 +31,22 @@ test.describe("Python Hello World", () => {
     await expect(page.locator("runno-editor .cm-content")).toHaveText(program);
   });
 
-  test("should say hello world", async ({ page }) => {
+  test("should say hello world", async ({ page, browserName }) => {
+    // https://github.com/microsoft/playwright/issues/14043
+    test.skip(
+      browserName === "webkit",
+      "Playwright doesn't support SharedArrayBuffer"
+    );
+
     await page.locator("button", { hasText: "Run" }).click();
 
     await page.locator("runno-controls:not([running])").waitFor();
 
     const terminal = await page.locator("runno-terminal").elementHandle();
     expect(terminal).not.toBeNull();
     const text = await terminal!.evaluate((terminal: TerminalElement) => {
-      terminal.wasmTerminal.xterm.selectAll();
-      return terminal.wasmTerminal.xterm.getSelection().trim();
+      terminal.terminal.selectAll();
+      return terminal.terminal.getSelection().trim();
     });
 
     expect(text).toEqual("hello world");

packages/host/src/index.ts

@@ -1,5 +1,5 @@
 import { ParentHandshake, RemoteHandle, WindowMessenger } from "post-me";
-import { Runtime, RuntimeMethods, RunResult, FS, Syntax } from "./types";
+import { Runtime, RuntimeMethods, RunResult, WASIFS, Syntax } from "./types";
 import { encode } from "url-safe-base64";
 
 export function generateEmbedURL(
@@ -100,15 +100,11 @@ export class RunnoHost implements RuntimeMethods {
   interactiveRunFS(
     runtime: Runtime,
     entryPath: string,
-    fs: FS
+    fs: WASIFS
   ): Promise<RunResult> {
     return this.remoteHandle.call("interactiveRunFS", runtime, entryPath, fs);
   }
 
-  interactiveUnsafeCommand(command: string, fs: FS): Promise<RunResult> {
-    return this.remoteHandle.call("interactiveUnsafeCommand", command, fs);
-  }
-
   interactiveStop(): Promise<void> {
     return this.remoteHandle.call("interactiveStop");
   }
@@ -124,7 +120,7 @@ export class RunnoHost implements RuntimeMethods {
   headlessRunFS(
     runtime: Runtime,
     entryPath: string,
-    fs: FS,
+    fs: WASIFS,
     stdin?: string
   ): Promise<RunResult> {
     return this.remoteHandle.call(
@@ -135,14 +131,6 @@ export class RunnoHost implements RuntimeMethods {
       stdin
     );
   }
-
-  headlessUnsafeCommand(
-    command: string,
-    fs: FS,
-    stdin?: string
-  ): Promise<RunResult> {
-    return this.remoteHandle.call("headlessUnsafeCommand", command, fs, stdin);
-  }
 }
 
 export async function ConnectRunno(

packages/host/src/types.ts

@@ -7,27 +7,59 @@ export type Runtime =
   | "ruby";
 export type Syntax = "python" | "js" | "sql" | "cpp" | "ruby" | undefined;
 
-export type CommandResult = {
+export type CrashResult = {
+  resultType: "crash";
+  error: {
+    message: string;
+    type: string;
+  };
+};
+
+export type CompleteResult = {
+  resultType: "complete";
   stdin: string;
   stdout: string;
   stderr: string;
   tty: string;
-  fs: FS;
-  exit: number;
+  fs: WASIFS;
+  exitCode: number;
 };
 
-export type RunResult = {
-  result?: CommandResult;
-  prepare?: CommandResult;
+export type TerminatedResult = {
+  resultType: "terminated";
+};
+
+export type RunResult = CompleteResult | CrashResult | TerminatedResult;
+
+export type WASIPath = string;
+
+export type WASIFS = {
+  [path: WASIPath]: WASIFile;
 };
 
-export type FS = {
-  [name: string]: File;
+export type WASITimestamps = {
+  access: Date;
+  modification: Date;
+  change: Date;
 };
 
-export type File = {
-  name: string;
-  content: string | Uint8Array;
+export type WASIFile = {
+  path: WASIPath; // TODO: This duplication is annoying, lets remove it
+  timestamps: WASITimestamps;
+} & (
+  | {
+      mode: "string";
+      content: string;
+    }
+  | {
+      mode: "binary";
+      content: Uint8Array;
+    }
+);
+
+export type WASIExecutionResult = {
+  exitCode: number;
+  fs: WASIFS;
 };
 
 export type RuntimeMethods = {
@@ -48,11 +80,9 @@ export type RuntimeMethods = {
   interactiveRunFS: (
     runtime: Runtime,
     entryPath: string,
-    fs: FS
+    fs: WASIFS
   ) => Promise<RunResult>;
 
-  interactiveUnsafeCommand: (command: string, fs: FS) => Promise<RunResult>;
-
   interactiveStop: () => void;
 
   headlessRunCode: (
@@ -64,13 +94,7 @@ export type RuntimeMethods = {
   headlessRunFS: (
     runtime: Runtime,
     entryPath: string,
-    fs: FS,
-    stdin?: string
-  ) => Promise<RunResult>;
-
-  headlessUnsafeCommand: (
-    command: string,
-    fs: FS,
+    fs: WASIFS,
     stdin?: string
   ) => Promise<RunResult>;
 };

packages/runtime/.env.development

@@ -0,0 +1,2 @@
+VITE_RUNTIME=http://localhost:1234
+VITE_HOST=http://localhost:4321

packages/runtime/.env.production

@@ -0,0 +1,2 @@
+VITE_RUNTIME=https://runno.run
+VITE_HOST=https://runno.dev