Add WASI test runner

build.rs

@@ -9,8 +9,8 @@ use std::fs;
 use std::path::PathBuf;
 use std::process::Command;
 use test_generator::{
-    build_ignores_from_textfile, test_directory, test_directory_module, wast_processor,
-    with_features, with_test_module, Testsuite,
+    build_ignores_from_textfile, test_directory, test_directory_module, wasi_processor,
+    wast_processor, with_features, with_test_module, Testsuite,
 };
 
 fn main() -> anyhow::Result<()> {
@@ -21,41 +21,75 @@ fn main() -> anyhow::Result<()> {
         env::var_os("OUT_DIR").expect("The OUT_DIR environment variable must be set"),
     );
     let ignores = build_ignores_from_textfile("tests/ignores.txt".into())?;
+    let backends = vec!["singlepass", "cranelift", "llvm"];
 
     // Spectests test generation
-    let mut spectests = Testsuite {
-        buffer: String::new(),
-        path: vec![],
-        ignores,
-    };
+    {
+        let mut spectests = Testsuite {
+            buffer: String::new(),
+            path: vec![],
+            ignores: ignores.clone(),
+        };
 
-    let backends = vec!["singlepass", "cranelift", "llvm"];
-    with_features(&mut spectests, &backends, |mut spectests| {
-        with_test_module(&mut spectests, "spec", |spectests| {
-            let _spec_tests = test_directory(spectests, "tests/wast/spec", wast_processor)?;
-            test_directory_module(
-                spectests,
-                "tests/wast/spec/proposals/multi-value",
-                wast_processor,
-            )?;
-            // test_directory_module(spectests, "tests/wast/spec/proposals/bulk-memory-operations", wast_processor)?;
+        with_features(&mut spectests, &backends, |mut spectests| {
+            with_test_module(&mut spectests, "spec", |spectests| {
+                let _spec_tests = test_directory(spectests, "tests/wast/spec", wast_processor)?;
+                test_directory_module(
+                    spectests,
+                    "tests/wast/spec/proposals/multi-value",
+                    wast_processor,
+                )?;
+                // test_directory_module(spectests, "tests/wast/spec/proposals/bulk-memory-operations", wast_processor)?;
+                Ok(())
+            })?;
+            with_test_module(&mut spectests, "wasmer", |spectests| {
+                let _spec_tests = test_directory(spectests, "tests/wast/wasmer", wast_processor)?;
+                Ok(())
+            })?;
             Ok(())
         })?;
-        with_test_module(&mut spectests, "wasmer", |spectests| {
-            let _spec_tests = test_directory(spectests, "tests/wast/wasmer", wast_processor)?;
+
+        let spectests_output = out_dir.join("generated_spectests.rs");
+        fs::write(&spectests_output, spectests.buffer)?;
+
+        // Write out our auto-generated tests and opportunistically format them with
+        // `rustfmt` if it's installed.
+        // Note: We need drop because we don't want to run `unwrap` or `expect` as
+        // the command might fail, but we don't care about it's result.
+        drop(Command::new("rustfmt").arg(&spectests_output).status());
+    }
+
+    // Wasitest test generation
+    {
+        let mut wasitests = Testsuite {
+            buffer: String::new(),
+            path: vec![],
+            ignores,
+        };
+        let wasi_versions = ["unstable", "snapshot1"];
+        with_features(&mut wasitests, &backends, |mut wasitests| {
+            for wasi_version in &wasi_versions {
+                with_test_module(
+                    &mut wasitests,
+                    &format!("wasitests_{}", wasi_version),
+                    |wasitests| {
+                        let _wasi_tests = test_directory(
+                            wasitests,
+                            format!("tests/wasi/wasi/{}", wasi_version),
+                            wasi_processor,
+                        )?;
+                        Ok(())
+                    },
+                )?;
+            }
             Ok(())
         })?;
-        Ok(())
-    })?;
 
-    let spectests_output = out_dir.join("generated_spectests.rs");
-    fs::write(&spectests_output, spectests.buffer)?;
+        let wasitests_output = out_dir.join("generated_wasitests.rs");
+        fs::write(&wasitests_output, wasitests.buffer)?;
 
-    // Write out our auto-generated tests and opportunistically format them with
-    // `rustfmt` if it's installed.
-    // Note: We need drop because we don't want to run `unwrap` or `expect` as
-    // the command might fail, but we don't care about it's result.
-    drop(Command::new("rustfmt").arg(&spectests_output).status());
+        drop(Command::new("rustfmt").arg(&wasitests_output).status());
+    }
 
     Ok(())
 }

tests/lib/test-generator/src/processors.rs

@@ -69,28 +69,24 @@ pub fn emscripten_processor(out: &mut Testsuite, p: PathBuf) -> Option<Test> {
 pub fn wasi_processor(out: &mut Testsuite, p: PathBuf) -> Option<Test> {
     let ext = p.extension()?;
     // Only look at wast files.
-    if ext != "wasm" {
+    // TODO: fix this once we rename `out` to `wast`
+    if ext != "out" {
         return None;
     }
 
-    let outfile = {
-        let mut out_ext = p.clone();
-        out_ext.set_extension("out");
-        if out_ext.exists() {
-            out_ext
-        } else {
-            return None;
-        }
+    let wasm_dir = {
+        let mut inner = p.clone();
+        inner.pop();
+        inner
     };
-
     let testname = extract_name(&p);
     let compiler = out.path.get(0).unwrap();
 
     // The implementation of `run_wasi` lives in /tests/wasitest.rs
     let body = format!(
-        "crate::wasi::run_wasi(r#\"{}\"#, r#\"{}\"#, \"{}\")",
+        "crate::run_wasi(r#\"{}\"#, \"{}\", \"{}\")",
         p.display(),
-        outfile.display(),
+        wasm_dir.display(),
         compiler
     );
 

tests/lib/wast/Cargo.toml

@@ -13,6 +13,7 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0"
 wasmer = { path = "../../../lib/api", version = "1.0.0-alpha.1", default-features = false }
+wasmer-wasi = { path = "../../../lib/wasi", version = "1.0.0-alpha.1" }
 wast = "17.0"
 thiserror = "1.0"
 

tests/lib/wast/src/lib.rs

@@ -26,6 +26,7 @@ mod wast;
 
 pub use crate::error::{DirectiveError, DirectiveErrors};
 pub use crate::spectest::spectest_importobject;
+pub use crate::wasi_wast::WasiTest;
 pub use crate::wast::Wast;
 
 /// Version number of this crate.

tests/lib/wast/src/wasi_wast.rs

@@ -1,7 +1,17 @@
-use wast::parser::{self, Parse, Parser};
-
+use anyhow::Context;
+use std::fs::File;
+use std::io::Read;
+use std::path::PathBuf;
+use std::sync::Arc;
+use wasmer::{ImportObject, Instance, Memory, Module, Store};
+use wasmer_wasi::{
+    generate_import_object_from_env, get_wasi_version, WasiEnv, WasiState, WasiVersion,
+};
+use wast::parser::{self, Parse, ParseBuffer, Parser};
+
+/// Crate holding metadata parsed from the WASI WAST about the test to be run.
 #[derive(Debug, Clone, Hash)]
-pub(crate) struct WasiTest<'a> {
+pub struct WasiTest<'a> {
     wasm_path: &'a str,
     args: Vec<&'a str>,
     envs: Vec<(&'a str, &'a str)>,
@@ -12,6 +22,85 @@ pub(crate) struct WasiTest<'a> {
     assert_stderr: Option<AssertStderr<'a>>,
 }
 
+#[allow(dead_code)]
+impl<'a> WasiTest<'a> {
+    /// Turn a WASI WAST string into a list of tokens.
+    pub fn lex_string(wast: &'a str) -> parser::Result<ParseBuffer<'a>> {
+        ParseBuffer::new(wast)
+    }
+
+    /// Turn a WASI WAST list of tokens into a `WasiTest` struct.
+    pub fn parse_tokens(tokens: &'a ParseBuffer<'a>) -> parser::Result<Self> {
+        parser::parse(tokens)
+    }
+
+    /// Execute the WASI test and assert.
+    pub fn run(&self, store: &Store, base_path: &str) -> anyhow::Result<bool> {
+        let mut pb = PathBuf::from(base_path);
+        pb.push(self.wasm_path);
+        let wasm_bytes = {
+            let mut wasm_module = File::open(pb)?;
+            let mut out = vec![];
+            wasm_module.read_to_end(&mut out)?;
+            out
+        };
+        let module = Module::new(&store, &wasm_bytes)?;
+        let mut env = self.create_wasi_env()?;
+        let imports = self.get_imports(store, &module, env.clone())?;
+        let instance = Instance::new(&module, &imports)?;
+        let memory: &Memory = instance.exports.get("memory")?;
+        // TODO:
+        env.set_memory(Arc::new(memory.clone()));
+
+        let start = instance.exports.get_function("_start")?;
+        start
+            .call(&[])
+            .with_context(|| "failed to run WASI `_start` function")?;
+        Ok(true)
+    }
+
+    /// Create the wasi env with the given metadata.
+    fn create_wasi_env(&self) -> anyhow::Result<WasiEnv> {
+        let mut builder = WasiState::new(self.wasm_path);
+        for (name, value) in &self.envs {
+            builder.env(name, value);
+        }
+        // TODO: implement map dirs
+        /*
+        // TODO: check the order
+        for (alias, real_dir) in &self.mapped_dirs {
+            builder.map_dir(alias, real_dir);
+        }*/
+
+        let out = builder
+            .args(&self.args)
+            .preopen_dirs(&self.dirs)?
+            // TODO: capture stdout and stderr
+            // can be done with a custom file, inserted here
+            .finalize()?;
+        Ok(out)
+    }
+
+    /// Get the correct [`WasiVersion`] from the Wasm [`Module`].
+    fn get_version(&self, module: &Module) -> anyhow::Result<WasiVersion> {
+        let version = get_wasi_version(module, true)
+            .with_context(|| "failed to detect a version of WASI from the module")?;
+        Ok(version)
+    }
+
+    /// Get the correct WASI import object for the given module and set it up with the
+    /// [`WasiEnv`].
+    fn get_imports(
+        &self,
+        store: &Store,
+        module: &Module,
+        env: WasiEnv,
+    ) -> anyhow::Result<ImportObject> {
+        let version = self.get_version(module)?;
+        Ok(generate_import_object_from_env(store, env, version))
+    }
+}
+
 mod wasi_kw {
     wast::custom_keyword!(wasi_test);
     wast::custom_keyword!(envs);

tests/lib/wast/src/wast.rs

@@ -504,5 +504,3 @@ impl NaNCheck for f64 {
         (self.to_bits() & 0x7fff_ffff_ffff_ffff) == 0x7ff8_0000_0000_0000
     }
 }
-
-wast::custom_keyword!(wasi_test);

tests/wasi.rs

@@ -0,0 +1,47 @@
+#![cfg(all(feature = "compiler", feature = "engine"))]
+
+use std::fs::File;
+use std::io::Read;
+use std::sync::Arc;
+use test_utils::get_compiler_config_from_str;
+use wasmer::{Features, Store, Tunables};
+#[cfg(feature = "jit")]
+use wasmer_engine_jit::JITEngine;
+use wasmer_wast::WasiTest;
+
+// The generated tests (from build.rs) look like:
+// #[cfg(test)]
+// mod singlepass {
+//     mod spec {
+//         #[test]
+//         fn address() -> anyhow::Result<()> {
+//             crate::run_wast("tests/spectests/address.wast", "singlepass")
+//         }
+//     }
+// }
+include!(concat!(env!("OUT_DIR"), "/generated_wasitests.rs"));
+
+fn run_wasi(wast_path: &str, base_dir: &str, compiler: &str) -> anyhow::Result<()> {
+    println!(
+        "Running wasi wast `{}` with the {} compiler",
+        wast_path, compiler
+    );
+    let features = Features::default();
+    let compiler_config = get_compiler_config_from_str(compiler, false, features);
+    let tunables = Tunables::for_target(compiler_config.target().triple());
+    let store = Store::new(Arc::new(JITEngine::new(compiler_config, tunables)));
+
+    let source = {
+        let mut out = String::new();
+        let mut f = File::open(wast_path)?;
+        f.read_to_string(&mut out);
+        out
+    };
+    let tokens = WasiTest::lex_string(&source)?;
+    let wasi_test = WasiTest::parse_tokens(&tokens)?;
+
+    let succeeded = wasi_test.run(&store, base_dir)?;
+    assert!(succeeded);
+
+    Ok(())
+}