test(wasi) Run the WASI test suites with the in-memory FS.

This PR updates how we generate the WASI test suites to test against
the `wasmer_vfs::host_fs` (the default), and `wasmer_vfs::mem_fs`
(that's new).

build.rs

@@ -64,18 +64,27 @@ fn main() -> anyhow::Result<()> {
             buffer: String::new(),
             path: vec![],
         };
-        let wasi_versions = ["unstable", "snapshot1"];
+
         with_test_module(&mut wasitests, "wasitests", |wasitests| {
-            for wasi_version in &wasi_versions {
+            for wasi_version in &["unstable", "snapshot1"] {
                 with_test_module(wasitests, wasi_version, |wasitests| {
-                    let _wasi_tests = test_directory(
-                        wasitests,
-                        format!("tests/wasi-wast/wasi/{}", wasi_version),
-                        wasi_processor,
-                    )?;
+                    for (wasi_filesystem_test_name, wasi_filesystem_kind) in &[
+                        ("host_fs", "WasiFileSystemKind::Host"),
+                        ("mem_fs", "WasiFileSystemKind::InMemory"),
+                    ] {
+                        with_test_module(wasitests, wasi_filesystem_test_name, |wasitests| {
+                            test_directory(
+                                wasitests,
+                                format!("tests/wasi-wast/wasi/{}", wasi_version),
+                                |out, path| wasi_processor(out, path, wasi_filesystem_kind),
+                            )
+                        })?;
+                    }
+
                     Ok(())
                 })?;
             }
+
             Ok(())
         })?;
 

tests/compilers/main.rs

@@ -20,3 +20,4 @@ mod wast;
 pub use crate::config::{Compiler, Config, Engine};
 pub use crate::wasi::run_wasi;
 pub use crate::wast::run_wast;
+pub use wasmer_wast::WasiFileSystemKind;

tests/compilers/wasi.rs

@@ -1,20 +1,27 @@
 use std::fs::File;
 use std::io::Read;
-use wasmer_wast::WasiTest;
+use wasmer_wast::{WasiFileSystemKind, WasiTest};
 
 // The generated tests (from build.rs) look like:
 // #[cfg(test)]
-// mod singlepass {
-//     mod spec {
-//         #[test]
-//         fn address() -> anyhow::Result<()> {
-//             crate::run_wast("tests/spectests/address.wast", "singlepass")
+// mod [compiler] {
+//     mod [spec] {
+//         mod [vfs] {
+//             #[test]
+//             fn [test_name]() -> anyhow::Result<()> {
+//                 crate::run_wasi("tests/spectests/[test_name].wast", "[compiler]", WasiFileSystemKind::[vfs])
+//             }
 //         }
 //     }
 // }
 include!(concat!(env!("OUT_DIR"), "/generated_wasitests.rs"));
 
-pub fn run_wasi(config: crate::Config, wast_path: &str, base_dir: &str) -> anyhow::Result<()> {
+pub fn run_wasi(
+    config: crate::Config,
+    wast_path: &str,
+    base_dir: &str,
+    filesystem_kind: WasiFileSystemKind,
+) -> anyhow::Result<()> {
     println!("Running wasi wast `{}`", wast_path);
     let store = config.store();
 
@@ -27,7 +34,8 @@ pub fn run_wasi(config: crate::Config, wast_path: &str, base_dir: &str) -> anyho
     let tokens = WasiTest::lex_string(&source)?;
     let wasi_test = WasiTest::parse_tokens(&tokens)?;
 
-    let succeeded = wasi_test.run(&store, base_dir)?;
+    let succeeded = wasi_test.run(&store, base_dir, filesystem_kind)?;
+
     assert!(succeeded);
 
     Ok(())

tests/compilers/wast.rs

@@ -4,11 +4,13 @@ use wasmer_wast::Wast;
 
 // The generated tests (from build.rs) look like:
 // #[cfg(test)]
-// mod singlepass {
-//     mod spec {
-//         #[test]
-//         fn address() -> anyhow::Result<()> {
-//             crate::run_wast("tests/spectests/address.wast", "singlepass")
+// mod [compiler] {
+//     mod [spec] {
+//         mod [vfs] {
+//             #[test]
+//             fn [test_name]() -> anyhow::Result<()> {
+//                 crate::run_wasi("tests/spectests/[test_name].wast", "[compiler]", WasiFileSystemKind::[vfs])
+//             }
 //         }
 //     }
 // }

tests/ignores.txt

@@ -71,53 +71,82 @@ cranelift spec::simd::simd_int_to_int_extend
 ### These tests don't pass due to race conditions in the new way we run tests.
 ### It's not built to be run in parallel with itself, so we disable it for now.
 
-wasitests::snapshot1::writing
-wasitests::unstable::writing
+wasitests::snapshot1::host_fs::writing
+wasitests::unstable::host_fs::writing
+wasitests::snapshot1::mem_fs::writing
+wasitests::unstable::mem_fs::writing
 
 ### due to hard-coded direct calls into WASI for wasi unstable
 
-wasitests::snapshot1::fd_read
-wasitests::snapshot1::poll_oneoff
-wasitests::snapshot1::fd_pread
-wasitests::snapshot1::fd_close
-wasitests::snapshot1::fd_allocate
-wasitests::snapshot1::close_preopen_fd
-wasitests::snapshot1::envvar
+wasitests::snapshot1::host_fs::fd_read
+wasitests::snapshot1::host_fs::poll_oneoff
+wasitests::snapshot1::host_fs::fd_pread
+wasitests::snapshot1::host_fs::fd_close
+wasitests::snapshot1::host_fs::fd_allocate
+wasitests::snapshot1::host_fs::close_preopen_fd
+wasitests::snapshot1::host_fs::envvar
+wasitests::snapshot1::mem_fs::fd_read
+wasitests::snapshot1::mem_fs::poll_oneoff
+wasitests::snapshot1::mem_fs::fd_pread
+wasitests::snapshot1::mem_fs::fd_close
+wasitests::snapshot1::mem_fs::fd_allocate
+wasitests::snapshot1::mem_fs::close_preopen_fd
+wasitests::snapshot1::mem_fs::envvar
 
 ### TODO: resolve the disabled tests below. These are newly disabled tests from the migration:
 
 ### due to git clone not preserving symlinks:
-wasitests::snapshot1::readlink
-wasitests::unstable::readlink
+wasitests::snapshot1::host_fs::readlink
+wasitests::unstable::host_fs::readlink
+wasitests::snapshot1::mem_fs::readlink
+wasitests::unstable::mem_fs::readlink
 
 ### failing due to `remove_dir_all`. this test is also bad for parallelism
-wasitests::snapshot1::create_dir
-wasitests::unstable::create_dir
+wasitests::snapshot1::host_fs::create_dir
+wasitests::unstable::host_fs::create_dir
+wasitests::snapshot1::mem_fs::create_dir
+wasitests::unstable::mem_fs::create_dir
 
 ### failing because it closes `stdout` which breaks our testing system
-wasitests::unstable::fd_close
+wasitests::unstable::host_fs::fd_close
+wasitests::unstable::mem_fs::fd_close
 
 ### failing because we're operating on stdout which is now overridden.
 ### TODO: check WasiFile implementation
 ### Alterative: split test into 2 parts, one printing to stderr, the other printing to stdout to test the real versions
-wasitests::unstable::poll_oneoff
+wasitests::unstable::host_fs::poll_oneoff
+wasitests::unstable::mem_fs::poll_oneoff
 
 ## Failing due to different line endings on Windows
 ## we need a better solution to this problem:
 
-windows wasitests::snapshot1::file_metadata
-windows wasitests::snapshot1::fseek
-windows wasitests::snapshot1::path_link
-windows wasitests::snapshot1::path_symlink
-windows wasitests::snapshot1::mapdir_with_leading_slash
-windows wasitests::unstable::fd_pread
-windows wasitests::unstable::fd_read
-windows wasitests::unstable::file_metadata
-windows wasitests::unstable::fseek
-windows wasitests::unstable::path_link
-windows wasitests::unstable::path_symlink
-windows wasitests::unstable::mapdir_with_leading_slash
+windows wasitests::snapshot1::host_fs::file_metadata
+windows wasitests::snapshot1::host_fs::fseek
+windows wasitests::snapshot1::host_fs::path_link
+windows wasitests::snapshot1::host_fs::path_symlink
+windows wasitests::snapshot1::host_fs::mapdir_with_leading_slash
+windows wasitests::unstable::host_fs::fd_pread
+windows wasitests::unstable::host_fs::fd_read
+windows wasitests::unstable::host_fs::file_metadata
+windows wasitests::unstable::host_fs::fseek
+windows wasitests::unstable::host_fs::path_link
+windows wasitests::unstable::host_fs::path_symlink
+windows wasitests::unstable::host_fs::mapdir_with_leading_slash
+windows wasitests::snapshot1::mem_fs::file_metadata
+windows wasitests::snapshot1::mem_fs::fseek
+windows wasitests::snapshot1::mem_fs::path_link
+windows wasitests::snapshot1::mem_fs::path_symlink
+windows wasitests::snapshot1::mem_fs::mapdir_with_leading_slash
+windows wasitests::unstable::mem_fs::fd_pread
+windows wasitests::unstable::mem_fs::fd_read
+windows wasitests::unstable::mem_fs::file_metadata
+windows wasitests::unstable::mem_fs::fseek
+windows wasitests::unstable::mem_fs::path_link
+windows wasitests::unstable::mem_fs::path_symlink
+windows wasitests::unstable::mem_fs::mapdir_with_leading_slash
 
 # This tests are disabled for now
-wasitests::unstable::unix_open_special_files
-wasitests::snapshot1::unix_open_special_files
+wasitests::unstable::host_fs::unix_open_special_files
+wasitests::snapshot1::host_fs::unix_open_special_files
+wasitests::unstable::mem_fs::unix_open_special_files
+wasitests::snapshot1::mem_fs::unix_open_special_files

tests/lib/test-generator/src/lib.rs

@@ -23,12 +23,10 @@ pub struct Test {
     pub body: String,
 }
 
-pub type ProcessorType = fn(&mut Testsuite, PathBuf) -> Option<Test>;
-
 pub fn test_directory_module(
     out: &mut Testsuite,
     path: impl AsRef<Path>,
-    processor: ProcessorType,
+    processor: impl Fn(&mut Testsuite, PathBuf) -> Option<Test>,
 ) -> anyhow::Result<usize> {
     let path = path.as_ref();
     let testsuite = &extract_name(path);
@@ -55,7 +53,7 @@ fn write_test(out: &mut Testsuite, testname: &str, body: &str) -> anyhow::Result
 pub fn test_directory(
     out: &mut Testsuite,
     path: impl AsRef<Path>,
-    processor: ProcessorType,
+    processor: impl Fn(&mut Testsuite, PathBuf) -> Option<Test>,
 ) -> anyhow::Result<usize> {
     let path = path.as_ref();
     let mut dir_entries: Vec<_> = path

tests/lib/test-generator/src/processors.rs

@@ -63,7 +63,11 @@ pub fn emscripten_processor(_out: &mut Testsuite, p: PathBuf) -> Option<Test> {
 
 /// Given a Testsuite and a path, process the path in case is a WASI
 /// wasm file.
-pub fn wasi_processor(_out: &mut Testsuite, p: PathBuf) -> Option<Test> {
+pub fn wasi_processor(
+    _out: &mut Testsuite,
+    p: PathBuf,
+    wasi_filesystem_kind: &str,
+) -> Option<Test> {
     let ext = p.extension()?;
     // Only look at wast files.
     if ext != "wast" {
@@ -77,11 +81,11 @@ pub fn wasi_processor(_out: &mut Testsuite, p: PathBuf) -> Option<Test> {
     };
     let testname = extract_name(&p);
 
-    // The implementation of `run_wasi` lives in /tests/wasitest.rs
     let body = format!(
-        "crate::run_wasi(config, r#\"{}\"#, \"{}\")",
+        "crate::run_wasi(config, r#\"{}\"#, \"{}\", crate::{})",
         p.display(),
         wasm_dir.display(),
+        wasi_filesystem_kind,
     );
 
     Some(Test {

tests/lib/wast/src/lib.rs

@@ -25,7 +25,7 @@ mod wast;
 
 pub use crate::error::{DirectiveError, DirectiveErrors};
 pub use crate::spectest::spectest_importobject;
-pub use crate::wasi_wast::WasiTest;
+pub use crate::wasi_wast::{WasiFileSystemKind, WasiTest};
 pub use crate::wast::Wast;
 
 /// Version number of this crate.

tests/lib/wast/src/wasi_wast.rs

@@ -10,6 +10,16 @@ use wasmer_wasi::{
 };
 use wast::parser::{self, Parse, ParseBuffer, Parser};
 
+/// The kind of filesystem `WasiTest` is going to use.
+#[derive(Debug)]
+pub enum WasiFileSystemKind {
+    /// Instruct the test runner to use `wasmer_vfs::host_fs`.
+    Host,
+
+    /// Instruct the test runner to use `wasmer_vfs::mem_fs`.
+    InMemory,
+}
+
 /// Crate holding metadata parsed from the WASI WAST about the test to be run.
 #[derive(Debug, Clone, Hash)]
 pub struct WasiTest<'a> {
@@ -70,7 +80,12 @@ impl<'a> WasiTest<'a> {
     }
 
     /// Execute the WASI test and assert.
-    pub fn run(&self, store: &Store, base_path: &str) -> anyhow::Result<bool> {
+    pub fn run(
+        &self,
+        store: &Store,
+        base_path: &str,
+        filesystem_kind: WasiFileSystemKind,
+    ) -> anyhow::Result<bool> {
         let mut pb = PathBuf::from(base_path);
         pb.push(self.wasm_path);
         let wasm_bytes = {
@@ -85,12 +100,14 @@ impl<'a> WasiTest<'a> {
         let instance = Instance::new(&module, &imports)?;
 
         let start = instance.exports.get_function("_start")?;
+
         if let Some(stdin) = &self.stdin {
             let mut state = env.state();
             let wasi_stdin = state.fs.stdin_mut()?.as_mut().unwrap();
             // Then we can write to it!
             write!(wasi_stdin, "{}", stdin.stream)?;
         }
+
         // TODO: handle errors here when the error fix gets shipped
         match start.call(&[]) {
             Ok(_) => {}
@@ -114,6 +131,7 @@ impl<'a> WasiTest<'a> {
             let stdout_str = get_stdout_output(&wasi_state)?;
             assert_eq!(stdout_str, expected_stdout.expected);
         }
+
         if let Some(expected_stderr) = &self.assert_stderr {
             let stderr_str = get_stderr_output(&wasi_state)?;
             assert_eq!(stderr_str, expected_stderr.expected);