From 971915a8c18cc8c803a09205dd723d881b9aa1e0 Mon Sep 17 00:00:00 2001 From: Ivan Enderlin Date: Tue, 31 Aug 2021 11:26:14 +0200 Subject: [PATCH] test(wasi) Run the WASI test suites with the in-memory FS. This PR updates how we generate the WASI test suites to test against the `wasmer_vfs::host_fs` (the default), and `wasmer_vfs::mem_fs` (that's new). --- build.rs | 23 ++++-- tests/compilers/main.rs | 1 + tests/compilers/wasi.rs | 24 ++++-- tests/compilers/wast.rs | 12 +-- tests/ignores.txt | 87 ++++++++++++++-------- tests/lib/test-generator/src/lib.rs | 6 +- tests/lib/test-generator/src/processors.rs | 10 ++- tests/lib/wast/src/lib.rs | 2 +- tests/lib/wast/src/wasi_wast.rs | 20 ++++- 9 files changed, 127 insertions(+), 58 deletions(-) diff --git a/build.rs b/build.rs index 649c56085f9..7707daa1848 100644 --- a/build.rs +++ b/build.rs @@ -64,18 +64,27 @@ fn main() -> anyhow::Result<()> { buffer: String::new(), path: vec![], }; - let wasi_versions = ["unstable", "snapshot1"]; + with_test_module(&mut wasitests, "wasitests", |wasitests| { - for wasi_version in &wasi_versions { + for wasi_version in &["unstable", "snapshot1"] { with_test_module(wasitests, wasi_version, |wasitests| { - let _wasi_tests = test_directory( - wasitests, - format!("tests/wasi-wast/wasi/{}", wasi_version), - wasi_processor, - )?; + for (wasi_filesystem_test_name, wasi_filesystem_kind) in &[ + ("host_fs", "WasiFileSystemKind::Host"), + ("mem_fs", "WasiFileSystemKind::InMemory"), + ] { + with_test_module(wasitests, wasi_filesystem_test_name, |wasitests| { + test_directory( + wasitests, + format!("tests/wasi-wast/wasi/{}", wasi_version), + |out, path| wasi_processor(out, path, wasi_filesystem_kind), + ) + })?; + } + Ok(()) })?; } + Ok(()) })?; diff --git a/tests/compilers/main.rs b/tests/compilers/main.rs index 5b85deb0382..290d31c8fd0 100644 --- a/tests/compilers/main.rs +++ b/tests/compilers/main.rs @@ -20,3 +20,4 @@ mod wast; pub use crate::config::{Compiler, Config, Engine}; pub use crate::wasi::run_wasi; pub use crate::wast::run_wast; +pub use wasmer_wast::WasiFileSystemKind; diff --git a/tests/compilers/wasi.rs b/tests/compilers/wasi.rs index 8261299e1e5..be339e3db78 100644 --- a/tests/compilers/wasi.rs +++ b/tests/compilers/wasi.rs @@ -1,20 +1,27 @@ use std::fs::File; use std::io::Read; -use wasmer_wast::WasiTest; +use wasmer_wast::{WasiFileSystemKind, WasiTest}; // The generated tests (from build.rs) look like: // #[cfg(test)] -// mod singlepass { -// mod spec { -// #[test] -// fn address() -> anyhow::Result<()> { -// crate::run_wast("tests/spectests/address.wast", "singlepass") +// mod [compiler] { +// mod [spec] { +// mod [vfs] { +// #[test] +// fn [test_name]() -> anyhow::Result<()> { +// crate::run_wasi("tests/spectests/[test_name].wast", "[compiler]", WasiFileSystemKind::[vfs]) +// } // } // } // } include!(concat!(env!("OUT_DIR"), "/generated_wasitests.rs")); -pub fn run_wasi(config: crate::Config, wast_path: &str, base_dir: &str) -> anyhow::Result<()> { +pub fn run_wasi( + config: crate::Config, + wast_path: &str, + base_dir: &str, + filesystem_kind: WasiFileSystemKind, +) -> anyhow::Result<()> { println!("Running wasi wast `{}`", wast_path); let store = config.store(); @@ -27,7 +34,8 @@ pub fn run_wasi(config: crate::Config, wast_path: &str, base_dir: &str) -> anyho let tokens = WasiTest::lex_string(&source)?; let wasi_test = WasiTest::parse_tokens(&tokens)?; - let succeeded = wasi_test.run(&store, base_dir)?; + let succeeded = wasi_test.run(&store, base_dir, filesystem_kind)?; + assert!(succeeded); Ok(()) diff --git a/tests/compilers/wast.rs b/tests/compilers/wast.rs index 5978dbdb2b6..fdae6d97cba 100644 --- a/tests/compilers/wast.rs +++ b/tests/compilers/wast.rs @@ -4,11 +4,13 @@ use wasmer_wast::Wast; // The generated tests (from build.rs) look like: // #[cfg(test)] -// mod singlepass { -// mod spec { -// #[test] -// fn address() -> anyhow::Result<()> { -// crate::run_wast("tests/spectests/address.wast", "singlepass") +// mod [compiler] { +// mod [spec] { +// mod [vfs] { +// #[test] +// fn [test_name]() -> anyhow::Result<()> { +// crate::run_wasi("tests/spectests/[test_name].wast", "[compiler]", WasiFileSystemKind::[vfs]) +// } // } // } // } diff --git a/tests/ignores.txt b/tests/ignores.txt index e02ce8b74c9..3ebd993dce5 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -71,53 +71,82 @@ cranelift spec::simd::simd_int_to_int_extend ### These tests don't pass due to race conditions in the new way we run tests. ### It's not built to be run in parallel with itself, so we disable it for now. -wasitests::snapshot1::writing -wasitests::unstable::writing +wasitests::snapshot1::host_fs::writing +wasitests::unstable::host_fs::writing +wasitests::snapshot1::mem_fs::writing +wasitests::unstable::mem_fs::writing ### due to hard-coded direct calls into WASI for wasi unstable -wasitests::snapshot1::fd_read -wasitests::snapshot1::poll_oneoff -wasitests::snapshot1::fd_pread -wasitests::snapshot1::fd_close -wasitests::snapshot1::fd_allocate -wasitests::snapshot1::close_preopen_fd -wasitests::snapshot1::envvar +wasitests::snapshot1::host_fs::fd_read +wasitests::snapshot1::host_fs::poll_oneoff +wasitests::snapshot1::host_fs::fd_pread +wasitests::snapshot1::host_fs::fd_close +wasitests::snapshot1::host_fs::fd_allocate +wasitests::snapshot1::host_fs::close_preopen_fd +wasitests::snapshot1::host_fs::envvar +wasitests::snapshot1::mem_fs::fd_read +wasitests::snapshot1::mem_fs::poll_oneoff +wasitests::snapshot1::mem_fs::fd_pread +wasitests::snapshot1::mem_fs::fd_close +wasitests::snapshot1::mem_fs::fd_allocate +wasitests::snapshot1::mem_fs::close_preopen_fd +wasitests::snapshot1::mem_fs::envvar ### TODO: resolve the disabled tests below. These are newly disabled tests from the migration: ### due to git clone not preserving symlinks: -wasitests::snapshot1::readlink -wasitests::unstable::readlink +wasitests::snapshot1::host_fs::readlink +wasitests::unstable::host_fs::readlink +wasitests::snapshot1::mem_fs::readlink +wasitests::unstable::mem_fs::readlink ### failing due to `remove_dir_all`. this test is also bad for parallelism -wasitests::snapshot1::create_dir -wasitests::unstable::create_dir +wasitests::snapshot1::host_fs::create_dir +wasitests::unstable::host_fs::create_dir +wasitests::snapshot1::mem_fs::create_dir +wasitests::unstable::mem_fs::create_dir ### failing because it closes `stdout` which breaks our testing system -wasitests::unstable::fd_close +wasitests::unstable::host_fs::fd_close +wasitests::unstable::mem_fs::fd_close ### failing because we're operating on stdout which is now overridden. ### TODO: check WasiFile implementation ### Alterative: split test into 2 parts, one printing to stderr, the other printing to stdout to test the real versions -wasitests::unstable::poll_oneoff +wasitests::unstable::host_fs::poll_oneoff +wasitests::unstable::mem_fs::poll_oneoff ## Failing due to different line endings on Windows ## we need a better solution to this problem: -windows wasitests::snapshot1::file_metadata -windows wasitests::snapshot1::fseek -windows wasitests::snapshot1::path_link -windows wasitests::snapshot1::path_symlink -windows wasitests::snapshot1::mapdir_with_leading_slash -windows wasitests::unstable::fd_pread -windows wasitests::unstable::fd_read -windows wasitests::unstable::file_metadata -windows wasitests::unstable::fseek -windows wasitests::unstable::path_link -windows wasitests::unstable::path_symlink -windows wasitests::unstable::mapdir_with_leading_slash +windows wasitests::snapshot1::host_fs::file_metadata +windows wasitests::snapshot1::host_fs::fseek +windows wasitests::snapshot1::host_fs::path_link +windows wasitests::snapshot1::host_fs::path_symlink +windows wasitests::snapshot1::host_fs::mapdir_with_leading_slash +windows wasitests::unstable::host_fs::fd_pread +windows wasitests::unstable::host_fs::fd_read +windows wasitests::unstable::host_fs::file_metadata +windows wasitests::unstable::host_fs::fseek +windows wasitests::unstable::host_fs::path_link +windows wasitests::unstable::host_fs::path_symlink +windows wasitests::unstable::host_fs::mapdir_with_leading_slash +windows wasitests::snapshot1::mem_fs::file_metadata +windows wasitests::snapshot1::mem_fs::fseek +windows wasitests::snapshot1::mem_fs::path_link +windows wasitests::snapshot1::mem_fs::path_symlink +windows wasitests::snapshot1::mem_fs::mapdir_with_leading_slash +windows wasitests::unstable::mem_fs::fd_pread +windows wasitests::unstable::mem_fs::fd_read +windows wasitests::unstable::mem_fs::file_metadata +windows wasitests::unstable::mem_fs::fseek +windows wasitests::unstable::mem_fs::path_link +windows wasitests::unstable::mem_fs::path_symlink +windows wasitests::unstable::mem_fs::mapdir_with_leading_slash # This tests are disabled for now -wasitests::unstable::unix_open_special_files -wasitests::snapshot1::unix_open_special_files +wasitests::unstable::host_fs::unix_open_special_files +wasitests::snapshot1::host_fs::unix_open_special_files +wasitests::unstable::mem_fs::unix_open_special_files +wasitests::snapshot1::mem_fs::unix_open_special_files diff --git a/tests/lib/test-generator/src/lib.rs b/tests/lib/test-generator/src/lib.rs index c2620183f0f..2c4f30f7501 100644 --- a/tests/lib/test-generator/src/lib.rs +++ b/tests/lib/test-generator/src/lib.rs @@ -23,12 +23,10 @@ pub struct Test { pub body: String, } -pub type ProcessorType = fn(&mut Testsuite, PathBuf) -> Option; - pub fn test_directory_module( out: &mut Testsuite, path: impl AsRef, - processor: ProcessorType, + processor: impl Fn(&mut Testsuite, PathBuf) -> Option, ) -> anyhow::Result { let path = path.as_ref(); let testsuite = &extract_name(path); @@ -55,7 +53,7 @@ fn write_test(out: &mut Testsuite, testname: &str, body: &str) -> anyhow::Result pub fn test_directory( out: &mut Testsuite, path: impl AsRef, - processor: ProcessorType, + processor: impl Fn(&mut Testsuite, PathBuf) -> Option, ) -> anyhow::Result { let path = path.as_ref(); let mut dir_entries: Vec<_> = path diff --git a/tests/lib/test-generator/src/processors.rs b/tests/lib/test-generator/src/processors.rs index c1c596bc0b4..ce7ca22cc3a 100644 --- a/tests/lib/test-generator/src/processors.rs +++ b/tests/lib/test-generator/src/processors.rs @@ -63,7 +63,11 @@ pub fn emscripten_processor(_out: &mut Testsuite, p: PathBuf) -> Option { /// Given a Testsuite and a path, process the path in case is a WASI /// wasm file. -pub fn wasi_processor(_out: &mut Testsuite, p: PathBuf) -> Option { +pub fn wasi_processor( + _out: &mut Testsuite, + p: PathBuf, + wasi_filesystem_kind: &str, +) -> Option { let ext = p.extension()?; // Only look at wast files. if ext != "wast" { @@ -77,11 +81,11 @@ pub fn wasi_processor(_out: &mut Testsuite, p: PathBuf) -> Option { }; let testname = extract_name(&p); - // The implementation of `run_wasi` lives in /tests/wasitest.rs let body = format!( - "crate::run_wasi(config, r#\"{}\"#, \"{}\")", + "crate::run_wasi(config, r#\"{}\"#, \"{}\", crate::{})", p.display(), wasm_dir.display(), + wasi_filesystem_kind, ); Some(Test { diff --git a/tests/lib/wast/src/lib.rs b/tests/lib/wast/src/lib.rs index 6f98a12ef67..a782730eecf 100644 --- a/tests/lib/wast/src/lib.rs +++ b/tests/lib/wast/src/lib.rs @@ -25,7 +25,7 @@ mod wast; pub use crate::error::{DirectiveError, DirectiveErrors}; pub use crate::spectest::spectest_importobject; -pub use crate::wasi_wast::WasiTest; +pub use crate::wasi_wast::{WasiFileSystemKind, WasiTest}; pub use crate::wast::Wast; /// Version number of this crate. diff --git a/tests/lib/wast/src/wasi_wast.rs b/tests/lib/wast/src/wasi_wast.rs index a588d59f994..459bc0839f8 100644 --- a/tests/lib/wast/src/wasi_wast.rs +++ b/tests/lib/wast/src/wasi_wast.rs @@ -10,6 +10,16 @@ use wasmer_wasi::{ }; use wast::parser::{self, Parse, ParseBuffer, Parser}; +/// The kind of filesystem `WasiTest` is going to use. +#[derive(Debug)] +pub enum WasiFileSystemKind { + /// Instruct the test runner to use `wasmer_vfs::host_fs`. + Host, + + /// Instruct the test runner to use `wasmer_vfs::mem_fs`. + InMemory, +} + /// Crate holding metadata parsed from the WASI WAST about the test to be run. #[derive(Debug, Clone, Hash)] pub struct WasiTest<'a> { @@ -70,7 +80,12 @@ impl<'a> WasiTest<'a> { } /// Execute the WASI test and assert. - pub fn run(&self, store: &Store, base_path: &str) -> anyhow::Result { + pub fn run( + &self, + store: &Store, + base_path: &str, + filesystem_kind: WasiFileSystemKind, + ) -> anyhow::Result { let mut pb = PathBuf::from(base_path); pb.push(self.wasm_path); let wasm_bytes = { @@ -85,12 +100,14 @@ impl<'a> WasiTest<'a> { let instance = Instance::new(&module, &imports)?; let start = instance.exports.get_function("_start")?; + if let Some(stdin) = &self.stdin { let mut state = env.state(); let wasi_stdin = state.fs.stdin_mut()?.as_mut().unwrap(); // Then we can write to it! write!(wasi_stdin, "{}", stdin.stream)?; } + // TODO: handle errors here when the error fix gets shipped match start.call(&[]) { Ok(_) => {} @@ -114,6 +131,7 @@ impl<'a> WasiTest<'a> { let stdout_str = get_stdout_output(&wasi_state)?; assert_eq!(stdout_str, expected_stdout.expected); } + if let Some(expected_stderr) = &self.assert_stderr { let stderr_str = get_stderr_output(&wasi_state)?; assert_eq!(stderr_str, expected_stderr.expected);