From ccc11c037efde9cf03f41c2930c3317c85054eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Matos?= Date: Wed, 3 Aug 2022 17:47:11 +0100 Subject: [PATCH] Refactor E2E test runner to run FileCheck on compile category tests. (#2450) This refactors the existing code in the test runner and generalizes it so FileCheck-based checks are also done for tests in the compile category. This will be used in an upcoming PR for https://github.com/FuelLabs/sway/issues/2423 to make sure that DCA warnings are not present in tests that should pass. --- test/src/e2e_vm_tests/harness.rs | 63 ++++++++++++++++++-------------- test/src/e2e_vm_tests/mod.rs | 40 ++++++++++++++------ 2 files changed, 64 insertions(+), 39 deletions(-) diff --git a/test/src/e2e_vm_tests/harness.rs b/test/src/e2e_vm_tests/harness.rs index 43c75ccd559..bac1af45070 100644 --- a/test/src/e2e_vm_tests/harness.rs +++ b/test/src/e2e_vm_tests/harness.rs @@ -95,52 +95,52 @@ pub(crate) fn runs_in_vm(file_name: &str, locked: bool) -> (ProgramState, Compil (*i.transact(tx_to_test).unwrap().state(), script) } -/// Returns Err(()) if code _does_ compile, used for test cases where the source -/// code should have been rejected by the compiler. When it fails to compile the -/// captured stdout is returned. -pub(crate) fn does_not_compile(file_name: &str, locked: bool) -> Result { - use std::io::Read; - +/// Compiles the code and captures the output of forc and the compilation. +/// Returns a tuple with the result of the compilation, as well as the output. +pub(crate) fn compile_and_capture_output( + file_name: &str, + locked: bool, +) -> (Result, String) { tracing::info!(" Compiling {}", file_name); - // Capture stdout to a buffer, compile the test and save stdout to a string. - let mut buf = gag::BufferRedirect::stdout().unwrap(); - let result = compile_to_bytes_verbose(file_name, locked, true); - let mut output = String::new(); - buf.read_to_string(&mut output).unwrap(); - drop(buf); + let (result, mut output) = compile_to_bytes_verbose(file_name, locked, true, true); // If verbosity is requested then print it out. if get_test_config_from_env() { tracing::info!("{output}"); } - // Invert the result; if it succeeds then return an Err. - match result { - Ok(_) => Err(()), - Err(e) => { - // Capture the result of the compilation (i.e., any errors Forc produces) and append to - // the stdout from the compiler. - write!(output, "\n{}", e).map_err(|_| ())?; - Ok(output) - } + // Capture the result of the compilation (i.e., any errors Forc produces) and append to + // the stdout from the compiler. + if let Err(ref e) = result { + write!(output, "\n{}", e).expect("error writing output"); } + + (result, output) } -/// Returns `true` if a file compiled without any errors or warnings, -/// and `false` if it did not. +/// Compiles the code and returns a result of the compilation, pub(crate) fn compile_to_bytes(file_name: &str, locked: bool) -> Result { - compile_to_bytes_verbose(file_name, locked, get_test_config_from_env()) + compile_to_bytes_verbose(file_name, locked, get_test_config_from_env(), false).0 } pub(crate) fn compile_to_bytes_verbose( file_name: &str, locked: bool, verbose: bool, -) -> Result { + capture_output: bool, +) -> (Result, String) { + use std::io::Read; tracing::info!(" Compiling {}", file_name); + + let mut buf: Option = None; + if capture_output { + // Capture stdout to a buffer, compile the test and save stdout to a string. + buf = Some(gag::BufferRedirect::stdout().unwrap()); + } + let manifest_dir = env!("CARGO_MANIFEST_DIR"); - forc_build::build(BuildCommand { + let compiled = forc_build::build(BuildCommand { path: Some(format!( "{}/src/e2e_vm_tests/test_programs/{}", manifest_dir, file_name @@ -148,7 +148,16 @@ pub(crate) fn compile_to_bytes_verbose( locked, silent_mode: !verbose, ..Default::default() - }) + }); + + let mut output = String::new(); + if capture_output { + let mut buf = buf.unwrap(); + buf.read_to_string(&mut output).unwrap(); + drop(buf); + } + + (compiled, output) } pub(crate) fn test_json_abi(file_name: &str, compiled: &Compiled) -> Result<()> { diff --git a/test/src/e2e_vm_tests/mod.rs b/test/src/e2e_vm_tests/mod.rs index 0f2e1917676..f504a87aab4 100644 --- a/test/src/e2e_vm_tests/mod.rs +++ b/test/src/e2e_vm_tests/mod.rs @@ -90,8 +90,12 @@ pub fn run(locked: bool, filter_regex: Option<®ex::Regex>) { } TestCategory::Compiles => { - let result = crate::e2e_vm_tests::harness::compile_to_bytes(&name, locked); + let (result, output) = + crate::e2e_vm_tests::harness::compile_and_capture_output(&name, locked); + assert!(result.is_ok()); + check_file_checker(checker, &name, &output); + let compiled = result.unwrap(); if validate_abi { assert!(crate::e2e_vm_tests::harness::test_json_abi(&name, &compiled).is_ok()); @@ -106,19 +110,15 @@ pub fn run(locked: bool, filter_regex: Option<®ex::Regex>) { } TestCategory::FailsToCompile => { - match crate::e2e_vm_tests::harness::does_not_compile(&name, locked) { - Ok(output) => match checker.explain(&output, filecheck::NO_VARIABLES) { - Ok((success, report)) if !success => { - panic!("For {name}:\nFilecheck failed:\n{report}"); - } - Err(e) => { - panic!("For {name}:\nFilecheck directive error: {e}"); - } - _ => (), - }, - Err(_) => { + let (result, output) = + crate::e2e_vm_tests::harness::compile_and_capture_output(&name, locked); + match result { + Ok(_) => { panic!("For {name}:\nFailing test did not fail."); } + Err(_) => { + check_file_checker(checker, &name, &output); + } } number_of_tests_executed += 1; } @@ -233,6 +233,22 @@ fn build_file_checker(content: &str) -> Result { Ok(checker.finish()) } +/// This functions gets passed the previously built FileCheck-based file checker, +/// along with the output of the compilation, and checks the output for the +/// FileCheck directives that were found in the test.toml file, panicking +/// if the checking fails. +fn check_file_checker(checker: filecheck::Checker, name: &String, output: &str) { + match checker.explain(output, filecheck::NO_VARIABLES) { + Ok((success, report)) if !success => { + panic!("For {name}:\nFilecheck failed:\n{report}"); + } + Err(e) => { + panic!("For {name}:\nFilecheck directive error: {e}"); + } + _ => (), + } +} + fn parse_test_toml(path: &Path) -> Result { let toml_content_str = std::fs::read_to_string(path).map_err(|e| e.to_string())?;