Refactor E2E test runner to run FileCheck on compile category tests. (F…

…uelLabs#2450)

This refactors the existing code in the test runner and generalizes it
so FileCheck-based checks are also done for tests in the compile
category.

This will be used in an upcoming PR for
FuelLabs#2423 to make sure that DCA
warnings are not present in tests
that should pass.

test/src/e2e_vm_tests/harness.rs

@@ -95,60 +95,69 @@ pub(crate) fn runs_in_vm(file_name: &str, locked: bool) -> (ProgramState, Compil
     (*i.transact(tx_to_test).unwrap().state(), script)
 }
 
-/// Returns Err(()) if code _does_ compile, used for test cases where the source
-/// code should have been rejected by the compiler.  When it fails to compile the
-/// captured stdout is returned.
-pub(crate) fn does_not_compile(file_name: &str, locked: bool) -> Result<String, ()> {
-    use std::io::Read;
-
+/// Compiles the code and captures the output of forc and the compilation.
+/// Returns a tuple with the result of the compilation, as well as the output.
+pub(crate) fn compile_and_capture_output(
+    file_name: &str,
+    locked: bool,
+) -> (Result<Compiled>, String) {
     tracing::info!(" Compiling {}", file_name);
 
-    // Capture stdout to a buffer, compile the test and save stdout to a string.
-    let mut buf = gag::BufferRedirect::stdout().unwrap();
-    let result = compile_to_bytes_verbose(file_name, locked, true);
-    let mut output = String::new();
-    buf.read_to_string(&mut output).unwrap();
-    drop(buf);
+    let (result, mut output) = compile_to_bytes_verbose(file_name, locked, true, true);
 
     // If verbosity is requested then print it out.
     if get_test_config_from_env() {
         tracing::info!("{output}");
     }
 
-    // Invert the result; if it succeeds then return an Err.
-    match result {
-        Ok(_) => Err(()),
-        Err(e) => {
-            // Capture the result of the compilation (i.e., any errors Forc produces) and append to
-            // the stdout from the compiler.
-            write!(output, "\n{}", e).map_err(|_| ())?;
-            Ok(output)
-        }
+    // Capture the result of the compilation (i.e., any errors Forc produces) and append to
+    // the stdout from the compiler.
+    if let Err(ref e) = result {
+        write!(output, "\n{}", e).expect("error writing output");
     }
+
+    (result, output)
 }
 
-/// Returns `true` if a file compiled without any errors or warnings,
-/// and `false` if it did not.
+/// Compiles the code and returns a result of the compilation,
 pub(crate) fn compile_to_bytes(file_name: &str, locked: bool) -> Result<Compiled> {
-    compile_to_bytes_verbose(file_name, locked, get_test_config_from_env())
+    compile_to_bytes_verbose(file_name, locked, get_test_config_from_env(), false).0
 }
 
 pub(crate) fn compile_to_bytes_verbose(
     file_name: &str,
     locked: bool,
     verbose: bool,
-) -> Result<Compiled> {
+    capture_output: bool,
+) -> (Result<Compiled>, String) {
+    use std::io::Read;
     tracing::info!(" Compiling {}", file_name);
+
+    let mut buf: Option<gag::BufferRedirect> = None;
+    if capture_output {
+        // Capture stdout to a buffer, compile the test and save stdout to a string.
+        buf = Some(gag::BufferRedirect::stdout().unwrap());
+    }
+
     let manifest_dir = env!("CARGO_MANIFEST_DIR");
-    forc_build::build(BuildCommand {
+    let compiled = forc_build::build(BuildCommand {
         path: Some(format!(
             "{}/src/e2e_vm_tests/test_programs/{}",
             manifest_dir, file_name
         )),
         locked,
         silent_mode: !verbose,
         ..Default::default()
-    })
+    });
+
+    let mut output = String::new();
+    if capture_output {
+        let mut buf = buf.unwrap();
+        buf.read_to_string(&mut output).unwrap();
+        drop(buf);
+    }
+
+    (compiled, output)
 }
 
 pub(crate) fn test_json_abi(file_name: &str, compiled: &Compiled) -> Result<()> {

test/src/e2e_vm_tests/mod.rs

@@ -90,8 +90,12 @@ pub fn run(locked: bool, filter_regex: Option<&regex::Regex>) {
             }
 
             TestCategory::Compiles => {
-                let result = crate::e2e_vm_tests::harness::compile_to_bytes(&name, locked);
+                let (result, output) =
+                    crate::e2e_vm_tests::harness::compile_and_capture_output(&name, locked);
+
                 assert!(result.is_ok());
+                check_file_checker(checker, &name, &output);
+
                 let compiled = result.unwrap();
                 if validate_abi {
                     assert!(crate::e2e_vm_tests::harness::test_json_abi(&name, &compiled).is_ok());
@@ -106,19 +110,15 @@ pub fn run(locked: bool, filter_regex: Option<&regex::Regex>) {
             }
 
             TestCategory::FailsToCompile => {
-                match crate::e2e_vm_tests::harness::does_not_compile(&name, locked) {
-                    Ok(output) => match checker.explain(&output, filecheck::NO_VARIABLES) {
-                        Ok((success, report)) if !success => {
-                            panic!("For {name}:\nFilecheck failed:\n{report}");
-                        }
-                        Err(e) => {
-                            panic!("For {name}:\nFilecheck directive error: {e}");
-                        }
-                        _ => (),
-                    },
-                    Err(_) => {
+                let (result, output) =
+                    crate::e2e_vm_tests::harness::compile_and_capture_output(&name, locked);
+                match result {
+                    Ok(_) => {
                         panic!("For {name}:\nFailing test did not fail.");
                     }
+                    Err(_) => {
+                        check_file_checker(checker, &name, &output);
+                    }
                 }
                 number_of_tests_executed += 1;
             }
@@ -233,6 +233,22 @@ fn build_file_checker(content: &str) -> Result<filecheck::Checker, String> {
     Ok(checker.finish())
 }
 
+/// This functions gets passed the previously built FileCheck-based file checker,
+/// along with the output of the compilation, and checks the output for the
+/// FileCheck directives that were found in the test.toml file, panicking
+/// if the checking fails.
+fn check_file_checker(checker: filecheck::Checker, name: &String, output: &str) {
+    match checker.explain(output, filecheck::NO_VARIABLES) {
+        Ok((success, report)) if !success => {
+            panic!("For {name}:\nFilecheck failed:\n{report}");
+        }
+        Err(e) => {
+            panic!("For {name}:\nFilecheck directive error: {e}");
+        }
+        _ => (),
+    }
+}
+
 fn parse_test_toml(path: &Path) -> Result<TestDescription, String> {
     let toml_content_str = std::fs::read_to_string(path).map_err(|e| e.to_string())?;