Skip to content

Commit

Permalink
Refactor E2E CLI runner. (FuelLabs#3294)
Browse files Browse the repository at this point in the history 
Now output handling is done in an uniform manner for all kinds of tests
and makes it possible to implement better developer experience.

Now the output is hidden by default for all tests, and only shown up
when a particular test is failing (indented so its easier to parse).

Failure handling is also more robust, with all tests being executed by
default, and failures being reported as they happen and in the summary
report at end of execution. There are still a couple panics here and
there for more esoteric errors, but those can be easily be transformed
to errors in the future as need arises.

Previous behavior of showing all output can be obtained by passing the
`--verbose` flag to the runner.

The only exception are the IR tests which still panic. However their
summary report was updated to match the improvements from above.

Closes FuelLabs#3060.

Example of new output with failed test report because `fuel-core` is not
running:

![Screenshot from 2022-11-09
17-53-05](https://user-images.githubusercontent.com/602268/200904172-c9327b78-b557-49b4-9e4f-d78c3d5a2d7b.png)

Co-authored-by: Mohammad Fawaz <[email protected]>
  • Loading branch information
@tritao @mohammadfawaz
tritao and mohammadfawaz authored Nov 14, 2022
1 parent c462cbc commit 4fe5a38
Show file tree
Hide file tree
Showing 6 changed files with 276 additions and 169 deletions.
22 changes: 22 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions test/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ regex = "1"
serde_json = "1.0.73"
sway-core = { path = "../sway-core" }
sway-ir = { path = "../sway-ir" }
textwrap = "0.16.0"
tokio = "1.12"
toml = "0.5"
tracing = "0.1"
219 changes: 110 additions & 109 deletions test/src/e2e_vm_tests/harness.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use anyhow::{bail, Result};
use colored::Colorize;
use forc_client::ops::{
deploy::{cmd::DeployCommand, op::deploy},
run::{cmd::RunCommand, op::run},
Expand All @@ -8,20 +9,55 @@ use fuel_tx::TransactionBuilder;
use fuel_vm::fuel_tx;
use fuel_vm::interpreter::Interpreter;
use fuel_vm::prelude::*;
use futures::Future;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use regex::{Captures, Regex};
use std::{fmt::Write, fs, io::Read, path::PathBuf, str::FromStr};
use std::{fs, io::Read, path::PathBuf, str::FromStr};

use super::RunConfig;

pub const NODE_URL: &str = "http://127.0.0.1:4000";
pub const SECRET_KEY: &str = "de97d8624a438121b86a1956544bd72ed68cd69f2c99555b08b1e8c51ffd511c";

pub(crate) async fn run_and_capture_output<F, Fut, T>(func: F) -> (T, String)
where
F: FnOnce() -> Fut,
Fut: Future<Output = T>,
{
let mut output = String::new();

// Capture both stdout and stderr to buffers, run the code and save to a string.
let buf_stdout = Some(gag::BufferRedirect::stdout().unwrap());
let buf_stderr = Some(gag::BufferRedirect::stderr().unwrap());

let result = func().await;

let mut buf_stdout = buf_stdout.unwrap();
let mut buf_stderr = buf_stderr.unwrap();
buf_stdout.read_to_string(&mut output).unwrap();
buf_stderr.read_to_string(&mut output).unwrap();
drop(buf_stdout);
drop(buf_stderr);

if cfg!(windows) {
// In windows output error and warning path files start with \\?\
// We replace \ by / so tests can check unix paths only
let regex = Regex::new(r"\\\\?\\(.*)").unwrap();
output = regex
.replace_all(output.as_str(), |caps: &Captures| {
caps[1].replace('\\', "/")
})
.to_string();
}

(result, output)
}

pub(crate) async fn deploy_contract(file_name: &str, run_config: &RunConfig) -> Result<ContractId> {
// build the contract
// deploy it
tracing::info!(" Deploying {}", file_name);
println!(" Deploying {} ...", file_name.bold());
let manifest_dir = env!("CARGO_MANIFEST_DIR");

deploy(DeployCommand {
Expand All @@ -48,43 +84,45 @@ pub(crate) async fn runs_on_node(
file_name: &str,
run_config: &RunConfig,
contract_ids: &[fuel_tx::ContractId],
) -> Result<Vec<fuel_tx::Receipt>> {
tracing::info!("Running on node: {}", file_name);
let manifest_dir = env!("CARGO_MANIFEST_DIR");
) -> (Result<Vec<fuel_tx::Receipt>>, String) {
run_and_capture_output(|| async {
println!(" Running on node {} ...", file_name.bold());
let manifest_dir = env!("CARGO_MANIFEST_DIR");

let mut contracts = Vec::<String>::with_capacity(contract_ids.len());
for contract_id in contract_ids {
let contract = format!("0x{:x}", contract_id);
contracts.push(contract);
}
let mut contracts = Vec::<String>::with_capacity(contract_ids.len());
for contract_id in contract_ids {
let contract = format!("0x{:x}", contract_id);
contracts.push(contract);
}

let command = RunCommand {
path: Some(format!(
"{}/src/e2e_vm_tests/test_programs/{}",
manifest_dir, file_name
)),
node_url: Some(NODE_URL.into()),
terse_mode: !run_config.verbose,
contract: Some(contracts),
locked: run_config.locked,
signing_key: Some(SecretKey::from_str(SECRET_KEY).unwrap()),
..Default::default()
};
run(command).await.map(|ran_scripts| {
ran_scripts
.into_iter()
.next()
.map(|ran_script| ran_script.receipts)
.unwrap()
let command = RunCommand {
path: Some(format!(
"{}/src/e2e_vm_tests/test_programs/{}",
manifest_dir, file_name
)),
node_url: Some(NODE_URL.into()),
terse_mode: !run_config.verbose,
contract: Some(contracts),
locked: run_config.locked,
signing_key: Some(SecretKey::from_str(SECRET_KEY).unwrap()),
..Default::default()
};
run(command).await.map(|ran_scripts| {
ran_scripts
.into_iter()
.next()
.map(|ran_script| ran_script.receipts)
.unwrap()
})
})
.await
}

/// Very basic check that code does indeed run in the VM.
/// `true` if it does, `false` if not.
pub(crate) fn runs_in_vm(
script: BuiltPackage,
script_data: Option<Vec<u8>>,
) -> (ProgramState, Vec<Receipt>, BuiltPackage) {
) -> Result<(ProgramState, Vec<Receipt>, BuiltPackage)> {
let storage = MemoryStorage::default();

let rng = &mut StdRng::seed_from_u64(2322u64);
Expand All @@ -104,27 +142,14 @@ pub(crate) fn runs_in_vm(
.finalize_checked(block_height as Word, params);

let mut i = Interpreter::with_storage(storage, Default::default());
let transition = i.transact(tx).unwrap();
(*transition.state(), transition.receipts().to_vec(), script)
let transition = i.transact(tx)?;
Ok((*transition.state(), transition.receipts().to_vec(), script))
}

/// Compiles the code and optionally captures the output of forc and the compilation.
/// Returns a tuple with the result of the compilation, as well as the output.
pub(crate) fn compile_to_bytes(
file_name: &str,
run_config: &RunConfig,
capture_output: bool,
) -> (Result<Built>, String) {
tracing::info!(" Compiling {}", file_name);

let mut buf_stdout: Option<gag::BufferRedirect> = None;
let mut buf_stderr: Option<gag::BufferRedirect> = None;
if capture_output {
// Capture both stdout and stderr to buffers, compile the test and save to a string.
buf_stdout = Some(gag::BufferRedirect::stdout().unwrap());
buf_stderr = Some(gag::BufferRedirect::stderr().unwrap());
}

pub(crate) async fn compile_to_bytes(file_name: &str, run_config: &RunConfig) -> Result<Built> {
println!("Compiling {} ...", file_name.bold());
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let build_opts = forc_pkg::BuildOpts {
pkg: forc_pkg::PkgOpts {
Expand All @@ -133,83 +158,59 @@ pub(crate) fn compile_to_bytes(
manifest_dir, file_name
)),
locked: run_config.locked,
terse: !(capture_output || run_config.verbose),
terse: false,
..Default::default()
},
..Default::default()
};
let result = forc_pkg::build_with_options(build_opts);

let mut output = String::new();
if capture_output {
let mut buf_stdout = buf_stdout.unwrap();
let mut buf_stderr = buf_stderr.unwrap();
buf_stdout.read_to_string(&mut output).unwrap();
buf_stderr.read_to_string(&mut output).unwrap();
drop(buf_stdout);
drop(buf_stderr);

// If verbosity is requested then print it out.
if run_config.verbose {
tracing::info!("{output}");
}

// Capture the result of the compilation (i.e., any errors Forc produces) and append to
// the stdout from the compiler.
if let Err(ref e) = result {
write!(output, "\n{}", e).expect("error writing output");
}

if cfg!(windows) {
// In windows output error and warning path files start with \\?\
// We replace \ by / so tests can check unix paths only
let regex = Regex::new(r"\\\\?\\(.*)").unwrap();
output = regex
.replace_all(output.as_str(), |caps: &Captures| {
caps[1].replace('\\', "/")
})
.to_string();
}
// Print the result of the compilation (i.e., any errors Forc produces).
if let Err(ref e) = result {
println!("\n{}", e);
}

(result, output)
result
}

/// Compiles the project's unit tests, then runs all unit tests.
/// Returns the tested package result.
pub(crate) fn compile_and_run_unit_tests(
pub(crate) async fn compile_and_run_unit_tests(
file_name: &str,
run_config: &RunConfig,
capture_output: bool,
) -> Result<Box<forc_test::TestedPackage>> {
tracing::info!(" Testing {}", file_name);

let manifest_dir = env!("CARGO_MANIFEST_DIR");
let path: PathBuf = [
manifest_dir,
"src",
"e2e_vm_tests",
"test_programs",
file_name,
]
.iter()
.collect();
let built_tests = forc_test::build(forc_test::Opts {
pkg: forc_pkg::PkgOpts {
path: Some(path.to_string_lossy().into_owned()),
locked: run_config.locked,
terse: !(capture_output || run_config.verbose),
) -> (Result<Box<forc_test::TestedPackage>>, String) {
run_and_capture_output(|| async {
tracing::info!("Compiling {} ...", file_name.bold());
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let path: PathBuf = [
manifest_dir,
"src",
"e2e_vm_tests",
"test_programs",
file_name,
]
.iter()
.collect();
let built_tests = forc_test::build(forc_test::Opts {
pkg: forc_pkg::PkgOpts {
path: Some(path.to_string_lossy().into_owned()),
locked: run_config.locked,
terse: !(capture_output || run_config.verbose),
..Default::default()
},
..Default::default()
},
..Default::default()
})?;
let tested = built_tests.run()?;
match tested {
forc_test::Tested::Package(tested_pkg) => Ok(tested_pkg),
forc_test::Tested::Workspace => {
bail!("testing full workspaces not yet implemented")
})?;
let tested = built_tests.run()?;

match tested {
forc_test::Tested::Package(tested_pkg) => Ok(tested_pkg),
forc_test::Tested::Workspace => Err(anyhow::Error::msg(
"testing full workspaces not yet implemented",
)),
}
}
})
.await
}

pub(crate) fn test_json_abi(file_name: &str, built_package: &BuiltPackage) -> Result<()> {
Expand Down Expand Up @@ -240,7 +241,7 @@ pub(crate) fn test_json_abi(file_name: &str, built_package: &BuiltPackage) -> Re
}

fn emit_json_abi(file_name: &str, built_package: &BuiltPackage) -> Result<()> {
tracing::info!(" ABI gen {}", file_name);
tracing::info!("ABI gen {} ...", file_name.bold());
let json_abi = serde_json::json!(built_package.json_abi_program);
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let file = std::fs::File::create(format!(
Expand Down Expand Up @@ -280,7 +281,7 @@ pub(crate) fn test_json_storage_slots(file_name: &str, built_package: &BuiltPack
}

fn emit_json_storage_slots(file_name: &str, built_package: &BuiltPackage) -> Result<()> {
tracing::info!(" storage slots JSON gen {}", file_name);
tracing::info!("Storage slots JSON gen {} ...", file_name.bold());
let json_storage_slots = serde_json::json!(built_package.storage_slots);
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let file = std::fs::File::create(format!(
Expand Down
Loading

0 comments on commit 4fe5a38

Please sign in to comment.