Update test runners to accept optional --locked flag. Use it in CI. (…

…FuelLabs#1964)

* Update E2E test runner to accept `--locked` flag

This allows users to run the tests without the `--locked` flag by
default, which is often the easiest way to update the `Forc.lock` files
for each test locally.

* Move `--locked` flag to an arg in stdlib test build script

This allows us to only lock the tests and runner during CI, making it
easier for users to update lock files locally by running the build
script.

.github/workflows/ci.yml

@@ -293,7 +293,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: run
-          args: --locked --release --bin test
+          args: --locked --release --bin test -- --locked
 
   # TODO: Remove this upon merging std tests with the rest of the E2E tests.
   cargo-test-lib-std:
@@ -327,7 +327,7 @@ jobs:
           command: run
           args: --locked --bin examples-checker build --all-examples
       - name: Build All Tests
-        run: cd test/src/sdk-harness && bash build.sh && cd ../../../
+        run: cd test/src/sdk-harness && bash build.sh --locked && cd ../../../
       - name: Cargo Test sway-lib-std
         uses: actions-rs/cargo@v1
         with:

test/src/e2e_vm_tests/harness.rs

@@ -9,7 +9,7 @@ use fuel_vm::prelude::*;
 use serde_json::Value;
 use std::fs;
 
-pub(crate) fn deploy_contract(file_name: &str) -> ContractId {
+pub(crate) fn deploy_contract(file_name: &str, locked: bool) -> ContractId {
     // build the contract
     // deploy it
     tracing::info!(" Deploying {}", file_name);
@@ -25,7 +25,7 @@ pub(crate) fn deploy_contract(file_name: &str) -> ContractId {
                 manifest_dir, file_name
             )),
             silent_mode: !verbose,
-            locked: true,
+            locked,
             ..Default::default()
         }))
         .unwrap()
@@ -34,6 +34,7 @@ pub(crate) fn deploy_contract(file_name: &str) -> ContractId {
 /// Run a given project against a node. Assumes the node is running at localhost:4000.
 pub(crate) fn runs_on_node(
     file_name: &str,
+    locked: bool,
     contract_ids: &[fuel_tx::ContractId],
 ) -> Vec<fuel_tx::Receipt> {
     tracing::info!("Running on node: {}", file_name);
@@ -55,7 +56,7 @@ pub(crate) fn runs_on_node(
         node_url: "http://127.0.0.1:4000".into(),
         silent_mode: !verbose,
         contract: Some(contracts),
-        locked: true,
+        locked,
         ..Default::default()
     };
     tokio::runtime::Runtime::new()
@@ -66,10 +67,10 @@ pub(crate) fn runs_on_node(
 
 /// Very basic check that code does indeed run in the VM.
 /// `true` if it does, `false` if not.
-pub(crate) fn runs_in_vm(file_name: &str) -> ProgramState {
+pub(crate) fn runs_in_vm(file_name: &str, locked: bool) -> ProgramState {
     let storage = MemoryStorage::default();
 
-    let script = compile_to_bytes(file_name).unwrap();
+    let script = compile_to_bytes(file_name, locked).unwrap();
     let gas_price = 10;
     let gas_limit = fuel_tx::default_parameters::MAX_GAS_PER_TX;
     let byte_price = 0;
@@ -99,17 +100,17 @@ pub(crate) fn runs_in_vm(file_name: &str) -> ProgramState {
 
 /// Panics if code _does_ compile, used for test cases where the source
 /// code should have been rejected by the compiler.
-pub(crate) fn does_not_compile(file_name: &str) {
+pub(crate) fn does_not_compile(file_name: &str, locked: bool) {
     assert!(
-        compile_to_bytes(file_name).is_err(),
+        compile_to_bytes(file_name, locked).is_err(),
         "{} should not have compiled.",
         file_name,
     )
 }
 
 /// Returns `true` if a file compiled without any errors or warnings,
 /// and `false` if it did not.
-pub(crate) fn compile_to_bytes(file_name: &str) -> Result<Vec<u8>> {
+pub(crate) fn compile_to_bytes(file_name: &str, locked: bool) -> Result<Vec<u8>> {
     tracing::info!(" Compiling {}", file_name);
     let manifest_dir = env!("CARGO_MANIFEST_DIR");
     let verbose = get_test_config_from_env();
@@ -118,7 +119,7 @@ pub(crate) fn compile_to_bytes(file_name: &str) -> Result<Vec<u8>> {
             "{}/src/e2e_vm_tests/test_programs/{}",
             manifest_dir, file_name
         )),
-        locked: true,
+        locked,
         silent_mode: !verbose,
         ..Default::default()
     })

test/src/e2e_vm_tests/mod.rs

@@ -2,7 +2,7 @@ mod harness;
 use assert_matches::assert_matches;
 use forc_util::init_tracing_subscriber;
 use fuel_vm::prelude::*;
-pub fn run(filter_regex: Option<regex::Regex>) {
+pub fn run(locked: bool, filter_regex: Option<regex::Regex>) {
     init_tracing_subscriber();
     let filter = |name| {
         filter_regex
@@ -24,7 +24,7 @@ pub fn run(filter_regex: Option<regex::Regex>) {
             .iter()
             .fold(0, |acc, (name, res)| {
                 if filter(name) {
-                    assert_eq!(crate::e2e_vm_tests::harness::runs_in_vm(name), *res);
+                    assert_eq!(crate::e2e_vm_tests::harness::runs_in_vm(name, locked), *res);
                     acc + 1
                 } else {
                     acc
@@ -505,7 +505,7 @@ pub fn run(filter_regex: Option<regex::Regex>) {
         .iter()
         .fold(0, |acc, (name, res)| {
             if filter(name) {
-                assert_eq!(crate::e2e_vm_tests::harness::runs_in_vm(name), *res);
+                assert_eq!(crate::e2e_vm_tests::harness::runs_in_vm(name, locked), *res);
                 assert_matches!(crate::e2e_vm_tests::harness::test_json_abi(name), Ok(_));
                 acc + 1
             } else {
@@ -593,7 +593,7 @@ pub fn run(filter_regex: Option<regex::Regex>) {
     ];
     number_of_tests_run += negative_project_names.iter().fold(0, |acc, name| {
         if filter(name) {
-            crate::e2e_vm_tests::harness::does_not_compile(name);
+            crate::e2e_vm_tests::harness::does_not_compile(name, locked);
             acc + 1
         } else {
             acc
@@ -692,12 +692,12 @@ pub fn run(filter_regex: Option<regex::Regex>) {
     number_of_tests_run += projects.len();
     let mut contract_ids = Vec::<fuel_tx::ContractId>::with_capacity(contracts.len());
     for name in contracts {
-        let contract_id = harness::deploy_contract(name);
+        let contract_id = harness::deploy_contract(name, locked);
         contract_ids.push(contract_id);
     }
 
     for (name, val) in projects.iter().zip(vals.iter()) {
-        let result = harness::runs_on_node(name, &contract_ids);
+        let result = harness::runs_on_node(name, locked, &contract_ids);
         assert!(result.iter().all(|r| !matches!(
             r,
             fuel_tx::Receipt::Revert { .. } | fuel_tx::Receipt::Panic { .. }

test/src/main.rs

@@ -1,10 +1,25 @@
 mod e2e_vm_tests;
 
 fn main() {
-    let filter_regex = std::env::args().nth(1).map(|filter_str| {
-        regex::Regex::new(&filter_str)
-            .unwrap_or_else(|_| panic!("Invalid filter regex: '{}'.", filter_str))
-    });
+    let mut locked = false;
+    let mut filter_regex = None;
+    for arg in std::env::args().skip(1) {
+        // Check for the `--locked` flag. Must precede the regex.
+        // Intended for use in `CI` to ensure test lock files are up to date.
+        if arg == "--locked" {
+            locked = true;
+            continue;
+        }
 
-    e2e_vm_tests::run(filter_regex);
+        // Check for a regex, used to filter the set of tests.
+        let regex = regex::Regex::new(&arg).unwrap_or_else(|_| {
+            panic!(
+                "Expected either `--locked` or a filter regex, found: {:?}.",
+                arg
+            )
+        });
+        filter_regex = Some(regex);
+    }
+
+    e2e_vm_tests::run(locked, filter_regex);
 }

test/src/sdk-harness/build.sh

@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+# Pass `--locked` when running this script in CI to ensure both cargo and forc
+# lock files are up to date.
+locked="$1"
+
 # Cross platform version of `realpath` or `readlink`.
 abs_path() {
   (cd "$1"; pwd)
@@ -13,7 +17,7 @@ parent_manifest_dir="${base_dir}"
 while true; do
   parent_manifest_dir=$(abs_path "${parent_manifest_dir}/..")
   if [[ -f "${parent_manifest_dir}/Cargo.toml" ]]; then
-    forc="cargo run --locked --manifest-path ${parent_manifest_dir}/Cargo.toml --package forc --"
+    forc="cargo run $locked --manifest-path ${parent_manifest_dir}/Cargo.toml --package forc --"
     break
   fi
   if [[ "${parent_manifest_dir}" = "/" ]]; then
@@ -28,7 +32,7 @@ test_dirs="${base_dir}/test_artifacts/* ${base_dir}/test_projects/*"
 for test_dir in $test_dirs; do
   if [[ -f "${test_dir}/Forc.toml" ]]; then
     echo "Building test $test_dir..."
-    ${forc} build --locked -o temp -p "${test_dir}" && echo ✔
+    ${forc} build $locked -o temp -p "${test_dir}" && echo ✔
     if ! [[ -f temp ]]; then
       echo  "❌  Failed to build $test_dir"
       exit 1