Skip to content

Commit

Permalink
fix(testool) remove introspection (privacy-scaling-explorations#1098)
Browse files Browse the repository at this point in the history
  • Loading branch information
@adria0
adria0 authored Jan 23, 2023
1 parent 37c9d93 commit 6e3d9dc
Show file tree
Hide file tree
Showing 8 changed files with 85 additions and 245 deletions.
174 changes: 9 additions & 165 deletions testool/Config.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,20 @@ id="default"
path="tests/src/GeneralStateTestsFiller/**/*"
max_gas = 500000
max_steps = 1000
unimplemented_opcodes = []
ignore_tests = []

[[suite]]
id="nightly"
path="tests/src/GeneralStateTestsFiller/**/*"
max_gas = 500000
max_steps = 1000
unimplemented_opcodes = [
"CREATE",
"CREATE2",
"SELFDESTRUCT"
]
ignore_tests=["&tofix", "&sigkill"]
max_gas = 0
max_steps = 100000
ignore_tests=["&sigkill"]

[[suite]]
id = "light"
path="tests/src/GeneralStateTestsFiller/**/*"
max_gas = 500000
max_steps = 1000
unimplemented_opcodes = []
allow_tests=[
"add_d0(add_neg1_neg1)_g0_v0",
"mul_d0(mul_2_3)_g0_v0",
Expand Down Expand Up @@ -173,169 +166,20 @@ allow_tests=[
id = "sigkill"
desc = "tests that sigkill"
tests = [
"CallToNameRegistratorMemOOGAndInsufficientBalance_d0_g0_v0"
]

[[set]]
id = "tofix"
desc = "***panicked at 'circuit should pass', contraint error"
tests=["random_d3(random3)_g0_v0","random_d4(random4)_g0_v0"]

[[set]]
id = "tofix"
desc = "***incorrect rw witness."
tests = ["invalidAddr_d31(ok)_g0_v0"]

[[set]]
id = "tofix"
desc = "gen_associated_ops memory check error"
tests = [
"callDataCopyOffset_d0_g0_v0",
"calldatacopy_dejavu2_d0_g0_v0",
"codeCopyOffset_d0_g0_v0",
"bufferSrcOffset_d10(ok)_g0_v0",
"bufferSrcOffset_d12(ok)_g0_v0",
"bufferSrcOffset_d13(ok)_g0_v0",
"bufferSrcOffset_d14(ok)_g0_v0",
"bufferSrcOffset_d2(ok)_g0_v0",
"bufferSrcOffset_d20(ok)_g0_v0",
"bufferSrcOffset_d21(ok)_g0_v0",
"bufferSrcOffset_d22(ok)_g0_v0",
"bufferSrcOffset_d23(ok)_g0_v0",
"bufferSrcOffset_d26(ok)_g0_v0",
"bufferSrcOffset_d30(ok)_g0_v0",
"bufferSrcOffset_d34(ok)_g0_v0",
"bufferSrcOffset_d36(ok)_g0_v0",
"bufferSrcOffset_d37(ok)_g0_v0",
"bufferSrcOffset_d38(ok)_g0_v0",
"bufferSrcOffset_d44(ok)_g0_v0",
"bufferSrcOffset_d45(ok)_g0_v0",
"bufferSrcOffset_d46(ok)_g0_v0",
"bufferSrcOffset_d47(ok)_g0_v0",
"bufferSrcOffset_d6(ok)_g0_v0"
]

[[set]]
id = "tofix"
desc = "NotEnoughRowsAvailable { current_k: 12 }"
tests = [
"oog_d2(success)_g0_v0",
"oog_d4(success)_g0_v0",
"push32withoutByte_d0_g0_v0"
"CallToNameRegistratorMemOOGAndInsufficientBalance_d0_g0_v0",
"randomStatetest101_d0_g0_v0"
]

[[set]]
id = "tofix"
desc = "a an inexistent contract at address is called, but parse_call() (or alike) does not support it yet"
tests = [
"invalidAddr_d41(ok)_g0_v0",
"underflowTest_d91(CALL-7)_g0_v0",
"CallToNameRegistratorAddressTooBigRight_d0_g0_v0",
"ZeroValue_CALL_d0_g0_v0",
"NonZeroValue_CALL_d0_g0_v0",
"randomStatetest138_d0_g0_v0",
]

[[set]]
id = "tofix"
id = "sigkill"
desc = "Integer overflow when casting to u64"
tests = ["envInfo_d3(codecopy_neg_offset)_g0_v0", "codecopy_dejavu2_d0_g0_v0","randomStatetest222_d0_g0_v0"]

[[set]]
id = "tofix"
desc = "call with value and not enough value to send"
tests= ["callWithHighValue_d0_g0_v0"]

[[set]]
id = "tofix"
desc = "Fully implement PUSHn #633"
tests = ["randomStatetest0_d0_g0_v0"]

# skipped tests, do not need to be fixed --------------------------------------------------

[[skip_tests]]
desc = "maximum geth trace size (10MB)"
tests = [
"randomStatetest163_d0_g0_v0",
"randomStatetest150_d0_g0_v0",
"randomStatetest154_d0_g0_v0",
"randomStatetest159_d0_g0_v0",
"randomStatetest36_d0_g0_v0",
"randomStatetest101_d0_g0_v0",
"randomStatetest185_d0_g0_v0",
"randomStatetest326_d0_g0_v0",
"randomStatetest177_d0_g0_v0",
"randomStatetest178_d0_g0_v0",
"randomStatetest263_d0_g0_v0",
"randomStatetest205_d0_g0_v0",
"randomStatetest209_d0_g0_v0",
"randomStatetest306_d0_g0_v0",
"randomStatetest384_d0_g0_v0",
"randomStatetest48_d0_g0_v0",
"MLOAD_Bounds3_d0_g0_v0",
"MLOAD_Bounds3_d0_g1_v0",
"MSTORE_Bounds2a_d0_g1_v0",
"Call1024PreCalls_d0_g2_v0",
"Call1024PreCalls_d0_g1_v0",
"msize_d5(farChunk)_g0_v0",
"CallcodeToPrecompileFromTransaction_d0_g0_v0",
"CallcodeToPrecompileFromCalledContract_d0_g0_v0",
"CallcodeToPrecompileFromContractInitialization_d0_g0_v0",
"Call1024PreCalls_d0_g1_v0",
"Call1024PreCalls_d0_g0_v0",
"static_Call1024PreCalls2_d0_g0_v0",
"shiftSignedCombinations_d0_g0_v0",
"CreateOOGafterMaxCodesize_d5(HighContractCount_DelegateCreate_CallCreate_SelfDestruct)_g0_v0",
"CallInfiniteLoop_d0_g0_v0",
"static_callCreate2_d2_g0_v0",
"static_CallIdentity_5_d0_g0_v0",
"static_CallSha256_5_d0_g0_v0",
"static_CallRipemd160_5_d0_g0_v0",
"static_callChangeRevert_d1_g0_v0",
"static_callcallcodecall_010_OOGE_d0_g0_v0",
"static_callcallcall_000_OOGMBefore_d0_g0_v0",
"static_callcallcodecall_010_OOGMBefore_d1_g0_v0",
"static_callcallcodecall_010_OOGMBefore2_d0_g0_v0",
"static_callcallcallcode_001_OOGMBefore2_d1_g0_v0",
"static_callcallcallcode_001_OOGMBefore_d0_g0_v0",
"static_Call10_d1_g0_v0",
"static_Call10_d0_g0_v0",
"jsonCall10_d0_g0_v0",
"block504980_d0_g0_v0",
"shiftCombinations_d0_g0_v0",
"static_LoopCallsThenRevert_d0_g1_v0",
"CallRecursiveMethods_d0_g0_v0",
"static_LoopCallsThenRevert_d0_g0_v0",
"static_callcallcodecall_010_OOGMAfter_d1_g0_v0",
"static_callcallcodecall_010_OOGMAfter_2_d1_g0_v0",
"static_callcallcallcode_001_OOGMAfter2_d1_g0_v0",
"static_callcallcallcode_001_OOGMAfter_3_d1_g0_v0",
"static_callcallcodecall_010_OOGMAfter_3_d1_g0_v0",
"static_callcodecallcall_100_OOGMAfter2_d0_g0_v0",
"static_callcallcallcode_001_OOGMAfter_d1_g0_v0",
"static_callcodecallcall_100_OOGMAfter_2_d0_g0_v0",
"static_callcodecallcall_100_OOGMAfter2_d0_g0_v1",
"ContractCreationSpam_d0_g0_v0",
"CreateOOGafterMaxCodesize_d4(HighContractCount_DelegateCreate_CallCreateOOG)_g0_v0",
"static_callcallcall_000_OOGMAfter2_d0_g0_v0",
"static_callcodecallcallcode_101_OOGMAfter_1_d0_g0_v0",
"createInitFailStackSizeLargerThan1024_d0_g0_v0",
"costRevert_d27(stackOver)_g0_v0",
"costRevert_d20(stackOver)_g0_v0",
"costRevert_d13(stackOver)_g0_v0",
"costRevert_d6(stackOver)_g0_v0",
"static_Call1MB1024Calldepth_d1_g0_v0",
"stateRevert_d6(stackOver)_g0_v0",
"DelegatecallToPrecompileFromTransaction_d0_g0_v0",
"DelegatecallToPrecompileFromCalledContract_d0_g0_v0",
"DelegatecallToPrecompileFromContractInitialization_d0_g0_v0",
"CallWithZeroValueToPrecompileFromTransaction_d0_g0_v0",
"CallWithZeroValueToPrecompileFromCalledContract_d0_g0_v0",
"CallWithZeroValueToPrecompileFromContractInitialization_d0_g0_v0",
"CreateOOGafterMaxCodesize_d3(HighContractCount_NoDelegateCreate_CallCreateOOG)_g0_v0",
"twoOps_d0_g0_v0",
"Call10_d0_g0_v0"
]
desc = ""
tests = []

# ignored paths -------------------------------------------------------------------------

Expand All @@ -345,7 +189,7 @@ paths = [
"EIP1559",
"EIP2930",
"stPreCompiledContracts",
"stZeroKnowledge"
"stZeroKnowledge"
]

[[skip_paths]]
Expand Down
18 changes: 10 additions & 8 deletions testool/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ These tests are written in `json` or `yml`, and, in general it specifies 4 secti

You can find (here)[https://ethereum-tests.readthedocs.io/en/latest/test_filler/blockchain_filler.html] the specification for these files in detail.

Official ethereum tests are mantained by the foundation but you can write your own.
Official ethereum tests are maintained by the foundation but you can write your own.

### Configuration file

Expand All @@ -70,11 +70,10 @@ In the config file you define `[[suite]]`s that defines how tests will be execut

- `id` is the identifier of the suite. The default suite is called `default`.
- `max_steps` the maximum number of executed opcodes. If this is reached, the test is marked to be ignored.
- `max_gas` the maximum gas of a test. If the is reached, the test is marked to be ignored.
- `unimplemented_opcodes`, if any of these opcodes are found in the execution trace, the test is marked to be ignored.
- you should define also only one of these paramers:
- `allow_tests` with the list of tests or testsets to execute. All others will be excluded. Test sets should be prefixed with `&`
- `ignore_tests` with the list of test or testsets to ignore. All others will be included. Test sets should be prefixed with `&`
- `max_gas` the maximum gas of a test. If the specified maximum gas is reached, the test is marked to be ignored. Put a `0` if you do not want to limit it.
- you should define also only one of these parameters:
- `allow_tests` with the list of tests or test sets to execute. All others will be excluded. Test sets should be prefixed with `&`
- `ignore_tests` with the list of test or test sets to ignore. All others will be included. Test sets should be prefixed with `&`

#### Test sets

Expand All @@ -99,12 +98,15 @@ When the command line parameter `--report` is defined, it automatically:
- `<timestamp>-<git_commit>.csv` with the raw results of the execution
- The HTML file also contains the diff with the previous result. The previous result file is the more recent csv file with different commit from the current one

Sometimes do you want to only re-execute tests that are marked as `Ignored` (because you are implementing something new). In this case, you can specify `--cache <>.csv` to use the previous results.

NOTE: if you do not execute with `--report` the tool will exit the process with `1` if there is any test that is not working.


### Manually executing the tests

Usually we have to debug and run the tests manually to check if everything works ok. We provide a set of command line parameters to help with this.

- `testool [--suite xxx] --cache` to execute all tests, and keeping the results (cache) CSV file. If you delete entries from the cache file, and re-run the tool again, only the deleted tests will be executed again
- `testool [--suite xxx] --cache <cache_file>` to execute all tests, and keeping the results (cache) CSV file. If you delete entries from the cache file, and re-run the tool again, only the deleted tests will be executed again

- `testool [--suite xxx] --inspect <test_id>` only executed the selected test (even if cached, or ignored). Use `RUST_BACKTRACE=1` here to check if anything fails. Also gives a dumop of the test as also to the geth steps executed.
- `testool [--suite xxx] --inspect <test_id>` only executed the selected test (even if cached, or ignored). Use `RUST_BACKTRACE=1` here to check if anything fails. Also gives a dump of the test as also to the geth steps executed.
4 changes: 0 additions & 4 deletions testool/src/config.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
use anyhow::{anyhow, bail, ensure, Context, Result};
use eth_types::evm_types::OpcodeId;
use serde::Deserialize;

const CONFIG_FILE: &str = "Config.toml";
Expand All @@ -19,8 +18,6 @@ pub struct TestSuite {
pub max_gas: u64,
pub max_steps: u64,

/// see [Implemented opcodes status](https://github.com/appliedzkp/zkevm-circuits/issues/477)
pub unimplemented_opcodes: Vec<OpcodeId>,
ignore_tests: Option<Vec<String>>,
allow_tests: Option<Vec<String>>,
}
Expand All @@ -32,7 +29,6 @@ impl Default for TestSuite {
path: String::default(),
max_gas: u64::MAX,
max_steps: u64::MAX,
unimplemented_opcodes: Vec::new(),
ignore_tests: Some(Vec::new()),
allow_tests: None,
}
Expand Down
33 changes: 18 additions & 15 deletions testool/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@ mod statetest;
mod utils;

use crate::config::TestSuite;
use crate::statetest::ResultLevel;
use anyhow::{bail, Result};
use clap::Parser;
use compiler::Compiler;
use config::Config;
use log::{error, info};
use log::info;
use statetest::{
geth_trace, load_statetests_suite, run_statetests_suite, run_test, CircuitsConfig, Results,
StateTest,
Expand Down Expand Up @@ -50,7 +51,7 @@ struct Args {

/// Cache execution results
#[clap(long)]
cache: bool,
cache: Option<String>,

/// Generates log and and html file with info.
#[clap(long)]
Expand All @@ -69,8 +70,6 @@ struct Args {
v: bool,
}

const RESULT_CACHE: &str = "result.cache";

fn run_single_test(test: StateTest, circuits_config: CircuitsConfig) -> Result<()> {
println!("{}", &test);
let trace = geth_trace(test.clone())?;
Expand Down Expand Up @@ -118,12 +117,10 @@ fn go() -> Result<()> {
}
return Ok(());
}

if let Some(test_id) = args.inspect {
// Test only one and return
let mut state_tests_filtered: Vec<_> =
state_tests.iter().filter(|t| t.id == test_id).collect();

if state_tests_filtered.is_empty() {
info!(
"Test '{}' not found but found some that partially matches:",
Expand All @@ -139,10 +136,6 @@ fn go() -> Result<()> {
};

if args.report {
if args.cache {
bail!("--cache is not compartible with --report");
}

let git_hash = utils::current_git_commit()?;
let timestamp = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
Expand All @@ -159,8 +152,18 @@ fn go() -> Result<()> {
REPORT_FOLDER, args.suite, timestamp, git_hash
);

let mut results = Results::with_cache(PathBuf::from(csv_filename))?;

// when running a report, the tests result of the containing cache file
// are used, but removing all Ignored tests
let mut results = if let Some(cache_filename) = args.cache {
let mut results = Results::from_file(PathBuf::from(cache_filename))?;
results
.tests
.retain(|_, test| test.level != ResultLevel::Ignored);
results
} else {
Results::default()
};
results.set_cache(PathBuf::from(csv_filename));
run_statetests_suite(state_tests, &circuits_config, &suite, &mut results)?;

// filter non-csv files and files from the same commit
Expand Down Expand Up @@ -190,8 +193,8 @@ fn go() -> Result<()> {
report.print_tty()?;
info!("{}", html_filename);
} else {
let mut results = if args.cache {
Results::with_cache(PathBuf::from(RESULT_CACHE))?
let mut results = if let Some(cache_filename) = args.cache {
Results::with_cache(PathBuf::from(cache_filename))?
} else {
Results::default()
};
Expand All @@ -213,6 +216,6 @@ fn go() -> Result<()> {

fn main() {
if let Err(err) = go() {
error!("{}", err);
eprintln!("Error found {}", err);
}
}
Loading

0 comments on commit 6e3d9dc

Please sign in to comment.