Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add timeouts to fuzz testing #9394

Merged
merged 11 commits into from
Nov 29, 2024
8 changes: 8 additions & 0 deletions crates/config/src/fuzz.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ pub struct FuzzConfig {
pub failure_persist_file: Option<String>,
/// show `console.log` in fuzz test, defaults to `false`
pub show_logs: bool,
/// Optional timeout for each property test
pub timeout_secs: Option<u64>,
/// Optionally allow timeouts to not be reported as failures
pub allow_timeouts: bool,
}

impl Default for FuzzConfig {
Expand All @@ -45,6 +49,8 @@ impl Default for FuzzConfig {
failure_persist_dir: None,
failure_persist_file: None,
show_logs: false,
timeout_secs: None,
allow_timeouts: false,
}
}
}
Expand All @@ -61,6 +67,8 @@ impl FuzzConfig {
failure_persist_dir: Some(cache_dir),
failure_persist_file: Some("failures".to_string()),
show_logs: false,
timeout_secs: None,
allow_timeouts: false,
}
}
}
Expand Down
34 changes: 26 additions & 8 deletions crates/evm/evm/src/executors/fuzz/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,19 @@ impl FuzzedExecutor {
let max_traces_to_collect = std::cmp::max(1, self.config.gas_report_samples) as usize;
let show_logs = self.config.show_logs;

// Start a timer if timeout is set
let start_time = self.config.timeout_secs.map(|timeout| {
(std::time::Instant::now(), std::time::Duration::from_secs(timeout))
});

let run_result = self.runner.clone().run(&strategy, |calldata| {
// Check if the timeout has been reached
if let Some((start_time, timeout)) = start_time {
if start_time.elapsed() > timeout {
DaniPopes marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess since this is opt-in, checking via elapsed for now is fine, and we can track improvements separately, but I'd suggest to wrap this if let Some into a helper type like Timeout(Option<Instant>) and an is_timed_out then this becomes easier to change

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mattsse I added FuzzTestTimer with da87095 please check

return Err(TestCaseError::fail("Timeout reached"))
grandizzy marked this conversation as resolved.
Show resolved Hide resolved
}
}

let fuzz_res = self.single_fuzz(address, should_fail, calldata)?;

// If running with progress then increment current run.
Expand Down Expand Up @@ -193,17 +205,23 @@ impl FuzzedExecutor {
}
Err(TestError::Fail(reason, _)) => {
let reason = reason.to_string();
result.reason = (!reason.is_empty()).then_some(reason);
result.reason = (!reason.is_empty()).then_some(reason.clone());

let args = if let Some(data) = calldata.get(4..) {
func.abi_decode_input(data, false).unwrap_or_default()
if reason == "Timeout reached" {
smartcontracts marked this conversation as resolved.
Show resolved Hide resolved
if self.config.allow_timeouts {
result.success = true;
}
} else {
vec![]
};
let args = if let Some(data) = calldata.get(4..) {
func.abi_decode_input(data, false).unwrap_or_default()
} else {
vec![]
};
smartcontracts marked this conversation as resolved.
Show resolved Hide resolved

result.counterexample = Some(CounterExample::Single(
BaseCounterExample::from_fuzz_call(calldata, args, call.traces),
));
result.counterexample = Some(CounterExample::Single(
BaseCounterExample::from_fuzz_call(calldata, args, call.traces),
));
}
}
}

Expand Down
14 changes: 14 additions & 0 deletions crates/forge/bin/cmd/test/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,14 @@ pub struct TestArgs {
#[arg(long, env = "FOUNDRY_FUZZ_RUNS", value_name = "RUNS")]
pub fuzz_runs: Option<u64>,

/// Timeout for each fuzz run in seconds.
#[arg(long, env = "FOUNDRY_FUZZ_TIMEOUT_SECS", value_name = "TIMEOUT_SECS")]
pub fuzz_timeout_secs: Option<u64>,

/// Allow timeouts to not be reported as failures.
#[arg(long, env = "FOUNDRY_FUZZ_ALLOW_TIMEOUTS", value_name = "ALLOW_TIMEOUTS")]
pub fuzz_allow_timeouts: bool,

/// File to rerun fuzz failures from.
#[arg(long)]
pub fuzz_input_file: Option<String>,
Expand Down Expand Up @@ -870,6 +878,12 @@ impl Provider for TestArgs {
if let Some(fuzz_runs) = self.fuzz_runs {
fuzz_dict.insert("runs".to_string(), fuzz_runs.into());
}
if let Some(fuzz_timeout_secs) = self.fuzz_timeout_secs {
fuzz_dict.insert("timeout_secs".to_string(), fuzz_timeout_secs.into());
}
if self.fuzz_allow_timeouts {
fuzz_dict.insert("allow_timeouts".to_string(), true.into());
}
if let Some(fuzz_input_file) = self.fuzz_input_file.clone() {
fuzz_dict.insert("failure_persist_file".to_string(), fuzz_input_file.into());
}
Expand Down
Loading