feat(invariant): add basic metrics report (#9158)

* feat(invariant): add basic metrics report

* Put metrics behind show-metrics invariant config

crates/config/src/invariant.rs

@@ -34,6 +34,8 @@ pub struct InvariantConfig {
     pub gas_report_samples: u32,
     /// Path where invariant failures are recorded and replayed.
     pub failure_persist_dir: Option<PathBuf>,
+    /// Whether to collect and display fuzzed selectors metrics.
+    pub show_metrics: bool,
 }
 
 impl Default for InvariantConfig {
@@ -48,6 +50,7 @@ impl Default for InvariantConfig {
             max_assume_rejects: 65536,
             gas_report_samples: 256,
             failure_persist_dir: None,
+            show_metrics: false,
         }
     }
 }
@@ -65,6 +68,7 @@ impl InvariantConfig {
             max_assume_rejects: 65536,
             gas_report_samples: 256,
             failure_persist_dir: Some(cache_dir),
+            show_metrics: false,
         }
     }
 
@@ -103,6 +107,7 @@ impl InlineConfigParser for InvariantConfig {
                     conf_clone.failure_persist_dir = Some(PathBuf::from(value))
                 }
                 "shrink-run-limit" => conf_clone.shrink_run_limit = parse_config_u32(key, value)?,
+                "show-metrics" => conf_clone.show_metrics = parse_config_bool(key, value)?,
                 _ => Err(InlineConfigParserError::InvalidConfigProperty(key.to_string()))?,
             }
         }

crates/evm/evm/Cargo.toml

@@ -50,3 +50,4 @@ proptest.workspace = true
 thiserror.workspace = true
 tracing.workspace = true
 indicatif = "0.17"
+serde.workspace = true

crates/evm/evm/src/executors/invariant/mod.rs

@@ -31,7 +31,11 @@ use proptest::{
 use result::{assert_after_invariant, assert_invariants, can_continue};
 use revm::primitives::HashMap;
 use shrink::shrink_sequence;
-use std::{cell::RefCell, collections::btree_map::Entry, sync::Arc};
+use std::{
+    cell::RefCell,
+    collections::{btree_map::Entry, HashMap as Map},
+    sync::Arc,
+};
 
 mod error;
 pub use error::{InvariantFailures, InvariantFuzzError};
@@ -42,6 +46,7 @@ pub use replay::{replay_error, replay_run};
 
 mod result;
 pub use result::InvariantFuzzTestResult;
+use serde::{Deserialize, Serialize};
 
 mod shrink;
 use crate::executors::EvmError;
@@ -101,6 +106,17 @@ sol! {
     }
 }
 
+/// Contains invariant metrics for a single fuzzed selector.
+#[derive(Default, Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
+pub struct InvariantMetrics {
+    // Count of fuzzed selector calls.
+    pub calls: usize,
+    // Count of fuzzed selector reverts.
+    pub reverts: usize,
+    // Count of fuzzed selector discards (through assume cheatcodes).
+    pub discards: usize,
+}
+
 /// Contains data collected during invariant test runs.
 pub struct InvariantTestData {
     // Consumed gas and calldata of every successful fuzz call.
@@ -115,6 +131,8 @@ pub struct InvariantTestData {
     pub last_call_results: Option<RawCallResult>,
     // Coverage information collected from all fuzzed calls.
     pub coverage: Option<HitMaps>,
+    // Metrics for each fuzzed selector.
+    pub metrics: Map<String, InvariantMetrics>,
 
     // Proptest runner to query for random values.
     // The strategy only comes with the first `input`. We fill the rest of the `inputs`
@@ -153,6 +171,7 @@ impl InvariantTest {
             gas_report_traces: vec![],
             last_call_results,
             coverage: None,
+            metrics: Map::default(),
             branch_runner,
         });
         Self { fuzz_state, targeted_contracts, execution_data }
@@ -191,6 +210,24 @@ impl InvariantTest {
         }
     }
 
+    /// Update metrics for a fuzzed selector, extracted from tx details.
+    /// Always increments number of calls; discarded runs (through assume cheatcodes) are tracked
+    /// separated from reverts.
+    pub fn record_metrics(&self, tx_details: &BasicTxDetails, reverted: bool, discarded: bool) {
+        if let Some(metric_key) =
+            self.targeted_contracts.targets.lock().fuzzed_metric_key(tx_details)
+        {
+            let test_metrics = &mut self.execution_data.borrow_mut().metrics;
+            let invariant_metrics = test_metrics.entry(metric_key).or_default();
+            invariant_metrics.calls += 1;
+            if discarded {
+                invariant_metrics.discards += 1;
+            } else if reverted {
+                invariant_metrics.reverts += 1;
+            }
+        }
+    }
+
     /// End invariant test run by collecting results, cleaning collected artifacts and reverting
     /// created fuzz state.
     pub fn end_run(&self, run: InvariantTestRun, gas_samples: usize) {
@@ -331,10 +368,15 @@ impl<'a> InvariantExecutor<'a> {
                         TestCaseError::fail(format!("Could not make raw evm call: {e}"))
                     })?;
 
+                let discarded = call_result.result.as_ref() == MAGIC_ASSUME;
+                if self.config.show_metrics {
+                    invariant_test.record_metrics(tx, call_result.reverted, discarded);
+                }
+
                 // Collect coverage from last fuzzed call.
                 invariant_test.merge_coverage(call_result.coverage.clone());
 
-                if call_result.result.as_ref() == MAGIC_ASSUME {
+                if discarded {
                     current_run.inputs.pop();
                     current_run.assume_rejects_counter += 1;
                     if current_run.assume_rejects_counter > self.config.max_assume_rejects {
@@ -443,6 +485,7 @@ impl<'a> InvariantExecutor<'a> {
             last_run_inputs: result.last_run_inputs,
             gas_report_traces: result.gas_report_traces,
             coverage: result.coverage,
+            metrics: result.metrics,
         })
     }
 

crates/evm/evm/src/executors/invariant/result.rs

@@ -1,6 +1,6 @@
 use super::{
     call_after_invariant_function, call_invariant_function, error::FailedInvariantCaseData,
-    InvariantFailures, InvariantFuzzError, InvariantTest, InvariantTestRun,
+    InvariantFailures, InvariantFuzzError, InvariantMetrics, InvariantTest, InvariantTestRun,
 };
 use crate::executors::{Executor, RawCallResult};
 use alloy_dyn_abi::JsonAbiExt;
@@ -13,7 +13,7 @@ use foundry_evm_fuzz::{
     FuzzedCases,
 };
 use revm_inspectors::tracing::CallTraceArena;
-use std::borrow::Cow;
+use std::{borrow::Cow, collections::HashMap};
 
 /// The outcome of an invariant fuzz test
 #[derive(Debug)]
@@ -30,6 +30,8 @@ pub struct InvariantFuzzTestResult {
     pub gas_report_traces: Vec<Vec<CallTraceArena>>,
     /// The coverage info collected during the invariant test runs.
     pub coverage: Option<HitMaps>,
+    /// Fuzzed selectors metrics collected during the invariant test runs.
+    pub metrics: HashMap<String, InvariantMetrics>,
 }
 
 /// Enriched results of an invariant run check.

crates/evm/fuzz/src/invariant/mod.rs

@@ -125,6 +125,18 @@ impl TargetedContracts {
             .filter(|(_, c)| !c.abi.functions.is_empty())
             .flat_map(|(contract, c)| c.abi_fuzzed_functions().map(move |f| (contract, f)))
     }
+
+    /// Identifies fuzzed contract and function based on given tx details and returns unique metric
+    /// key composed from contract identifier and function name.
+    pub fn fuzzed_metric_key(&self, tx: &BasicTxDetails) -> Option<String> {
+        self.inner.get(&tx.call_details.target).and_then(|contract| {
+            contract
+                .abi
+                .functions()
+                .find(|f| f.selector() == tx.call_details.calldata[..4])
+                .map(|function| format!("{}.{}", contract.identifier.clone(), function.name))
+        })
+    }
 }
 
 impl std::ops::Deref for TargetedContracts {

crates/forge/bin/cmd/snapshot.rs

@@ -242,6 +242,7 @@ impl FromStr for GasSnapshotEntry {
                                         runs: runs.as_str().parse().unwrap(),
                                         calls: calls.as_str().parse().unwrap(),
                                         reverts: reverts.as_str().parse().unwrap(),
+                                        metrics: HashMap::default(),
                                     },
                                 })
                         }
@@ -486,7 +487,12 @@ mod tests {
             GasSnapshotEntry {
                 contract_name: "Test".to_string(),
                 signature: "deposit()".to_string(),
-                gas_used: TestKindReport::Invariant { runs: 256, calls: 100, reverts: 200 }
+                gas_used: TestKindReport::Invariant {
+                    runs: 256,
+                    calls: 100,
+                    reverts: 200,
+                    metrics: HashMap::default()
+                }
             }
         );
     }
@@ -500,7 +506,12 @@ mod tests {
             GasSnapshotEntry {
                 contract_name: "ERC20Invariants".to_string(),
                 signature: "invariantBalanceSum()".to_string(),
-                gas_used: TestKindReport::Invariant { runs: 256, calls: 3840, reverts: 2388 }
+                gas_used: TestKindReport::Invariant {
+                    runs: 256,
+                    calls: 3840,
+                    reverts: 2388,
+                    metrics: HashMap::default()
+                }
             }
         );
     }

crates/forge/bin/cmd/test/mod.rs

@@ -54,7 +54,9 @@ mod summary;
 use quick_junit::{NonSuccessKind, Report, TestCase, TestCaseStatus, TestSuite};
 use summary::TestSummaryReporter;
 
+use crate::cmd::test::summary::print_invariant_metrics;
 pub use filter::FilterArgs;
+use forge::result::TestKind;
 
 // Loads project's figment and merges the build cli arguments into it
 foundry_config::merge_impl_figment_convert!(TestArgs, opts, evm_opts);
@@ -621,6 +623,13 @@ impl TestArgs {
                 if !silent {
                     sh_println!("{}", result.short_result(name))?;
 
+                    // Display invariant metrics if invariant kind.
+                    if let TestKind::Invariant { runs: _, calls: _, reverts: _, metrics } =
+                        &result.kind
+                    {
+                        print_invariant_metrics(metrics);
+                    }
+
                     // We only display logs at level 2 and above
                     if verbosity >= 2 {
                         // We only decode logs from Hardhat and DS-style console events

crates/forge/bin/cmd/test/summary.rs

@@ -1,7 +1,11 @@
 use crate::cmd::test::TestOutcome;
 use comfy_table::{
-    modifiers::UTF8_ROUND_CORNERS, Attribute, Cell, CellAlignment, Color, Row, Table,
+    modifiers::UTF8_ROUND_CORNERS, presets::ASCII_MARKDOWN, Attribute, Cell, CellAlignment, Color,
+    Row, Table,
 };
+use foundry_evm::executors::invariant::InvariantMetrics;
+use itertools::Itertools;
+use std::collections::HashMap;
 
 /// A simple summary reporter that prints the test results in a table.
 pub struct TestSummaryReporter {
@@ -91,3 +95,37 @@ impl TestSummaryReporter {
         println!("\n{}", self.table);
     }
 }
+
+/// Helper to create and render invariant metrics summary table:
+/// | Contract              | Selector       | Calls | Reverts | Discards |
+/// |-----------------------|----------------|-------|---------|----------|
+/// | AnotherCounterHandler | doWork         |  7451 |   123   |   4941   |
+/// | AnotherCounterHandler | doWorkThing    |  7279 |   137   |   4849   |
+/// | CounterHandler        | doAnotherThing |  7302 |   150   |   4794   |
+/// | CounterHandler        | doSomething    |  7382 |   160   |   4830   |
+pub(crate) fn print_invariant_metrics(test_metrics: &HashMap<String, InvariantMetrics>) {
+    if !test_metrics.is_empty() {
+        let mut table = Table::new();
+        table.load_preset(ASCII_MARKDOWN);
+        table.set_header(["Contract", "Selector", "Calls", "Reverts", "Discards"]);
+
+        for name in test_metrics.keys().sorted() {
+            if let Some((contract, selector)) =
+                name.split_once(':').and_then(|(_, contract)| contract.split_once('.'))
+            {
+                let mut row = Row::new();
+                row.add_cell(Cell::new(contract).set_alignment(CellAlignment::Left));
+                row.add_cell(Cell::new(selector).set_alignment(CellAlignment::Left));
+                if let Some(metrics) = test_metrics.get(name) {
+                    row.add_cell(Cell::new(metrics.calls).set_alignment(CellAlignment::Center));
+                    row.add_cell(Cell::new(metrics.reverts).set_alignment(CellAlignment::Center));
+                    row.add_cell(Cell::new(metrics.discards).set_alignment(CellAlignment::Center));
+                }
+
+                table.add_row(row);
+            }
+        }
+
+        println!("{table}\n");
+    }
+}