diff --git a/proptest-state-machine/CHANGELOG.md b/proptest-state-machine/CHANGELOG.md index 02756482..949c323f 100644 --- a/proptest-state-machine/CHANGELOG.md +++ b/proptest-state-machine/CHANGELOG.md @@ -9,3 +9,4 @@ - Removed the limit of number of transitions that can be deleted in shrinking that depended on the number the of transitions given to `prop_state_machine!` or `ReferenceStateMachine::sequential_strategy`. - Fixed state-machine macro's inability to handle missing config - Fixed logging of state machine transitions to be enabled when verbose config is >= 1. The "std" feature is added to proptest-state-machine as a default feature that allows to switch the logging off in non-std env. +- Fixed an issue where after simplification of the initial state causes the test to succeed, the initial state would not be re-complicated - causing the test to report a succeeding input as the simplest failing input. diff --git a/proptest-state-machine/proptest-regressions/strategy.txt b/proptest-state-machine/proptest-regressions/strategy.txt new file mode 100644 index 00000000..d65b92e5 --- /dev/null +++ b/proptest-state-machine/proptest-regressions/strategy.txt @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 2e5fb0b2e70b08bd8a567bf4d9ae66ee2478d832d6bdfb9670f5b98203e78178 # shrinks to seed = [120, 233, 214, 148, 193, 171, 178, 64, 157, 62, 78, 165, 215, 79, 177, 175, 171, 202, 51, 93, 79, 238, 39, 104, 174, 79, 152, 255, 45, 174, 27, 168] +cc 39927f23e5f67ac32c5219c226c49d87e3e3c995a73cd969d72fbcdf52ac895b # shrinks to seed = [0, 10, 244, 19, 249, 125, 161, 150, 61, 56, 77, 245, 12, 228, 187, 180, 148, 61, 17, 32, 189, 118, 70, 47, 147, 210, 94, 127, 210, 23, 128, 75] diff --git a/proptest-state-machine/src/strategy.rs b/proptest-state-machine/src/strategy.rs index fca723a7..926dd93c 100644 --- a/proptest-state-machine/src/strategy.rs +++ b/proptest-state-machine/src/strategy.rs @@ -357,6 +357,7 @@ impl< // Store the valid initial state self.last_valid_initial_state = self.initial_state.current(); + self.last_shrink = Some(self.shrink); return true; } else { // If the shrink is not acceptable, clear it out @@ -533,7 +534,6 @@ impl< false } Some(InitialState) => { - self.last_shrink = None; if self.initial_state.complicate() && self.check_acceptable(None) { @@ -752,4 +752,114 @@ mod test { } } } + + /// A tests that verifies that the strategy finds a simplest failing case, and + /// that this simplest failing case is ultimately reported by the test runner, + /// as opposed to reporting input that actually passes the test. + /// + /// This module defines a state machine test that is designed to fail. + /// The reference state machine consists of a lower bound the acceptable value + /// of a transition. And the test fails if an unacceptably low transition + /// value is observed, given the reference state's limit. + /// + /// This intentionally-failing state machine test is then run inside a proptest + /// to verify that it reports a simplest failing input when it fails. + mod find_simplest_failure { + use proptest::prelude::*; + use proptest::strategy::BoxedStrategy; + use proptest::test_runner::TestRng; + use proptest::{ + collection, + strategy::Strategy, + test_runner::{Config, TestError, TestRunner}, + }; + + use crate::{ReferenceStateMachine, StateMachineTest}; + + const MIN_TRANSITION: u32 = 10; + const MAX_TRANSITION: u32 = 20; + + const MIN_LIMIT: u32 = 2; + const MAX_LIMIT: u32 = 50; + + #[derive(Debug, Default, Clone)] + struct FailIfLessThan(u32); + impl ReferenceStateMachine for FailIfLessThan { + type State = Self; + type Transition = u32; + + fn init_state() -> BoxedStrategy { + (MIN_LIMIT..MAX_LIMIT).prop_map(FailIfLessThan).boxed() + } + + fn transitions(_: &Self::State) -> BoxedStrategy { + (MIN_TRANSITION..MAX_TRANSITION).boxed() + } + + fn apply(state: Self::State, _: &Self::Transition) -> Self::State { + state + } + } + + /// Defines a test that is intended to fail, so that we can inspect the + /// failing input. + struct FailIfLessThanTest; + impl StateMachineTest for FailIfLessThanTest { + type SystemUnderTest = (); + type Reference = FailIfLessThan; + + fn init_test(ref_state: &FailIfLessThan) { + println!(); + println!("starting {ref_state:?}"); + } + + fn apply( + (): Self::SystemUnderTest, + ref_state: &FailIfLessThan, + transition: u32, + ) -> Self::SystemUnderTest { + // Fail on any transition that is less than the ref state's limit. + let FailIfLessThan(limit) = ref_state; + println!("{transition} < {}?", limit); + if transition < ref_state.0 { + panic!("{transition} < {}", limit); + } + } + } + + proptest! { + #[test] + fn test_returns_simplest_failure( + seed in collection::vec(any::(), 32).no_shrink()) { + + // We need to explicitly run create a runner so that we can + // inspect the output, and determine if it does return an input that + // should fail, and is minimal. + let mut runner = TestRunner::new_with_rng( + Config::default(), TestRng::from_seed(Default::default(), &seed)); + let result = runner.run( + &FailIfLessThan::sequential_strategy(10..50_usize), + |(ref_state, transitions)| { + Ok(FailIfLessThanTest::test_sequential( + Default::default(), + ref_state, + transitions, + )) + }, + ); + if let Err(TestError::Fail( + _, + (FailIfLessThan(limit), transitions), + )) = result + { + assert_eq!(transitions.len(), 1, "The minimal failing case should be "); + assert_eq!(limit, MIN_TRANSITION + 1); + assert!(transitions[0] < limit); + } else { + prop_assume!(false, + "If the state machine doesn't fail as intended, we need a case that fails."); + } + } + } + } }