Skip to content

Commit

Permalink
fix tests on newer Rust nightlies
Browse files Browse the repository at this point in the history 
Newer versions of Rust seem to return SIGABRT, not SIGSEGV. This is all right.
  • Loading branch information
@sunshowers
sunshowers committed Feb 4, 2025
1 parent 73eea05 commit ae2472f
Show file tree
Hide file tree
Showing 4 changed files with 133 additions and 63 deletions.
4 changes: 3 additions & 1 deletion integration-tests/tests/integration/fixtures.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,9 @@ impl CheckResult {
CheckResult::Leak => Regex::new(&format!(r"LEAK \[.*\] *{name}")).unwrap(),
CheckResult::Fail => Regex::new(&format!(r"FAIL \[.*\] *{name}")).unwrap(),
CheckResult::FailLeak => Regex::new(&format!(r"FAIL \+ LEAK \[.*\] *{name}")).unwrap(),
CheckResult::Abort => Regex::new(&format!(r"(ABORT|SIGSEGV) \[.*\] *{name}")).unwrap(),
CheckResult::Abort => {
Regex::new(&format!(r"(ABORT|SIGSEGV|SIGABRT) \[.*\] *{name}")).unwrap()
}
}
}

Expand Down
58 changes: 34 additions & 24 deletions nextest-runner/tests/integration/basic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

use crate::fixtures::*;
use cfg_if::cfg_if;
use color_eyre::eyre::ensure;
use color_eyre::eyre::Result;
use fixture_data::{
models::{TestCaseFixtureStatus, TestSuiteFixture},
Expand Down Expand Up @@ -148,21 +149,23 @@ fn test_run() -> Result<()> {
fixture.name
)
});
let valid = match &instance_value.status {
InstanceStatus::Skipped(_) => fixture.status.is_ignored(),
InstanceStatus::Finished(run_statuses) => {
// This test should not have been retried since retries aren't configured.
assert_eq!(
run_statuses.len(),
1,
"test {} should have been run exactly once",
fixture.name
);
let run_status = run_statuses.last_status();

if run_status.result != make_execution_result(fixture.status, 1) {
false
} else {
let valid = || {
match &instance_value.status {
InstanceStatus::Skipped(_) => {
ensure!(fixture.status.is_ignored(), "test should be skipped");
Ok(())
}
InstanceStatus::Finished(run_statuses) => {
// This test should not have been retried since retries aren't configured.
assert_eq!(
run_statuses.len(),
1,
"test {} should have been run exactly once",
fixture.name
);
let run_status = run_statuses.last_status();

ensure_execution_result(&run_status.result, fixture.status, 1)?;
// Extracting descriptions works for segfaults on Unix but not on Windows.
#[cfg_attr(not(unix), expect(unused_mut))]
let mut can_extract_description = fixture.status
Expand Down Expand Up @@ -197,14 +200,15 @@ fn test_run() -> Result<()> {
stderr.as_str_lossy(),
);
}
true

Ok(())
}
}
};
if !valid {
if let Err(error) = valid() {
panic!(
"for test {}, mismatch in status: expected {:?}, actual {:?}",
fixture.name, fixture.status, instance_value.status
"for test {}, mismatch in status: expected {:?}, actual {:?}, error: {}",
fixture.name, fixture.status, instance_value.status, error,
);
}
}
Expand Down Expand Up @@ -275,7 +279,13 @@ fn test_run_ignored() -> Result<()> {
let instance_value =
&instance_statuses[&(test_binary.binary_path.as_path(), fixture.name)];
let valid = match &instance_value.status {
InstanceStatus::Skipped(_) => !fixture.status.is_ignored(),
InstanceStatus::Skipped(_) => {
ensure!(
!fixture.status.is_ignored(),
"non-ignored tests should be skipped"
);
Ok(())
}
InstanceStatus::Finished(run_statuses) => {
// This test should not have been retried since retries aren't configured.
assert_eq!(
Expand All @@ -285,13 +295,13 @@ fn test_run_ignored() -> Result<()> {
fixture.name
);
let run_status = run_statuses.last_status();
run_status.result == make_execution_result(fixture.status, 1)
ensure_execution_result(&run_status.result, fixture.status, 1)
}
};
if !valid {
if let Err(error) = valid {
panic!(
"for test {}, mismatch in status: expected {:?}, actual {:?}",
fixture.name, fixture.status, instance_value.status
"for test {}, mismatch in status: expected {:?}, actual {:?}, error: {}",
fixture.name, fixture.status, instance_value.status, error,
);
}
}
Expand Down
100 changes: 75 additions & 25 deletions nextest-runner/tests/integration/fixtures.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// SPDX-License-Identifier: MIT OR Apache-2.0

use camino::{Utf8Path, Utf8PathBuf};
use color_eyre::eyre::{Context, Result};
use color_eyre::eyre::{bail, ensure, Context, Result};
use duct::cmd;
use fixture_data::models::TestCaseFixtureStatus;
use guppy::{graph::PackageGraph, MetadataCommand};
Expand Down Expand Up @@ -32,51 +32,101 @@ use std::{
sync::Arc,
};

pub(crate) fn make_execution_result(
pub(crate) fn ensure_execution_result(
actual: &ExecutionResult,
status: TestCaseFixtureStatus,
total_attempts: usize,
) -> ExecutionResult {
) -> Result<()> {
match status {
TestCaseFixtureStatus::Pass | TestCaseFixtureStatus::IgnoredPass => ExecutionResult::Pass,
TestCaseFixtureStatus::Pass | TestCaseFixtureStatus::IgnoredPass => {
ensure!(
actual == &ExecutionResult::Pass,
"pass: actual result ({actual:?}) matches expected"
);
}
TestCaseFixtureStatus::Flaky { pass_attempt } => {
if pass_attempt <= total_attempts {
ExecutionResult::Pass
ensure!(
actual == &ExecutionResult::Pass,
"flaky (passing attempt): actual result ({actual:?}) matches expected"
);
} else {
ExecutionResult::Fail {
abort_status: None,
leaked: false,
}
ensure!(
actual
== &ExecutionResult::Fail {
abort_status: None,
leaked: false
},
"flaky (failing attempt): actual result ({actual:?}) matches expected"
);
}
}
TestCaseFixtureStatus::Segfault => {
cfg_if::cfg_if! {
if #[cfg(unix)] {
// SIGSEGV is 11.
let abort_status = Some(AbortStatus::UnixSignal(11));
// SIGSEGV is 11. Newer versions of Rust may use SIGABRT
// instead, which is 6. Check for either.
let (abort_status, leaked) = match actual {
ExecutionResult::Fail {
abort_status,
leaked,
} => (abort_status, *leaked),
_ => bail!("expected ExecutionResult::Fail, found {actual:?}"),
};

ensure!(
*abort_status == Some(AbortStatus::UnixSignal(11))
|| *abort_status == Some(AbortStatus::UnixSignal(6)),
"segfault: expected SIGSEGV or SIGABRT, found {abort_status:?}"
);
ensure!(!leaked, "segfault: expected no leaks, found leaked");
} else if #[cfg(windows)] {
// A segfault is an access violation on Windows.
let abort_status = Some(AbortStatus::WindowsNtStatus(
windows_sys::Win32::Foundation::STATUS_ACCESS_VIOLATION,
));
ensure!(
actual == &ExecutionResult::Fail {
abort_status,
leaked: false,
},
"segfault: actual result ({actual:?}) matches expected"
);
} else {
let abort_status = None;
// Unsupported platform.
compile_error!("unsupported platform");
}
}
ExecutionResult::Fail {
abort_status,
leaked: false,
}
}
TestCaseFixtureStatus::Fail | TestCaseFixtureStatus::IgnoredFail => ExecutionResult::Fail {
abort_status: None,
leaked: false,
},
TestCaseFixtureStatus::FailLeak => ExecutionResult::Fail {
abort_status: None,
leaked: true,
},
TestCaseFixtureStatus::Leak => ExecutionResult::Leak,
TestCaseFixtureStatus::Fail | TestCaseFixtureStatus::IgnoredFail => {
ensure!(
actual
== &ExecutionResult::Fail {
abort_status: None,
leaked: false
},
"fail: actual result ({actual:?}) matches expected"
);
}
TestCaseFixtureStatus::FailLeak => {
ensure!(
actual
== &ExecutionResult::Fail {
abort_status: None,
leaked: true
},
"fail + leak: actual result ({actual:?}) matches expected"
);
}
TestCaseFixtureStatus::Leak => {
ensure!(
actual == &ExecutionResult::Leak,
"leak: actual result ({actual:?}) matches expected"
);
}
}

Ok(())
}

#[track_caller]
Expand Down
34 changes: 21 additions & 13 deletions nextest-runner/tests/integration/target_runner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

use crate::fixtures::*;
use camino::Utf8Path;
use color_eyre::Result;
use color_eyre::{eyre::ensure, Result};
use fixture_data::nextest_tests::EXPECTED_TEST_SUITES;
use nextest_runner::{
cargo_config::{CargoConfigs, TargetTriple},
Expand Down Expand Up @@ -263,7 +263,10 @@ fn test_run_with_target_runner() -> Result<()> {
)
});
let valid = match &instance_value.status {
InstanceStatus::Skipped(_) => fixture.status.is_ignored(),
InstanceStatus::Skipped(_) => {
ensure!(fixture.status.is_ignored(), "test should be skipped");
Ok(())
}
InstanceStatus::Finished(run_statuses) => {
// This test should not have been retried since retries aren't configured.
assert_eq!(
Expand All @@ -274,26 +277,31 @@ fn test_run_with_target_runner() -> Result<()> {
);
let run_status = run_statuses.last_status();

#[cfg_attr(not(unix), expect(unused_mut))]
let mut expected_status = make_execution_result(fixture.status, 1);
// On Unix, segfaults aren't passed through by the passthrough runner.
cfg_if::cfg_if! {
if #[cfg(unix)] {
// On Unix, segfaults aren't passed through by the
// passthrough runner.
if fixture.status == fixture_data::models::TestCaseFixtureStatus::Segfault {
expected_status = nextest_runner::reporter::events::ExecutionResult::Fail {
abort_status: None,
leaked: false,
};
ensure_execution_result(
&run_status.result,
fixture_data::models::TestCaseFixtureStatus::Fail,
1,
)
} else {
ensure_execution_result(&run_status.result, fixture.status, 1)
}
} else if #[cfg(windows)] {
ensure_execution_result(&run_status.result, fixture.status, 1)
} else {
compile_error!("unsupported platform")
}
}
run_status.result == expected_status
}
};
if !valid {
if let Err(error) = valid {
panic!(
"for test {}, mismatch in status: expected {:?}, actual {:?}",
fixture.name, fixture.status, instance_value.status
"for test {}, mismatch in status: expected {:?}, actual {:?}, error: {}",
fixture.name, fixture.status, instance_value.status, error
);
}
}
Expand Down

0 comments on commit ae2472f

Please sign in to comment.