Skip to content

Commit

Permalink
Rollup merge of rust-lang#42219 - pwoolcoc:add-allow-fail-to-libtest,…
Browse files Browse the repository at this point in the history 
… r=GuillaumeGomez

add `allow_fail` test attribute

This change allows the user to add an `#[allow_fail]` attribute to
tests that will cause the test to compile & run, but if the test fails
it will not cause the entire test run to fail. The test output will
show the failure, but in yellow instead of red, and also indicate that
it was an allowed failure.

Here is an example of the output: http://imgur.com/a/wt7ga
  • Loading branch information
Ariel Ben-Yehuda committed Jun 29, 2017
2 parents dc2003b + 4154f89 commit bea4e60
Show file tree
Hide file tree
Showing 8 changed files with 140 additions and 30 deletions.
45 changes: 27 additions & 18 deletions src/librustdoc/html/markdown.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -769,7 +769,7 @@ pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position
block_info.should_panic, block_info.no_run,
block_info.ignore, block_info.test_harness,
block_info.compile_fail, block_info.error_codes,
line, filename);
line, filename, block_info.allow_fail);
} else {
tests.add_old_test(text, filename);
}
Expand Down Expand Up @@ -859,7 +859,7 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Sp
block_info.should_panic, block_info.no_run,
block_info.ignore, block_info.test_harness,
block_info.compile_fail, block_info.error_codes,
line, filename);
line, filename, block_info.allow_fail);
prev_offset = offset;
}
Event::Start(Tag::Header(level)) => {
Expand Down Expand Up @@ -889,6 +889,7 @@ struct LangString {
test_harness: bool,
compile_fail: bool,
error_codes: Vec<String>,
allow_fail: bool,
}

impl LangString {
Expand All @@ -902,6 +903,7 @@ impl LangString {
test_harness: false,
compile_fail: false,
error_codes: Vec::new(),
allow_fail: false,
}
}

Expand Down Expand Up @@ -930,6 +932,7 @@ impl LangString {
}
"no_run" => { data.no_run = true; seen_rust_tags = !seen_other_tags; }
"ignore" => { data.ignore = true; seen_rust_tags = !seen_other_tags; }
"allow_fail" => { data.allow_fail = true; seen_rust_tags = !seen_other_tags; }
"rust" => { data.rust = true; seen_rust_tags = true; }
"test_harness" => {
data.test_harness = true;
Expand Down Expand Up @@ -1118,7 +1121,7 @@ mod tests {
fn test_lang_string_parse() {
fn t(s: &str,
should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool,
compile_fail: bool, error_codes: Vec<String>) {
compile_fail: bool, allow_fail: bool, error_codes: Vec<String>) {
assert_eq!(LangString::parse(s), LangString {
should_panic: should_panic,
no_run: no_run,
Expand All @@ -1128,25 +1131,31 @@ mod tests {
compile_fail: compile_fail,
error_codes: error_codes,
original: s.to_owned(),
allow_fail: allow_fail,
})
}

fn v() -> Vec<String> {
Vec::new()
}

// marker | should_panic| no_run| ignore| rust | test_harness| compile_fail
// | error_codes
t("", false, false, false, true, false, false, Vec::new());
t("rust", false, false, false, true, false, false, Vec::new());
t("sh", false, false, false, false, false, false, Vec::new());
t("ignore", false, false, true, true, false, false, Vec::new());
t("should_panic", true, false, false, true, false, false, Vec::new());
t("no_run", false, true, false, true, false, false, Vec::new());
t("test_harness", false, false, false, true, true, false, Vec::new());
t("compile_fail", false, true, false, true, false, true, Vec::new());
t("{.no_run .example}", false, true, false, true, false, false, Vec::new());
t("{.sh .should_panic}", true, false, false, false, false, false, Vec::new());
t("{.example .rust}", false, false, false, true, false, false, Vec::new());
t("{.test_harness .rust}", false, false, false, true, true, false, Vec::new());
t("text, no_run", false, true, false, false, false, false, Vec::new());
t("text,no_run", false, true, false, false, false, false, Vec::new());
// | allow_fail | error_codes
t("", false, false, false, true, false, false, false, v());
t("rust", false, false, false, true, false, false, false, v());
t("sh", false, false, false, false, false, false, false, v());
t("ignore", false, false, true, true, false, false, false, v());
t("should_panic", true, false, false, true, false, false, false, v());
t("no_run", false, true, false, true, false, false, false, v());
t("test_harness", false, false, false, true, true, false, false, v());
t("compile_fail", false, true, false, true, false, true, false, v());
t("allow_fail", false, false, false, true, false, false, true, v());
t("{.no_run .example}", false, true, false, true, false, false, false, v());
t("{.sh .should_panic}", true, false, false, false, false, false, false, v());
t("{.example .rust}", false, false, false, true, false, false, false, v());
t("{.test_harness .rust}", false, false, false, true, true, false, false, v());
t("text, no_run", false, true, false, false, false, false, false, v());
t("text,no_run", false, true, false, false, false, false, false, v());
}

#[test]
Expand Down
3 changes: 2 additions & 1 deletion src/librustdoc/test.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,7 @@ impl Collector {
pub fn add_test(&mut self, test: String,
should_panic: bool, no_run: bool, should_ignore: bool,
as_test_harness: bool, compile_fail: bool, error_codes: Vec<String>,
line: usize, filename: String) {
line: usize, filename: String, allow_fail: bool) {
let name = self.generate_name(line, &filename);
// to be removed when hoedown is removed
if self.render_type == RenderType::Pulldown {
Expand Down Expand Up @@ -499,6 +499,7 @@ impl Collector {
ignore: should_ignore,
// compiler failures are test failures
should_panic: testing::ShouldPanic::No,
allow_fail: allow_fail,
},
testfn: testing::DynTestFn(box move |()| {
let panic = io::set_panic(None);
Expand Down
8 changes: 8 additions & 0 deletions src/libsyntax/feature_gate.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,9 @@ declare_features! (

// rustc internal
(active, abi_thiscall, "1.19.0", None),

// Allows a test to fail without failing the whole suite
(active, allow_fail, "1.19.0", Some(42219)),
);

declare_features! (
Expand Down Expand Up @@ -812,6 +815,11 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG
"used internally by rustc",
cfg_fn!(rustc_attrs))),

("allow_fail", Normal, Gated(Stability::Unstable,
"allow_fail",
"allow_fail attribute is currently unstable",
cfg_fn!(allow_fail))),

// Crate level attributes
("crate_name", CrateLevel, Ungated),
("crate_type", CrateLevel, Ungated),
Expand Down
14 changes: 11 additions & 3 deletions src/libsyntax/test.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ struct Test {
path: Vec<Ident> ,
bench: bool,
ignore: bool,
should_panic: ShouldPanic
should_panic: ShouldPanic,
allow_fail: bool,
}

struct TestCtxt<'a> {
Expand Down Expand Up @@ -133,7 +134,8 @@ impl<'a> fold::Folder for TestHarnessGenerator<'a> {
path: self.cx.path.clone(),
bench: is_bench_fn(&self.cx, &i),
ignore: is_ignored(&i),
should_panic: should_panic(&i, &self.cx)
should_panic: should_panic(&i, &self.cx),
allow_fail: is_allowed_fail(&i),
};
self.cx.testfns.push(test);
self.tests.push(i.ident);
Expand Down Expand Up @@ -383,6 +385,10 @@ fn is_ignored(i: &ast::Item) -> bool {
i.attrs.iter().any(|attr| attr.check_name("ignore"))
}

fn is_allowed_fail(i: &ast::Item) -> bool {
i.attrs.iter().any(|attr| attr.check_name("allow_fail"))
}

fn should_panic(i: &ast::Item, cx: &TestCtxt) -> ShouldPanic {
match i.attrs.iter().find(|attr| attr.check_name("should_panic")) {
Some(attr) => {
Expand Down Expand Up @@ -668,14 +674,16 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P<ast::Expr> {
}
}
};
let allow_fail_expr = ecx.expr_bool(span, test.allow_fail);

// self::test::TestDesc { ... }
let desc_expr = ecx.expr_struct(
span,
test_path("TestDesc"),
vec![field("name", name_expr),
field("ignore", ignore_expr),
field("should_panic", fail_expr)]);
field("should_panic", fail_expr),
field("allow_fail", allow_fail_expr)]);


let mut visible_path = match cx.toplevel_reexport {
Expand Down
Loading

0 comments on commit bea4e60

Please sign in to comment.