From 02b8533ac8c1cc4521b1da91e52c1327f656c44a Mon Sep 17 00:00:00 2001 From: Philipp Hansch Date: Sun, 24 Mar 2019 14:06:47 +0100 Subject: [PATCH 1/4] Add a way to track Rustfix UI test coverage This came out of the first Rustfix WG meeting. One of the goals is to enable Rustfix tests for all UI tests that trigger lints with `MachineApplicable` suggestions. In order to do that we first want to create a tracking issue that lists all files with missing `// run-rustfix` headers. This PR adds a `--rustfix-coverage` flag to `./x.py` and compiletest to list the files with the missing headers in `/tmp/rustfix_missing_coverage.txt`. From that file we can create the tracking issue and at some point also enforce the `// run-rustfix` flag on UI tests with `MachineApplicable` lints. --- src/bootstrap/flags.rs | 15 +++++++++++++++ src/bootstrap/test.rs | 4 ++++ src/tools/compiletest/src/common.rs | 5 +++++ src/tools/compiletest/src/main.rs | 19 +++++++++++++++++++ src/tools/compiletest/src/runtest.rs | 24 +++++++++++++++++++++++- 5 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 0f9a4271ac062..23719378c840a 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -56,6 +56,7 @@ pub enum Subcommand { rustc_args: Vec, fail_fast: bool, doc_tests: DocTests, + rustfix_coverage: bool, }, Bench { paths: Vec, @@ -188,6 +189,12 @@ To learn more about a subcommand, run `./x.py -h`" "mode describing what file the actual ui output will be compared to", "COMPARE MODE", ); + opts.optflag( + "", + "rustfix-coverage", + "enable this to generate a Rustfix coverage file, which is saved in \ + `/tmp/rustfix_missing_coverage.txt`", + ); } "bench" => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); @@ -363,6 +370,7 @@ Arguments: test_args: matches.opt_strs("test-args"), rustc_args: matches.opt_strs("rustc-args"), fail_fast: !matches.opt_present("no-fail-fast"), + rustfix_coverage: matches.opt_present("rustfix-coverage"), doc_tests: if matches.opt_present("doc") { DocTests::Only } else if matches.opt_present("no-doc") { @@ -467,6 +475,13 @@ impl Subcommand { } } + pub fn rustfix_coverage(&self) -> bool { + match *self { + Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, + _ => false, + } + } + pub fn compare_mode(&self) -> Option<&str> { match *self { Subcommand::Test { diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index bbe1872d3958d..41c73f307b6d0 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1284,6 +1284,10 @@ impl Step for Compiletest { cmd.arg("--android-cross-path").arg(""); } + if builder.config.cmd.rustfix_coverage() { + cmd.arg("--rustfix-coverage"); + } + builder.ci_env.force_coloring_in_ci(&mut cmd); let _folder = builder.fold_output(|| format!("test_{}", suite)); diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index 80b8a8b728bb2..fe0c4f14940ad 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -245,6 +245,11 @@ pub struct Config { /// mode describing what file the actual ui output will be compared to pub compare_mode: Option, + /// If true, this will generate a coverage file with UI test files that run `MachineApplicable` + /// lints but are missing `run-rustfix` annotations. The generated coverage file is created in + /// `/tmp/rustfix_missing_coverage.txt` + pub rustfix_coverage: bool, + // Configuration for various run-make tests frobbing things like C compilers // or querying about various LLVM component information. pub cc: String, diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index 86cdadade108f..1045ea1bf0c96 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -233,6 +233,12 @@ pub fn parse_config(args: Vec) -> Config { "mode describing what file the actual ui output will be compared to", "COMPARE MODE", ) + .optflag( + "", + "rustfix-coverage", + "enable this to generate a Rustfix coverage file, which is saved in \ + `/tmp/rustfix_missing_coverage.txt`", + ) .optflag("h", "help", "show this message"); let (argv0, args_) = args.split_first().unwrap(); @@ -336,6 +342,7 @@ pub fn parse_config(args: Vec) -> Config { color, remote_test_client: matches.opt_str("remote-test-client").map(PathBuf::from), compare_mode: matches.opt_str("compare-mode").map(CompareMode::parse), + rustfix_coverage: matches.opt_present("rustfix-coverage"), cc: matches.opt_str("cc").unwrap(), cxx: matches.opt_str("cxx").unwrap(), @@ -475,6 +482,18 @@ pub fn run_tests(config: &Config) { let _ = fs::remove_dir_all("tmp/partitioning-tests"); } + // If we want to collect rustfix coverage information, + // we first make sure that the coverage file does not exist. + // It will be created later on. + if config.rustfix_coverage { + let coverage_file_path = Path::new("/tmp/rustfix_missing_coverage.txt"); + if coverage_file_path.exists() { + if let Err(e) = fs::remove_file(coverage_file_path) { + panic!("Could not delete {} due to {}", coverage_file_path.display(), e) + } + } + } + let opts = test_opts(config); let tests = make_tests(config); // sadly osx needs some file descriptor limits raised for running tests in diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 3e3499edf60f0..0ca656bb13328 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -19,7 +19,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::env; use std::ffi::{OsStr, OsString}; use std::fmt; -use std::fs::{self, create_dir_all, File}; +use std::fs::{self, create_dir_all, File, OpenOptions}; use std::hash::{Hash, Hasher}; use std::io::prelude::*; use std::io::{self, BufReader}; @@ -2818,6 +2818,28 @@ impl<'test> TestCx<'test> { if self.config.compare_mode.is_some() { // don't test rustfix with nll right now + } else if self.config.rustfix_coverage { + // Find out which tests have `MachineApplicable` suggestions but are missing + // `run-rustfix` or `run-rustfix-only-machine-applicable` headers + let suggestions = get_suggestions_from_json( + &proc_res.stderr, + &HashSet::new(), + Filter::MachineApplicableOnly + ).unwrap(); + if suggestions.len() > 0 + && !self.props.run_rustfix + && !self.props.rustfix_only_machine_applicable { + let coverage_file_path = Path::new("/tmp/rustfix_missing_coverage.txt"); + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(coverage_file_path) + .expect("could not create or open file"); + + if let Err(_) = writeln!(file, "{}", self.testpaths.file.display()) { + panic!("couldn't write to {}", coverage_file_path.display()); + } + } } else if self.props.run_rustfix { // Apply suggestions from rustc to the code itself let unfixed_code = self From d485ebfc211533900464927d57d6500e29e519f3 Mon Sep 17 00:00:00 2001 From: Philipp Hansch Date: Mon, 25 Mar 2019 06:57:32 +0100 Subject: [PATCH 2/4] Fix two bootstrap tests --- src/bootstrap/builder.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index f93f3e72f83e7..7851ea3e363a4 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -1856,6 +1856,7 @@ mod __test { doc_tests: DocTests::No, bless: false, compare_mode: None, + rustfix_coverage: false, }; let build = Build::new(config); @@ -1897,6 +1898,7 @@ mod __test { doc_tests: DocTests::No, bless: false, compare_mode: None, + rustfix_coverage: false, }; let build = Build::new(config); From 98d7c5463b4f8099d7e1e50087deaa195589294f Mon Sep 17 00:00:00 2001 From: Philipp Hansch Date: Mon, 25 Mar 2019 06:58:30 +0100 Subject: [PATCH 3/4] s/lints/diagnostics Not all suggestions come from lints --- src/tools/compiletest/src/common.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index fe0c4f14940ad..7a15d753e2df1 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -246,8 +246,8 @@ pub struct Config { pub compare_mode: Option, /// If true, this will generate a coverage file with UI test files that run `MachineApplicable` - /// lints but are missing `run-rustfix` annotations. The generated coverage file is created in - /// `/tmp/rustfix_missing_coverage.txt` + /// diagnostics but are missing `run-rustfix` annotations. The generated coverage file is + /// created in `/tmp/rustfix_missing_coverage.txt` pub rustfix_coverage: bool, // Configuration for various run-make tests frobbing things like C compilers From d808bd46515e4de5d21a62b2c1557ac9d2751846 Mon Sep 17 00:00:00 2001 From: Philipp Hansch Date: Mon, 25 Mar 2019 22:48:35 +0100 Subject: [PATCH 4/4] Save coverage file in build_base path, not /tmp --- src/bootstrap/flags.rs | 2 +- src/tools/compiletest/src/common.rs | 2 +- src/tools/compiletest/src/main.rs | 7 ++++--- src/tools/compiletest/src/runtest.rs | 7 +++++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 23719378c840a..a1f89d6c86f1d 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -193,7 +193,7 @@ To learn more about a subcommand, run `./x.py -h`" "", "rustfix-coverage", "enable this to generate a Rustfix coverage file, which is saved in \ - `/tmp/rustfix_missing_coverage.txt`", + `//rustfix_missing_coverage.txt`", ); } "bench" => { diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index 7a15d753e2df1..9d4effb4f5860 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -247,7 +247,7 @@ pub struct Config { /// If true, this will generate a coverage file with UI test files that run `MachineApplicable` /// diagnostics but are missing `run-rustfix` annotations. The generated coverage file is - /// created in `/tmp/rustfix_missing_coverage.txt` + /// created in `//rustfix_missing_coverage.txt` pub rustfix_coverage: bool, // Configuration for various run-make tests frobbing things like C compilers diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index 1045ea1bf0c96..b9ddf04fad987 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -237,7 +237,7 @@ pub fn parse_config(args: Vec) -> Config { "", "rustfix-coverage", "enable this to generate a Rustfix coverage file, which is saved in \ - `/tmp/rustfix_missing_coverage.txt`", + `.//rustfix_missing_coverage.txt`", ) .optflag("h", "help", "show this message"); @@ -486,9 +486,10 @@ pub fn run_tests(config: &Config) { // we first make sure that the coverage file does not exist. // It will be created later on. if config.rustfix_coverage { - let coverage_file_path = Path::new("/tmp/rustfix_missing_coverage.txt"); + let mut coverage_file_path = config.build_base.clone(); + coverage_file_path.push("rustfix_missing_coverage.txt"); if coverage_file_path.exists() { - if let Err(e) = fs::remove_file(coverage_file_path) { + if let Err(e) = fs::remove_file(&coverage_file_path) { panic!("Could not delete {} due to {}", coverage_file_path.display(), e) } } diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 0ca656bb13328..c18b6db9a010f 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -2829,11 +2829,14 @@ impl<'test> TestCx<'test> { if suggestions.len() > 0 && !self.props.run_rustfix && !self.props.rustfix_only_machine_applicable { - let coverage_file_path = Path::new("/tmp/rustfix_missing_coverage.txt"); + let mut coverage_file_path = self.config.build_base.clone(); + coverage_file_path.push("rustfix_missing_coverage.txt"); + debug!("coverage_file_path: {}", coverage_file_path.display()); + let mut file = OpenOptions::new() .create(true) .append(true) - .open(coverage_file_path) + .open(coverage_file_path.as_path()) .expect("could not create or open file"); if let Err(_) = writeln!(file, "{}", self.testpaths.file.display()) {