Run the miri test suite on the aux builder and travis

.gitmodules

@@ -39,3 +39,6 @@
 [submodule "src/tools/rustfmt"]
 	path = src/tools/rustfmt
 	url = https://github.com/rust-lang-nursery/rustfmt.git
+[submodule "src/tools/miri"]
+	path = src/tools/miri
+	url = https://github.com/solson/miri.git

config.toml.example

@@ -291,6 +291,10 @@
 # When creating source tarballs whether or not to create a source tarball.
 #dist-src = false
 
+# Whether to also run the Miri tests suite when running tests.
+# As a side-effect also generates MIR for all libraries.
+#test-miri = false
+
 # =============================================================================
 # Options for specific targets
 #

src/bootstrap/bin/rustc.rs

@@ -246,6 +246,12 @@ fn main() {
             }
         }
 
+        // When running miri tests, we need to generate MIR for all libraries
+        if env::var("TEST_MIRI").ok().map_or(false, |val| val == "true") {
+            cmd.arg("-Zalways-encode-mir");
+            cmd.arg("-Zmir-emit-validate=1");
+        }
+
         // Force all crates compiled by this compiler to (a) be unstable and (b)
         // allow the `rustc_private` feature to link to other unstable crates
         // also in the sysroot.

src/bootstrap/builder.rs

@@ -249,18 +249,18 @@ impl<'a> Builder<'a> {
                 tool::UnstableBookGen, tool::Tidy, tool::Linkchecker, tool::CargoTest,
                 tool::Compiletest, tool::RemoteTestServer, tool::RemoteTestClient,
                 tool::RustInstaller, tool::Cargo, tool::Rls, tool::Rustdoc, tool::Clippy,
-                native::Llvm, tool::Rustfmt),
+                native::Llvm, tool::Rustfmt, tool::Miri),
             Kind::Test => describe!(check::Tidy, check::Bootstrap, check::DefaultCompiletest,
                 check::HostCompiletest, check::Crate, check::CrateLibrustc, check::Rustdoc,
                 check::Linkcheck, check::Cargotest, check::Cargo, check::Rls, check::Docs,
-                check::ErrorIndex, check::Distcheck, check::Rustfmt),
+                check::ErrorIndex, check::Distcheck, check::Rustfmt, check::Miri),
             Kind::Bench => describe!(check::Crate, check::CrateLibrustc),
             Kind::Doc => describe!(doc::UnstableBook, doc::UnstableBookGen, doc::TheBook,
                 doc::Standalone, doc::Std, doc::Test, doc::Rustc, doc::ErrorIndex, doc::Nomicon,
                 doc::Reference, doc::Rustdoc, doc::CargoBook),
             Kind::Dist => describe!(dist::Docs, dist::Mingw, dist::Rustc, dist::DebuggerScripts,
                 dist::Std, dist::Analysis, dist::Src, dist::PlainSourceTarball, dist::Cargo,
-                dist::Rls, dist::Extended, dist::HashSign),
+                dist::Rls, dist::Extended, dist::HashSign, dist::DontDistWithMiriEnabled),
             Kind::Install => describe!(install::Docs, install::Std, install::Cargo, install::Rls,
                 install::Analysis, install::Src, install::Rustc),
         }
@@ -475,6 +475,7 @@ impl<'a> Builder<'a> {
              } else {
                  PathBuf::from("/path/to/nowhere/rustdoc/not/required")
              })
+             .env("TEST_MIRI", self.config.test_miri.to_string())
              .env("RUSTC_FLAGS", self.rustc_flags(target).join(" "));
 
         if mode != Mode::Tool {

src/bootstrap/check.rs

@@ -23,7 +23,7 @@ use std::path::{PathBuf, Path};
 use std::process::Command;
 use std::io::Read;
 
-use build_helper::{self, output};
+use build_helper::{self, output, BuildExpectation};
 
 use builder::{Kind, RunConfig, ShouldRun, Builder, Compiler, Step};
 use cache::{INTERNER, Interned};
@@ -33,6 +33,7 @@ use native;
 use tool::{self, Tool};
 use util::{self, dylib_path, dylib_path_var};
 use {Build, Mode};
+use toolstate::ToolState;
 
 const ADB_TEST_DIR: &str = "/data/tmp/work";
 
@@ -64,17 +65,21 @@ impl fmt::Display for TestKind {
     }
 }
 
-fn try_run(build: &Build, cmd: &mut Command) {
+fn try_run_expecting(build: &Build, cmd: &mut Command, expect: BuildExpectation) {
     if !build.fail_fast {
-        if !build.try_run(cmd) {
+        if !build.try_run(cmd, expect) {
             let failures = build.delayed_failures.get();
             build.delayed_failures.set(failures + 1);
         }
     } else {
-        build.run(cmd);
+        build.run_expecting(cmd, expect);
     }
 }
 
+fn try_run(build: &Build, cmd: &mut Command) {
+    try_run_expecting(build, cmd, BuildExpectation::None)
+}
+
 fn try_run_quiet(build: &Build, cmd: &mut Command) {
     if !build.fail_fast {
         if !build.try_run_quiet(cmd) {
@@ -294,6 +299,56 @@ impl Step for Rustfmt {
     }
 }
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct Miri {
+    host: Interned<String>,
+}
+
+impl Step for Miri {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        let test_miri = run.builder.build.config.test_miri;
+        run.path("src/tools/miri").default_condition(test_miri)
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(Miri {
+            host: run.target,
+        });
+    }
+
+    /// Runs `cargo test` for miri.
+    fn run(self, builder: &Builder) {
+        let build = builder.build;
+        let host = self.host;
+        let compiler = builder.compiler(1, host);
+
+        let miri = builder.ensure(tool::Miri { compiler, target: self.host });
+        let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test");
+        cargo.arg("--manifest-path").arg(build.src.join("src/tools/miri/Cargo.toml"));
+
+        // Don't build tests dynamically, just a pain to work with
+        cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
+        // miri tests need to know about the stage sysroot
+        cargo.env("MIRI_SYSROOT", builder.sysroot(compiler));
+        cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler));
+        cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
+        cargo.env("MIRI_PATH", miri);
+
+        builder.add_rustc_lib_path(compiler, &mut cargo);
+
+        try_run_expecting(
+            build,
+            &mut cargo,
+            builder.build.config.toolstate.miri.passes(ToolState::Testing),
+        );
+    }
+}
+
+
 fn path_for_cargo(builder: &Builder, compiler: Compiler) -> OsString {
     // Configure PATH to find the right rustc. NB. we have to use PATH
     // and not RUSTC because the Cargo test suite has tests that will

src/bootstrap/config.rs

@@ -27,6 +27,7 @@ use util::exe;
 use cache::{INTERNER, Interned};
 use flags::Flags;
 pub use flags::Subcommand;
+use toolstate::ToolStates;
 
 /// Global configuration for the entire build and/or bootstrap.
 ///
@@ -111,6 +112,7 @@ pub struct Config {
     pub low_priority: bool,
     pub channel: String,
     pub quiet_tests: bool,
+    pub test_miri: bool,
     // Fallback musl-root for all targets
     pub musl_root: Option<PathBuf>,
     pub prefix: Option<PathBuf>,
@@ -130,6 +132,8 @@ pub struct Config {
     // These are either the stage0 downloaded binaries or the locally installed ones.
     pub initial_cargo: PathBuf,
     pub initial_rustc: PathBuf,
+
+    pub toolstate: ToolStates,
 }
 
 /// Per-target configuration stored in the global configuration structure.
@@ -269,6 +273,7 @@ struct Rust {
     debug: Option<bool>,
     dist_src: Option<bool>,
     quiet_tests: Option<bool>,
+    test_miri: Option<bool>,
 }
 
 /// TOML representation of how each build target is configured.
@@ -304,6 +309,7 @@ impl Config {
         config.codegen_tests = true;
         config.ignore_git = false;
         config.rust_dist_src = true;
+        config.test_miri = false;
 
         config.on_fail = flags.on_fail;
         config.stage = flags.stage;
@@ -330,6 +336,18 @@ impl Config {
             }
         }).unwrap_or_else(|| TomlConfig::default());
 
+        let toolstate_toml_path = config.src.join("src/tools/toolstate.toml");
+        let parse_toolstate = || -> Result<_, Box<::std::error::Error>> {
+            let mut f = File::open(toolstate_toml_path)?;
+            let mut contents = String::new();
+            f.read_to_string(&mut contents)?;
+            Ok(toml::from_str(&contents)?)
+        };
+        config.toolstate = parse_toolstate().unwrap_or_else(|err| {
+            println!("failed to parse TOML configuration 'toolstate.toml': {}", err);
+            process::exit(2);
+        });
+
         let build = toml.build.clone().unwrap_or(Build::default());
         set(&mut config.build, build.build.clone().map(|x| INTERNER.intern_string(x)));
         set(&mut config.build, flags.build);
@@ -444,6 +462,7 @@ impl Config {
             set(&mut config.channel, rust.channel.clone());
             set(&mut config.rust_dist_src, rust.dist_src);
             set(&mut config.quiet_tests, rust.quiet_tests);
+            set(&mut config.test_miri, rust.test_miri);
             config.rustc_default_linker = rust.default_linker.clone();
             config.rustc_default_ar = rust.default_ar.clone();
             config.musl_root = rust.musl_root.clone().map(PathBuf::from);

src/bootstrap/configure.py

@@ -38,6 +38,7 @@ def v(*args):
 o("docs", "build.docs", "build standard library documentation")
 o("compiler-docs", "build.compiler-docs", "build compiler documentation")
 o("optimize-tests", "rust.optimize-tests", "build tests with optimizations")
+o("test-miri", "rust.test-miri", "run miri's test suite")
 o("debuginfo-tests", "rust.debuginfo-tests", "build tests with debugger metadata")
 o("quiet-tests", "rust.quiet-tests", "enable quieter output when running tests")
 o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds")

src/bootstrap/dist.rs

@@ -1111,6 +1111,31 @@ impl Step for Rls {
     }
 }
 
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct DontDistWithMiriEnabled;
+
+impl Step for DontDistWithMiriEnabled {
+    type Output = PathBuf;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        let build_miri = run.builder.build.config.test_miri;
+        run.default_condition(build_miri)
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(DontDistWithMiriEnabled);
+    }
+
+    fn run(self, _: &Builder) -> PathBuf {
+        panic!("Do not distribute with miri enabled.\n\
+                The distributed libraries would include all MIR (increasing binary size).
+                The distributed MIR would include validation statements.");
+    }
+}
+
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Extended {
     stage: u32,

src/bootstrap/lib.rs

@@ -143,7 +143,8 @@ use std::path::{PathBuf, Path};
 use std::process::Command;
 use std::slice;
 
-use build_helper::{run_silent, run_suppressed, try_run_silent, try_run_suppressed, output, mtime};
+use build_helper::{run_silent, run_suppressed, try_run_silent, try_run_suppressed, output, mtime,
+                   BuildExpectation};
 
 use util::{exe, libdir, OutputFolder, CiEnv};
 
@@ -164,6 +165,7 @@ pub mod util;
 mod builder;
 mod cache;
 mod tool;
+mod toolstate;
 
 #[cfg(windows)]
 mod job;
@@ -542,32 +544,39 @@ impl Build {
             .join(libdir(&self.config.build))
     }
 
+    /// Runs a command, printing out nice contextual information if its build
+    /// status is not the expected one
+    fn run_expecting(&self, cmd: &mut Command, expect: BuildExpectation) {
+        self.verbose(&format!("running: {:?}", cmd));
+        run_silent(cmd, expect)
+    }
+
     /// Runs a command, printing out nice contextual information if it fails.
     fn run(&self, cmd: &mut Command) {
-        self.verbose(&format!("running: {:?}", cmd));
-        run_silent(cmd)
+        self.run_expecting(cmd, BuildExpectation::None)
     }
 
     /// Runs a command, printing out nice contextual information if it fails.
     fn run_quiet(&self, cmd: &mut Command) {
         self.verbose(&format!("running: {:?}", cmd));
-        run_suppressed(cmd)
+        run_suppressed(cmd, BuildExpectation::None)
     }
 
-    /// Runs a command, printing out nice contextual information if it fails.
-    /// Exits if the command failed to execute at all, otherwise returns its
-    /// `status.success()`.
-    fn try_run(&self, cmd: &mut Command) -> bool {
+    /// Runs a command, printing out nice contextual information if its build
+    /// status is not the expected one.
+    /// Exits if the command failed to execute at all, otherwise returns whether
+    /// the expectation was met
+    fn try_run(&self, cmd: &mut Command, expect: BuildExpectation) -> bool {
         self.verbose(&format!("running: {:?}", cmd));
-        try_run_silent(cmd)
+        try_run_silent(cmd, expect)
     }
 
     /// Runs a command, printing out nice contextual information if it fails.
     /// Exits if the command failed to execute at all, otherwise returns its
     /// `status.success()`.
     fn try_run_quiet(&self, cmd: &mut Command) -> bool {
         self.verbose(&format!("running: {:?}", cmd));
-        try_run_suppressed(cmd)
+        try_run_suppressed(cmd, BuildExpectation::None)
     }
 
     pub fn is_verbose(&self) -> bool {

src/bootstrap/mk/Makefile.in

@@ -56,6 +56,7 @@ check-aux:
 		src/tools/cargo \
 		src/tools/rls \
 		src/tools/rustfmt \
+		src/tools/miri \
 		src/test/pretty \
 		src/test/run-pass/pretty \
 		src/test/run-fail/pretty \