From f012427aa6abeb4013971108211d1bd6819b46f3 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 12:56:39 +0300 Subject: [PATCH 01/22] handle `ToolRustc` build stages automatically Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 23 +++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 75bfff340862a..b8ce92ac8af1a 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -76,11 +76,16 @@ impl Step for ToolBuild { /// This will build the specified tool with the specified `host` compiler in /// `stage` into the normal cargo output directory. fn run(self, builder: &Builder<'_>) -> PathBuf { - let compiler = self.compiler; let target = self.target; let mut tool = self.tool; let path = self.path; + let compiler = if self.mode == Mode::ToolRustc { + get_tool_rustc_compiler(builder, self.compiler) + } else { + self.compiler + }; + match self.mode { Mode::ToolRustc => { builder.ensure(compile::Std::new(compiler, compiler.host)); @@ -147,6 +152,9 @@ pub fn prepare_tool_cargo( source_type: SourceType, extra_features: &[String], ) -> CargoCommand { + let compiler = + if mode == Mode::ToolRustc { get_tool_rustc_compiler(builder, compiler) } else { compiler }; + let mut cargo = builder::Cargo::new(builder, compiler, mode, source_type, target, cmd_kind); let dir = builder.src.join(path); @@ -240,6 +248,19 @@ pub fn prepare_tool_cargo( cargo } +fn get_tool_rustc_compiler(builder: &Builder<'_>, target_compiler: Compiler) -> Compiler { + if builder.download_rustc() && target_compiler.stage == 1 { + // We already have the stage 1 compiler, we don't need to cut the stage. + builder.compiler(target_compiler.stage, builder.config.build) + } else { + // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise + // we'd have stageN/bin/rustc and stageN/bin/$rustc_tool be effectively different stage + // compilers, which isn't what we want. Rustc tools should be linked in the same way as the + // compiler it's paired with, so it must be built with the previous stage compiler. + builder.compiler(target_compiler.stage.saturating_sub(1), builder.config.build) + } +} + /// Links a built tool binary with the given `name` from the build directory to the /// tools directory. fn copy_link_tool_bin( From 164f2408a215a33f3b9d638dde2944a4dbeae6ea Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 12:59:20 +0300 Subject: [PATCH 02/22] pass target_compiler from Rustdoc Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index b8ce92ac8af1a..05c9ab29e057f 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -650,16 +650,7 @@ impl Step for Rustdoc { } } - let build_compiler = if builder.download_rustc() && target_compiler.stage == 1 { - // We already have the stage 1 compiler, we don't need to cut the stage. - builder.compiler(target_compiler.stage, builder.config.build) - } else { - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - builder.compiler(target_compiler.stage - 1, builder.config.build) - }; + let build_compiler = get_tool_rustc_compiler(builder, target_compiler); // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build @@ -682,7 +673,7 @@ impl Step for Rustdoc { // NOTE: Never modify the rustflags here, it breaks the build cache for other tools! let mut cargo = prepare_tool_cargo( builder, - build_compiler, + target_compiler, Mode::ToolRustc, target, Kind::Build, From 7638abb5807cb77548092420af25c012a399568e Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 13:05:36 +0300 Subject: [PATCH 03/22] bless tests Signed-off-by: onur-ozkan --- src/bootstrap/src/core/builder/tests.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index 445b5dfbeab22..433e02299c148 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -525,6 +525,7 @@ mod dist { first(cache.all::()), &[ rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), + rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 0), rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1), ] ); @@ -796,7 +797,10 @@ mod dist { // stage minus 1 if --stage is not 0. Very confusing! assert_eq!( first(builder.cache.all::()), - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] + &[ + tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, + tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, + ] ); } From 72e67e87da600492930c5289cfb018441ca8c732 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 13:34:33 +0300 Subject: [PATCH 04/22] return more advanced type from `ToolBuild` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 223 ++++++++++++--------- 1 file changed, 125 insertions(+), 98 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 05c9ab29e057f..8175c9213b97d 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -64,8 +64,15 @@ impl Builder<'_> { } } +#[derive(Clone)] +struct ToolBuildResult { + tool_path: PathBuf, + build_compiler: Compiler, + target_compiler: Compiler, +} + impl Step for ToolBuild { - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.never() @@ -75,12 +82,13 @@ impl Step for ToolBuild { /// /// This will build the specified tool with the specified `host` compiler in /// `stage` into the normal cargo output directory. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(mut self, builder: &Builder<'_>) -> ToolBuildResult { let target = self.target; let mut tool = self.tool; let path = self.path; - let compiler = if self.mode == Mode::ToolRustc { + let target_compiler = self.compiler; + self.compiler = if self.mode == Mode::ToolRustc { get_tool_rustc_compiler(builder, self.compiler) } else { self.compiler @@ -88,17 +96,17 @@ impl Step for ToolBuild { match self.mode { Mode::ToolRustc => { - builder.ensure(compile::Std::new(compiler, compiler.host)); - builder.ensure(compile::Rustc::new(compiler, target)); + builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); + builder.ensure(compile::Rustc::new(self.compiler, target)); } - Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)), + Mode::ToolStd => builder.ensure(compile::Std::new(self.compiler, target)), Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs _ => panic!("unexpected Mode for tool build"), } let mut cargo = prepare_tool_cargo( builder, - compiler, + self.compiler, self.mode, target, Kind::Build, @@ -136,7 +144,10 @@ impl Step for ToolBuild { if tool == "tidy" { tool = "rust-tidy"; } - copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool) + let tool_path = + copy_link_tool_bin(builder, target_compiler, self.target, self.mode, tool); + + ToolBuildResult { tool_path, build_compiler: self.compiler, target_compiler } } } } @@ -361,7 +372,7 @@ macro_rules! bootstrap_tool { extra_features: vec![], allow_features: concat!($($allow_features)*), cargo_args: vec![] - }) + }).tool_path } } )+ @@ -429,17 +440,19 @@ impl Step for OptimizedDist { // the tool requires it to be in place to run. builder.require_submodule("src/tools/rustc-perf", None); - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "opt-dist", - mode: Mode::ToolBootstrap, - path: "src/tools/opt-dist", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "opt-dist", + mode: Mode::ToolBootstrap, + path: "src/tools/opt-dist", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + .tool_path } } @@ -483,7 +496,7 @@ impl Step for RustcPerf { // a CLI. cargo_args: vec!["-p".to_string(), "collector".to_string()], }; - let collector_bin = builder.ensure(tool.clone()); + let collector_bin = builder.ensure(tool.clone()).tool_path; // We also need to symlink the `rustc-fake` binary to the corresponding directory, // because `collector` expects it in the same directory. copy_link_tool_bin(builder, tool.compiler, tool.target, tool.mode, "rustc-fake"); @@ -533,17 +546,19 @@ impl Step for ErrorIndex { } fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.compiler.host, - tool: "error_index_generator", - mode: Mode::ToolRustc, - path: "src/tools/error_index_generator", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.compiler.host, + tool: "error_index_generator", + mode: Mode::ToolRustc, + path: "src/tools/error_index_generator", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + .tool_path } } @@ -568,17 +583,19 @@ impl Step for RemoteTestServer { } fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "remote-test-server", - mode: Mode::ToolStd, - path: "src/tools/remote-test-server", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "remote-test-server", + mode: Mode::ToolStd, + path: "src/tools/remote-test-server", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + .tool_path } } @@ -754,17 +771,19 @@ impl Step for Cargo { fn run(self, builder: &Builder<'_>) -> PathBuf { builder.build.require_submodule("src/tools/cargo", None); - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "cargo", - mode: Mode::ToolRustc, - path: "src/tools/cargo", - source_type: SourceType::Submodule, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "cargo", + mode: Mode::ToolRustc, + path: "src/tools/cargo", + source_type: SourceType::Submodule, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + .tool_path } } @@ -797,17 +816,19 @@ impl Step for LldWrapper { let target = self.target_compiler.host; - let executable = builder.ensure(ToolBuild { - compiler: self.build_compiler, - target, - tool: "lld-wrapper", - mode: Mode::ToolStd, - path: "src/tools/lld-wrapper", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }); + let executable = builder + .ensure(ToolBuild { + compiler: self.build_compiler, + target, + tool: "lld-wrapper", + mode: Mode::ToolStd, + path: "src/tools/lld-wrapper", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + .tool_path; let libdir_bin = builder.sysroot_target_bindir(self.target_compiler, target); t!(fs::create_dir_all(&libdir_bin)); @@ -854,17 +875,19 @@ impl Step for RustAnalyzer { } fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer", - mode: Mode::ToolRustc, - path: "src/tools/rust-analyzer", - extra_features: vec!["in-rust-tree".to_owned()], - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - cargo_args: Vec::new(), - }) + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer", + mode: Mode::ToolRustc, + path: "src/tools/rust-analyzer", + extra_features: vec!["in-rust-tree".to_owned()], + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + cargo_args: Vec::new(), + }) + .tool_path } } @@ -898,17 +921,19 @@ impl Step for RustAnalyzerProcMacroSrv { } fn run(self, builder: &Builder<'_>) -> Option { - let path = builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer-proc-macro-srv", - mode: Mode::ToolRustc, - path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", - extra_features: vec!["in-rust-tree".to_owned()], - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - cargo_args: Vec::new(), - }); + let path = builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer-proc-macro-srv", + mode: Mode::ToolRustc, + path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", + extra_features: vec!["in-rust-tree".to_owned()], + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + cargo_args: Vec::new(), + }) + .tool_path; // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` // so that r-a can use it. @@ -1146,17 +1171,19 @@ fn run_tool_build_step( path: &'static str, add_bins_to_sysroot: Option<&[&str]>, ) -> PathBuf { - let tool = builder.ensure(ToolBuild { - compiler, - target, - tool: tool_name, - mode: Mode::ToolRustc, - path, - extra_features: vec![], - source_type: SourceType::InTree, - allow_features: "", - cargo_args: vec![], - }); + let tool = builder + .ensure(ToolBuild { + compiler, + target, + tool: tool_name, + mode: Mode::ToolRustc, + path, + extra_features: vec![], + source_type: SourceType::InTree, + allow_features: "", + cargo_args: vec![], + }) + .tool_path; // FIXME: This should just be an if-let-chain, but those are unstable. if let Some(add_bins_to_sysroot) = From 8e011e5e672440e9fb45476385d6e9866b72ee12 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 14:01:22 +0300 Subject: [PATCH 05/22] migrate llvm-bitcode-linker to `ToolBuild` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 78 ++++++++-------------- 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 8175c9213b97d..8bcb5770e39ff 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -145,7 +145,7 @@ impl Step for ToolBuild { tool = "rust-tidy"; } let tool_path = - copy_link_tool_bin(builder, target_compiler, self.target, self.mode, tool); + copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool); ToolBuildResult { tool_path, build_compiler: self.compiler, target_compiler } } @@ -976,50 +976,30 @@ impl Step for LlvmBitcodeLinker { instrument(level = "debug", name = "LlvmBitcodeLinker::run", skip_all) )] fn run(self, builder: &Builder<'_>) -> PathBuf { - let bin_name = "llvm-bitcode-linker"; - - // If enabled, use ci-rustc and skip building the in-tree compiler. - if !builder.download_rustc() { - builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); - builder.ensure(compile::Rustc::new(self.compiler, self.target)); - } - - let cargo = prepare_tool_cargo( - builder, - self.compiler, - Mode::ToolRustc, - self.target, - Kind::Build, - "src/tools/llvm-bitcode-linker", - SourceType::InTree, - &self.extra_features, - ); - - let _guard = builder.msg_tool( - Kind::Build, - Mode::ToolRustc, - bin_name, - self.compiler.stage, - &self.compiler.host, - &self.target, - ); - - cargo.into_cmd().run(builder); - - let tool_out = builder - .cargo_out(self.compiler, Mode::ToolRustc, self.target) - .join(exe(bin_name, self.compiler.host)); + let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "llvm-bitcode-linker", + mode: Mode::ToolRustc, + path: "src/tools/llvm-bitcode-linker", + source_type: SourceType::InTree, + extra_features: self.extra_features, + allow_features: "", + cargo_args: Vec::new(), + }); - if self.compiler.stage > 0 { + if target_compiler.stage > 0 { let bindir_self_contained = builder - .sysroot(self.compiler) + .sysroot(target_compiler) .join(format!("lib/rustlib/{}/bin/self-contained", self.target.triple)); t!(fs::create_dir_all(&bindir_self_contained)); - let bin_destination = bindir_self_contained.join(exe(bin_name, self.compiler.host)); - builder.copy_link(&tool_out, &bin_destination); + let bin_destination = + bindir_self_contained.join(exe("llvm-bitcode-linker", target_compiler.host)); + builder.copy_link(&tool_path, &bin_destination); bin_destination } else { - tool_out + tool_path } } } @@ -1171,7 +1151,7 @@ fn run_tool_build_step( path: &'static str, add_bins_to_sysroot: Option<&[&str]>, ) -> PathBuf { - let tool = builder + let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = builder .ensure(ToolBuild { compiler, target, @@ -1182,28 +1162,24 @@ fn run_tool_build_step( source_type: SourceType::InTree, allow_features: "", cargo_args: vec![], - }) - .tool_path; + }); // FIXME: This should just be an if-let-chain, but those are unstable. if let Some(add_bins_to_sysroot) = - add_bins_to_sysroot.filter(|bins| !bins.is_empty() && compiler.stage > 0) + add_bins_to_sysroot.filter(|bins| !bins.is_empty() && target_compiler.stage > 0) { - let bindir = builder.sysroot(compiler).join("bin"); + let bindir = builder.sysroot(target_compiler).join("bin"); t!(fs::create_dir_all(&bindir)); - let tools_out = builder.cargo_out(compiler, Mode::ToolRustc, target); - for add_bin in add_bins_to_sysroot { - let bin_source = tools_out.join(exe(add_bin, target)); - let bin_destination = bindir.join(exe(add_bin, compiler.host)); - builder.copy_link(&bin_source, &bin_destination); + let bin_destination = bindir.join(exe(add_bin, target_compiler.host)); + builder.copy_link(&tool_path, &bin_destination); } // Return a path into the bin dir. - bindir.join(exe(tool_name, compiler.host)) + bindir.join(exe(tool_name, target_compiler.host)) } else { - tool + tool_path } } From a33843968ceae214b2a2c8699e0612bb582a3a4e Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 15:06:23 +0300 Subject: [PATCH 06/22] return `ToolBuildResult` to utilize them from callers Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/dist.rs | 14 +-- src/bootstrap/src/core/build_steps/test.rs | 8 +- src/bootstrap/src/core/build_steps/tool.rs | 118 +++++++++------------ src/bootstrap/src/core/builder/mod.rs | 6 +- src/bootstrap/src/core/builder/tests.rs | 5 +- 5 files changed, 66 insertions(+), 85 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 85b224771bb31..403aa48bad9b7 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1180,7 +1180,7 @@ impl Step for Rls { let mut tarball = Tarball::new(builder, "rls", &target.triple); tarball.set_overlay(OverlayKind::Rls); tarball.is_preview(true); - tarball.add_file(rls, "bin", 0o755); + tarball.add_file(rls.tool_path, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/rls"); Some(tarball.generate()) } @@ -1268,8 +1268,8 @@ impl Step for Clippy { let mut tarball = Tarball::new(builder, "clippy", &target.triple); tarball.set_overlay(OverlayKind::Clippy); tarball.is_preview(true); - tarball.add_file(clippy, "bin", 0o755); - tarball.add_file(cargoclippy, "bin", 0o755); + tarball.add_file(clippy.tool_path, "bin", 0o755); + tarball.add_file(cargoclippy.tool_path, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/clippy"); Some(tarball.generate()) } @@ -1318,8 +1318,8 @@ impl Step for Miri { let mut tarball = Tarball::new(builder, "miri", &target.triple); tarball.set_overlay(OverlayKind::Miri); tarball.is_preview(true); - tarball.add_file(miri, "bin", 0o755); - tarball.add_file(cargomiri, "bin", 0o755); + tarball.add_file(miri.tool_path, "bin", 0o755); + tarball.add_file(cargomiri.tool_path, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/miri"); Some(tarball.generate()) } @@ -1449,8 +1449,8 @@ impl Step for Rustfmt { let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); tarball.set_overlay(OverlayKind::Rustfmt); tarball.is_preview(true); - tarball.add_file(rustfmt, "bin", 0o755); - tarball.add_file(cargofmt, "bin", 0o755); + tarball.add_file(rustfmt.tool_path, "bin", 0o755); + tarball.add_file(cargofmt.tool_path, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/rustfmt"); Some(tarball.generate()) } diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index b3f4a7bad99c2..1820c13a861cd 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -427,7 +427,8 @@ impl Step for Rustfmt { let host = self.host; let compiler = builder.compiler(stage, host); - builder.ensure(tool::Rustfmt { compiler, target: self.host }); + let tool_result = builder.ensure(tool::Rustfmt { compiler, target: self.host }); + let compiler = tool_result.build_compiler; let mut cargo = tool::prepare_tool_cargo( builder, @@ -571,7 +572,7 @@ impl Step for Miri { // miri tests need to know about the stage sysroot cargo.env("MIRI_SYSROOT", &miri_sysroot); cargo.env("MIRI_HOST_SYSROOT", &host_sysroot); - cargo.env("MIRI", &miri); + cargo.env("MIRI", &miri.tool_path); // Set the target. cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg()); @@ -743,7 +744,8 @@ impl Step for Clippy { let host = self.host; let compiler = builder.compiler(stage, host); - builder.ensure(tool::Clippy { compiler, target: self.host }); + let tool_result = builder.ensure(tool::Clippy { compiler, target: self.host }); + let compiler = tool_result.build_compiler; let mut cargo = tool::prepare_tool_cargo( builder, compiler, diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 8bcb5770e39ff..cbb973428e2ca 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -65,10 +65,10 @@ impl Builder<'_> { } #[derive(Clone)] -struct ToolBuildResult { - tool_path: PathBuf, - build_compiler: Compiler, - target_compiler: Compiler, +pub struct ToolBuildResult { + pub tool_path: PathBuf, + pub build_compiler: Compiler, + pub target_compiler: Compiler, } impl Step for ToolBuild { @@ -114,10 +114,28 @@ impl Step for ToolBuild { self.source_type, &self.extra_features, ); + + if path.ends_with("/rustdoc") && + // rustdoc is performance sensitive, so apply LTO to it. + is_lto_stage(&self.compiler) + { + let lto = match builder.config.rust_lto { + RustcLto::Off => Some("off"), + RustcLto::Thin => Some("thin"), + RustcLto::Fat => Some("fat"), + RustcLto::ThinLocal => None, + }; + if let Some(lto) = lto { + cargo.env(cargo_profile_var("LTO", &builder.config), lto); + } + } + if !self.allow_features.is_empty() { cargo.allow_features(self.allow_features); } + cargo.args(self.cargo_args); + let _guard = builder.msg_tool( Kind::Build, self.mode, @@ -163,9 +181,6 @@ pub fn prepare_tool_cargo( source_type: SourceType, extra_features: &[String], ) -> CargoCommand { - let compiler = - if mode == Mode::ToolRustc { get_tool_rustc_compiler(builder, compiler) } else { compiler }; - let mut cargo = builder::Cargo::new(builder, compiler, mode, source_type, target, cmd_kind); let dir = builder.src.join(path); @@ -667,14 +682,6 @@ impl Step for Rustdoc { } } - let build_compiler = get_tool_rustc_compiler(builder, target_compiler); - - // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually - // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build - // it. - builder.ensure(compile::Std::new(build_compiler, target_compiler.host)); - builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host)); - // The presence of `target_compiler` ensures that the necessary libraries (codegen backends, // compiler libraries, ...) are built. Rustdoc does not require the presence of any // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since @@ -682,65 +689,39 @@ impl Step for Rustdoc { // libraries here. The intuition here is that If we've built a compiler, we should be able // to build rustdoc. // - let mut features = Vec::new(); + let mut extra_features = Vec::new(); if builder.config.jemalloc(target) { - features.push("jemalloc".to_string()); + extra_features.push("jemalloc".to_string()); } - // NOTE: Never modify the rustflags here, it breaks the build cache for other tools! - let mut cargo = prepare_tool_cargo( - builder, - target_compiler, - Mode::ToolRustc, - target, - Kind::Build, - "src/tools/rustdoc", - SourceType::InTree, - features.as_slice(), - ); - - // rustdoc is performance sensitive, so apply LTO to it. - if is_lto_stage(&build_compiler) { - let lto = match builder.config.rust_lto { - RustcLto::Off => Some("off"), - RustcLto::Thin => Some("thin"), - RustcLto::Fat => Some("fat"), - RustcLto::ThinLocal => None, - }; - if let Some(lto) = lto { - cargo.env(cargo_profile_var("LTO", &builder.config), lto); - } - } - - let _guard = builder.msg_tool( - Kind::Build, - Mode::ToolRustc, - "rustdoc", - build_compiler.stage, - &self.compiler.host, - &target, - ); - cargo.into_cmd().run(builder); - - // Cargo adds a number of paths to the dylib search path on windows, which results in - // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" - // rustdoc a different name. - let tool_rustdoc = builder - .cargo_out(build_compiler, Mode::ToolRustc, target) - .join(exe("rustdoc_tool_binary", target_compiler.host)); + let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = + builder.ensure(ToolBuild { + compiler: target_compiler, + target, + // Cargo adds a number of paths to the dylib search path on windows, which results in + // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" + // rustdoc a different name. + tool: "rustdoc_tool_binary", + mode: Mode::ToolRustc, + path: "src/tools/rustdoc", + source_type: SourceType::InTree, + extra_features, + allow_features: "", + cargo_args: Vec::new(), + }); // don't create a stage0-sysroot/bin directory. if target_compiler.stage > 0 { if builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None { // Due to LTO a lot of debug info from C++ dependencies such as jemalloc can make it into // our final binaries - compile::strip_debug(builder, target, &tool_rustdoc); + compile::strip_debug(builder, target, &tool_path); } let bin_rustdoc = bin_rustdoc(); - builder.copy_link(&tool_rustdoc, &bin_rustdoc); + builder.copy_link(&tool_path, &bin_rustdoc); bin_rustdoc } else { - tool_rustdoc + tool_path } } } @@ -1084,7 +1065,7 @@ macro_rules! tool_extended { } impl Step for $name { - type Output = PathBuf; + type Output = ToolBuildResult; const DEFAULT: bool = true; // Overridden by `should_run_tool_build_step` const ONLY_HOSTS: bool = true; @@ -1104,7 +1085,7 @@ macro_rules! tool_extended { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { let Self { compiler, target } = self; run_tool_build_step( builder, @@ -1150,9 +1131,9 @@ fn run_tool_build_step( tool_name: &'static str, path: &'static str, add_bins_to_sysroot: Option<&[&str]>, -) -> PathBuf { - let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = builder - .ensure(ToolBuild { +) -> ToolBuildResult { + let ToolBuildResult { tool_path, build_compiler, target_compiler } = + builder.ensure(ToolBuild { compiler, target, tool: tool_name, @@ -1177,9 +1158,10 @@ fn run_tool_build_step( } // Return a path into the bin dir. - bindir.join(exe(tool_name, target_compiler.host)) + let path = bindir.join(exe(tool_name, target_compiler.host)); + ToolBuildResult { tool_path: path, build_compiler, target_compiler } } else { - tool_path + ToolBuildResult { tool_path, build_compiler, target_compiler } } } diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index daef8fa3c8a34..b7820acc7e8b3 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -1415,7 +1415,7 @@ impl<'a> Builder<'a> { let mut dylib_path = helpers::dylib_path(); dylib_path.insert(0, self.sysroot(run_compiler).join("lib")); - let mut cmd = command(cargo_clippy); + let mut cmd = command(cargo_clippy.tool_path); cmd.env(helpers::dylib_path_var(), env::join_paths(&dylib_path).unwrap()); cmd.env("CARGO", &self.initial_cargo); cmd @@ -1430,8 +1430,8 @@ impl<'a> Builder<'a> { let cargo_miri = self.ensure(tool::CargoMiri { compiler: build_compiler, target: self.build.build }); // Invoke cargo-miri, make sure it can find miri and cargo. - let mut cmd = command(cargo_miri); - cmd.env("MIRI", &miri); + let mut cmd = command(cargo_miri.tool_path); + cmd.env("MIRI", &miri.tool_path); cmd.env("CARGO", &self.initial_cargo); // Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler`, // so they match the Miri we just built. However this means they are actually living one diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index 433e02299c148..a828a844f8103 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -797,10 +797,7 @@ mod dist { // stage minus 1 if --stage is not 0. Very confusing! assert_eq!( first(builder.cache.all::()), - &[ - tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, - tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, - ] + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] ); } From 6cf650fce7c9a599b18df72b1bac202a19eb81cd Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 16:32:13 +0300 Subject: [PATCH 07/22] remove manually handled stage offs Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/run.rs | 6 +----- src/bootstrap/src/core/build_steps/test.rs | 25 +++++----------------- src/bootstrap/src/core/build_steps/tool.rs | 17 ++++++--------- src/bootstrap/src/core/builder/mod.rs | 25 ++++++++++------------ 4 files changed, 24 insertions(+), 49 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 2b17e02cae5a4..fea8232296eca 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -126,11 +126,7 @@ impl Step for Miri { // This compiler runs on the host, we'll just use it for the target. let target_compiler = builder.compiler(stage, host); - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - let host_compiler = builder.compiler(stage - 1, host); + let host_compiler = tool::get_tool_rustc_compiler(builder, target_compiler); // Get a target sysroot for Miri. let miri_sysroot = test::Miri::build_miri_sysroot(builder, target_compiler, target); diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 1820c13a861cd..e871a40731e34 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -523,16 +523,11 @@ impl Step for Miri { // This compiler runs on the host, we'll just use it for the target. let target_compiler = builder.compiler(stage, host); - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - let host_compiler = builder.compiler(stage - 1, host); // Build our tools. - let miri = builder.ensure(tool::Miri { compiler: host_compiler, target: host }); + let miri = builder.ensure(tool::Miri { compiler: target_compiler, target: host }); // the ui tests also assume cargo-miri has been built - builder.ensure(tool::CargoMiri { compiler: host_compiler, target: host }); + builder.ensure(tool::CargoMiri { compiler: target_compiler, target: host }); // We also need sysroots, for Miri and for the host (the latter for build scripts). // This is for the tests so everything is done with the target compiler. @@ -543,7 +538,8 @@ impl Step for Miri { // Miri has its own "target dir" for ui test dependencies. Make sure it gets cleared when // the sysroot gets rebuilt, to avoid "found possibly newer version of crate `std`" errors. if !builder.config.dry_run() { - let ui_test_dep_dir = builder.stage_out(host_compiler, Mode::ToolStd).join("miri_ui"); + let ui_test_dep_dir = + builder.stage_out(miri.build_compiler, Mode::ToolStd).join("miri_ui"); // The mtime of `miri_sysroot` changes when the sysroot gets rebuilt (also see // ). // We can hence use that directly as a signal to clear the ui test dir. @@ -554,7 +550,7 @@ impl Step for Miri { // This is with the Miri crate, so it uses the host compiler. let mut cargo = tool::prepare_tool_cargo( builder, - host_compiler, + miri.build_compiler, Mode::ToolRustc, host, Kind::Test, @@ -1722,17 +1718,6 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the // If we're using `--stage 0`, we should provide the bootstrap cargo. builder.initial_cargo.clone() } else { - // We need to properly build cargo using the suitable stage compiler. - - let compiler = builder.download_rustc().then_some(compiler).unwrap_or_else(|| - // HACK: currently tool stages are off-by-one compared to compiler stages, i.e. if - // you give `tool::Cargo` a stage 1 rustc, it will cause stage 2 rustc to be built - // and produce a cargo built with stage 2 rustc. To fix this, we need to chop off - // the compiler stage by 1 to align with expected `./x test run-make --stage N` - // behavior, i.e. we need to pass `N - 1` compiler stage to cargo. See also Miri - // which does a similar hack. - builder.compiler(builder.top_stage - 1, compiler.host)); - builder.ensure(tool::Cargo { compiler, target: compiler.host }) }; diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index cbb973428e2ca..f734776a3d871 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -274,7 +274,10 @@ pub fn prepare_tool_cargo( cargo } -fn get_tool_rustc_compiler(builder: &Builder<'_>, target_compiler: Compiler) -> Compiler { +pub(crate) fn get_tool_rustc_compiler( + builder: &Builder<'_>, + target_compiler: Compiler, +) -> Compiler { if builder.download_rustc() && target_compiler.stage == 1 { // We already have the stage 1 compiler, we don't need to cut the stage. builder.compiler(target_compiler.stage, builder.config.build) @@ -555,8 +558,7 @@ impl Step for ErrorIndex { // src/tools/error-index-generator` which almost nobody does. // Normally, `x.py test` or `x.py doc` will use the // `ErrorIndex::command` function instead. - let compiler = - run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build); + let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); run.builder.ensure(ErrorIndex { compiler }); } @@ -631,13 +633,8 @@ impl Step for Rustdoc { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustdoc { - // NOTE: this is somewhat unique in that we actually want a *target* - // compiler here, because rustdoc *is* a compiler. We won't be using - // this as the compiler to build with, but rather this is "what - // compiler are we producing"? - compiler: run.builder.compiler(run.builder.top_stage, run.target), - }); + run.builder + .ensure(Rustdoc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); } fn run(self, builder: &Builder<'_>) -> PathBuf { diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index b7820acc7e8b3..c94bdd0a24684 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -1408,10 +1408,9 @@ impl<'a> Builder<'a> { return cmd; } - let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build); - self.ensure(tool::Clippy { compiler: build_compiler, target: self.build.build }); + let _ = self.ensure(tool::Clippy { compiler: run_compiler, target: self.build.build }); let cargo_clippy = - self.ensure(tool::CargoClippy { compiler: build_compiler, target: self.build.build }); + self.ensure(tool::CargoClippy { compiler: run_compiler, target: self.build.build }); let mut dylib_path = helpers::dylib_path(); dylib_path.insert(0, self.sysroot(run_compiler).join("lib")); @@ -1423,23 +1422,21 @@ impl<'a> Builder<'a> { pub fn cargo_miri_cmd(&self, run_compiler: Compiler) -> BootstrapCommand { assert!(run_compiler.stage > 0, "miri can not be invoked at stage 0"); - let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build); - // Prepare the tools - let miri = self.ensure(tool::Miri { compiler: build_compiler, target: self.build.build }); + let miri = self.ensure(tool::Miri { compiler: run_compiler, target: self.build.build }); let cargo_miri = - self.ensure(tool::CargoMiri { compiler: build_compiler, target: self.build.build }); + self.ensure(tool::CargoMiri { compiler: run_compiler, target: self.build.build }); // Invoke cargo-miri, make sure it can find miri and cargo. let mut cmd = command(cargo_miri.tool_path); cmd.env("MIRI", &miri.tool_path); cmd.env("CARGO", &self.initial_cargo); - // Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler`, - // so they match the Miri we just built. However this means they are actually living one - // stage up, i.e. we are running `stage0-tools-bin/miri` with the libraries in `stage1/lib`. - // This is an unfortunate off-by-1 caused (possibly) by the fact that Miri doesn't have an - // "assemble" step like rustc does that would cross the stage boundary. We can't use - // `add_rustc_lib_path` as that's a NOP on Windows but we do need these libraries added to - // the PATH due to the stage mismatch. + // Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler` + // in `tool::ToolBuild` step, so they match the Miri we just built. However this means they + // are actually living one stage up, i.e. we are running `stage0-tools-bin/miri` with the + // libraries in `stage1/lib`. This is an unfortunate off-by-1 caused (possibly) by the fact + // that Miri doesn't have an "assemble" step like rustc does that would cross the stage boundary. + // We can't use `add_rustc_lib_path` as that's a NOP on Windows but we do need these libraries + // added to the PATH due to the stage mismatch. // Also see https://github.com/rust-lang/rust/pull/123192#issuecomment-2028901503. add_dylib_path(self.rustc_lib_paths(run_compiler), &mut cmd); cmd From 5e5b1b054aae3407f4226ccae2b321e114990776 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 15:34:34 +0000 Subject: [PATCH 08/22] document tool implementations Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/tool.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index f734776a3d871..7adb7b4681fbd 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -1,3 +1,14 @@ +//! This module handles building and managing various tools in bootstrap +//! build system. +//! +//! **What It Does** +//! - Defines how tools are built, configured and installed. +//! - Manages tool dependencies and build steps. +//! - Copies built tool binaries to the correct locations. +//! +//! Each Rust tool **MUST** utilize `ToolBuild` inside their `Step` logic, +//! return `ToolBuildResult` and should never prepare `cargo` invocations manually. + use std::path::PathBuf; use std::{env, fs}; @@ -64,10 +75,16 @@ impl Builder<'_> { } } +/// Result of the tool build process. Each `Step` in this module is responsible +/// for using this type as `type Output = ToolBuildResult;` #[derive(Clone)] pub struct ToolBuildResult { + /// Executable path of the corresponding tool that was built. pub tool_path: PathBuf, + /// Compiler used to build the tool. For non-`ToolRustc` tools this is equal to `target_compiler`. + /// For `ToolRustc` this is one stage before of the `target_compiler`. pub build_compiler: Compiler, + /// Target compiler passed to `Step`. pub target_compiler: Compiler, } @@ -274,6 +291,7 @@ pub fn prepare_tool_cargo( cargo } +/// Handle stage-off logic for `ToolRustc` tools when necessary. pub(crate) fn get_tool_rustc_compiler( builder: &Builder<'_>, target_compiler: Compiler, From baef666f788108620263b5d851d845735346acaa Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 16:00:06 +0000 Subject: [PATCH 09/22] adapt tool module to `ToolBuildResult` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/compile.rs | 26 +- src/bootstrap/src/core/build_steps/dist.rs | 8 +- src/bootstrap/src/core/build_steps/perf.rs | 2 +- src/bootstrap/src/core/build_steps/test.rs | 18 +- src/bootstrap/src/core/build_steps/tool.rs | 312 +++++++++--------- src/bootstrap/src/core/builder/mod.rs | 2 +- 6 files changed, 189 insertions(+), 179 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index aa08dd7e42488..8a0d8b979c37b 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1977,13 +1977,14 @@ impl Step for Assemble { let maybe_install_llvm_bitcode_linker = |compiler| { if builder.config.llvm_bitcode_linker_enabled { trace!("llvm-bitcode-linker enabled, installing"); - let src_path = builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker { - compiler, - target: target_compiler.host, - extra_features: vec![], - }); + let llvm_bitcode_linker = + builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker { + compiler, + target: target_compiler.host, + extra_features: vec![], + }); let tool_exe = exe("llvm-bitcode-linker", target_compiler.host); - builder.copy_link(&src_path, &libdir_bin.join(tool_exe)); + builder.copy_link(&llvm_bitcode_linker.tool_path, &libdir_bin.join(tool_exe)); } }; @@ -2171,14 +2172,13 @@ impl Step for Assemble { // logic to create the final binary. This is used by the // `wasm32-wasip2` target of Rust. if builder.tool_enabled("wasm-component-ld") { - let wasm_component_ld_exe = - builder.ensure(crate::core::build_steps::tool::WasmComponentLd { - compiler: build_compiler, - target: target_compiler.host, - }); + let wasm_component = builder.ensure(crate::core::build_steps::tool::WasmComponentLd { + compiler: build_compiler, + target: target_compiler.host, + }); builder.copy_link( - &wasm_component_ld_exe, - &libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()), + &wasm_component.tool_path, + &libdir_bin.join(wasm_component.tool_path.file_name().unwrap()), ); } diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 403aa48bad9b7..f742a277a95a7 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -430,7 +430,7 @@ impl Step for Rustc { }, builder.kind, ) { - builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755); + builder.install(&ra_proc_macro_srv.tool_path, &image.join("libexec"), 0o755); } let libdir_relative = builder.libdir_relative(compiler); @@ -1134,7 +1134,7 @@ impl Step for Cargo { let mut tarball = Tarball::new(builder, "cargo", &target.triple); tarball.set_overlay(OverlayKind::Cargo); - tarball.add_file(cargo, "bin", 0o755); + tarball.add_file(cargo.tool_path, "bin", 0o755); tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); tarball.add_dir(etc.join("man"), "share/man/man1"); @@ -1222,7 +1222,7 @@ impl Step for RustAnalyzer { let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); tarball.set_overlay(OverlayKind::RustAnalyzer); tarball.is_preview(true); - tarball.add_file(rust_analyzer, "bin", 0o755); + tarball.add_file(rust_analyzer.tool_path, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); Some(tarball.generate()) } @@ -2272,7 +2272,7 @@ impl Step for LlvmBitcodeLinker { tarball.set_overlay(OverlayKind::LlvmBitcodeLinker); tarball.is_preview(true); - tarball.add_file(llbc_linker, self_contained_bin_dir, 0o755); + tarball.add_file(llbc_linker.tool_path, self_contained_bin_dir, 0o755); Some(tarball.generate()) } diff --git a/src/bootstrap/src/core/build_steps/perf.rs b/src/bootstrap/src/core/build_steps/perf.rs index 98c63a41e768b..6962001fdc240 100644 --- a/src/bootstrap/src/core/build_steps/perf.rs +++ b/src/bootstrap/src/core/build_steps/perf.rs @@ -166,7 +166,7 @@ Consider setting `rust.debuginfo-level = 1` in `config.toml`."#); let results_dir = rustc_perf_dir.join("results"); builder.create_dir(&results_dir); - let mut cmd = command(collector); + let mut cmd = command(collector.tool_path); // We need to set the working directory to `src/tools/rustc-perf`, so that it can find the directory // with compile-time benchmarks. diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index e871a40731e34..c7e9675faa2a1 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -263,7 +263,7 @@ impl Step for Cargotest { let _time = helpers::timeit(builder); let mut cmd = builder.tool_cmd(Tool::CargoTest); - cmd.arg(&cargo) + cmd.arg(&cargo.tool_path) .arg(&out_dir) .args(builder.config.test_args()) .env("RUSTC", builder.rustc(compiler)) @@ -1718,7 +1718,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the // If we're using `--stage 0`, we should provide the bootstrap cargo. builder.initial_cargo.clone() } else { - builder.ensure(tool::Cargo { compiler, target: compiler.host }) + builder.ensure(tool::Cargo { compiler, target: compiler.host }).tool_path }; cmd.arg("--cargo-path").arg(cargo_path); @@ -1739,9 +1739,10 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the // Use the beta compiler for jsondocck let json_compiler = compiler.with_stage(0); cmd.arg("--jsondocck-path") - .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target })); - cmd.arg("--jsondoclint-path") - .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); + .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target }).tool_path); + cmd.arg("--jsondoclint-path").arg( + builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }).tool_path, + ); } if matches!(mode, "coverage-map" | "coverage-run") { @@ -2976,12 +2977,15 @@ impl Step for RemoteCopyLibs { builder.info(&format!("REMOTE copy libs to emulator ({target})")); - let server = builder.ensure(tool::RemoteTestServer { compiler, target }); + let remote_test_server = builder.ensure(tool::RemoteTestServer { compiler, target }); // Spawn the emulator and wait for it to come online let tool = builder.tool_exe(Tool::RemoteTestClient); let mut cmd = command(&tool); - cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); + cmd.arg("spawn-emulator") + .arg(target.triple) + .arg(&remote_test_server.tool_path) + .arg(builder.tempdir()); if let Some(rootfs) = builder.qemu_rootfs(target) { cmd.arg(rootfs); } diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 7adb7b4681fbd..39acb646dff4d 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -347,7 +347,7 @@ macro_rules! bootstrap_tool { self.ensure($name { compiler: self.compiler(0, self.config.build), target: self.config.build, - }), + }).tool_path, )+ } } @@ -361,7 +361,7 @@ macro_rules! bootstrap_tool { } impl Step for $name { - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path($path) @@ -383,7 +383,7 @@ macro_rules! bootstrap_tool { skip_all, ), )] - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { $( for submodule in $submodules { builder.require_submodule(submodule, None); @@ -408,7 +408,7 @@ macro_rules! bootstrap_tool { extra_features: vec![], allow_features: concat!($($allow_features)*), cargo_args: vec![] - }).tool_path + }) } } )+ @@ -458,7 +458,7 @@ pub struct OptimizedDist { } impl Step for OptimizedDist { - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path("src/tools/opt-dist") @@ -471,24 +471,22 @@ impl Step for OptimizedDist { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { // We need to ensure the rustc-perf submodule is initialized when building opt-dist since // the tool requires it to be in place to run. builder.require_submodule("src/tools/rustc-perf", None); - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "opt-dist", - mode: Mode::ToolBootstrap, - path: "src/tools/opt-dist", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - .tool_path + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "opt-dist", + mode: Mode::ToolBootstrap, + path: "src/tools/opt-dist", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) } } @@ -502,7 +500,7 @@ pub struct RustcPerf { impl Step for RustcPerf { /// Path to the built `collector` binary. - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path("src/tools/rustc-perf") @@ -515,7 +513,7 @@ impl Step for RustcPerf { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { // We need to ensure the rustc-perf submodule is initialized. builder.require_submodule("src/tools/rustc-perf", None); @@ -532,12 +530,12 @@ impl Step for RustcPerf { // a CLI. cargo_args: vec!["-p".to_string(), "collector".to_string()], }; - let collector_bin = builder.ensure(tool.clone()).tool_path; + let res = builder.ensure(tool.clone()); // We also need to symlink the `rustc-fake` binary to the corresponding directory, // because `collector` expects it in the same directory. copy_link_tool_bin(builder, tool.compiler, tool.target, tool.mode, "rustc-fake"); - collector_bin + res } } @@ -552,7 +550,7 @@ impl ErrorIndex { // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc. let host = builder.config.build; let compiler = builder.compiler_for(builder.top_stage, host, host); - let mut cmd = command(builder.ensure(ErrorIndex { compiler })); + let mut cmd = command(builder.ensure(ErrorIndex { compiler }).tool_path); let mut dylib_paths = builder.rustc_lib_paths(compiler); dylib_paths.push(PathBuf::from(&builder.sysroot_target_libdir(compiler, compiler.host))); add_dylib_path(dylib_paths, &mut cmd); @@ -561,16 +559,13 @@ impl ErrorIndex { } impl Step for ErrorIndex { - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path("src/tools/error_index_generator") } fn make_run(run: RunConfig<'_>) { - // Compile the error-index in the same stage as rustdoc to avoid - // recompiling rustdoc twice if we can. - // // NOTE: This `make_run` isn't used in normal situations, only if you // manually build the tool with `x.py build // src/tools/error-index-generator` which almost nobody does. @@ -580,20 +575,18 @@ impl Step for ErrorIndex { run.builder.ensure(ErrorIndex { compiler }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.compiler.host, - tool: "error_index_generator", - mode: Mode::ToolRustc, - path: "src/tools/error_index_generator", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - .tool_path + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.compiler.host, + tool: "error_index_generator", + mode: Mode::ToolRustc, + path: "src/tools/error_index_generator", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) } } @@ -604,7 +597,7 @@ pub struct RemoteTestServer { } impl Step for RemoteTestServer { - type Output = PathBuf; + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.path("src/tools/remote-test-server") @@ -617,20 +610,18 @@ impl Step for RemoteTestServer { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "remote-test-server", - mode: Mode::ToolStd, - path: "src/tools/remote-test-server", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - .tool_path + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "remote-test-server", + mode: Mode::ToolStd, + path: "src/tools/remote-test-server", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) } } @@ -642,7 +633,7 @@ pub struct Rustdoc { } impl Step for Rustdoc { - type Output = PathBuf; + type Output = ToolBuildResult; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -655,15 +646,21 @@ impl Step for Rustdoc { .ensure(Rustdoc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { let target_compiler = self.compiler; + let target = target_compiler.host; + if target_compiler.stage == 0 { if !target_compiler.is_snapshot(builder) { panic!("rustdoc in stage 0 must be snapshot rustdoc"); } - return builder.initial_rustdoc.clone(); + + return ToolBuildResult { + tool_path: builder.initial_rustdoc.clone(), + build_compiler: target_compiler, + target_compiler, + }; } - let target = target_compiler.host; let bin_rustdoc = || { let sysroot = builder.sysroot(target_compiler); @@ -693,7 +690,12 @@ impl Step for Rustdoc { let bin_rustdoc = bin_rustdoc(); builder.copy_link(&precompiled_rustdoc, &bin_rustdoc); - return bin_rustdoc; + + return ToolBuildResult { + tool_path: bin_rustdoc, + build_compiler: target_compiler, + target_compiler, + }; } } @@ -709,7 +711,7 @@ impl Step for Rustdoc { extra_features.push("jemalloc".to_string()); } - let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = + let ToolBuildResult { tool_path, build_compiler, target_compiler } = builder.ensure(ToolBuild { compiler: target_compiler, target, @@ -734,9 +736,9 @@ impl Step for Rustdoc { } let bin_rustdoc = bin_rustdoc(); builder.copy_link(&tool_path, &bin_rustdoc); - bin_rustdoc + ToolBuildResult { tool_path: bin_rustdoc, build_compiler, target_compiler } } else { - tool_path + ToolBuildResult { tool_path, build_compiler, target_compiler } } } } @@ -748,7 +750,7 @@ pub struct Cargo { } impl Step for Cargo { - type Output = PathBuf; + type Output = ToolBuildResult; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -764,22 +766,20 @@ impl Step for Cargo { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { builder.build.require_submodule("src/tools/cargo", None); - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "cargo", - mode: Mode::ToolRustc, - path: "src/tools/cargo", - source_type: SourceType::Submodule, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - .tool_path + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "cargo", + mode: Mode::ToolRustc, + path: "src/tools/cargo", + source_type: SourceType::Submodule, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) } } @@ -790,7 +790,7 @@ pub struct LldWrapper { } impl Step for LldWrapper { - type Output = (); + type Output = ToolBuildResult; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.never() @@ -805,26 +805,29 @@ impl Step for LldWrapper { fields(build_compiler = ?self.build_compiler, target_compiler = ?self.target_compiler), ), )] - fn run(self, builder: &Builder<'_>) { + + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { if builder.config.dry_run() { - return; + return ToolBuildResult { + tool_path: Default::default(), + build_compiler: self.build_compiler, + target_compiler: self.target_compiler, + }; } let target = self.target_compiler.host; - let executable = builder - .ensure(ToolBuild { - compiler: self.build_compiler, - target, - tool: "lld-wrapper", - mode: Mode::ToolStd, - path: "src/tools/lld-wrapper", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - .tool_path; + let tool_result = builder.ensure(ToolBuild { + compiler: self.build_compiler, + target, + tool: "lld-wrapper", + mode: Mode::ToolStd, + path: "src/tools/lld-wrapper", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }); let libdir_bin = builder.sysroot_target_bindir(self.target_compiler, target); t!(fs::create_dir_all(&libdir_bin)); @@ -838,8 +841,11 @@ impl Step for LldWrapper { t!(fs::create_dir_all(&self_contained_lld_dir)); for name in crate::LLD_FILE_NAMES { - builder.copy_link(&executable, &self_contained_lld_dir.join(exe(name, target))); + builder + .copy_link(&tool_result.tool_path, &self_contained_lld_dir.join(exe(name, target))); } + + tool_result } } @@ -854,7 +860,7 @@ impl RustAnalyzer { } impl Step for RustAnalyzer { - type Output = PathBuf; + type Output = ToolBuildResult; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -870,20 +876,18 @@ impl Step for RustAnalyzer { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer", - mode: Mode::ToolRustc, - path: "src/tools/rust-analyzer", - extra_features: vec!["in-rust-tree".to_owned()], - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - cargo_args: Vec::new(), - }) - .tool_path + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer", + mode: Mode::ToolRustc, + path: "src/tools/rust-analyzer", + extra_features: vec!["in-rust-tree".to_owned()], + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + cargo_args: Vec::new(), + }) } } @@ -894,7 +898,7 @@ pub struct RustAnalyzerProcMacroSrv { } impl Step for RustAnalyzerProcMacroSrv { - type Output = Option; + type Output = Option; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -916,28 +920,27 @@ impl Step for RustAnalyzerProcMacroSrv { }); } - fn run(self, builder: &Builder<'_>) -> Option { - let path = builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer-proc-macro-srv", - mode: Mode::ToolRustc, - path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", - extra_features: vec!["in-rust-tree".to_owned()], - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - cargo_args: Vec::new(), - }) - .tool_path; + fn run(self, builder: &Builder<'_>) -> Option { + let tool_result = builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer-proc-macro-srv", + mode: Mode::ToolRustc, + path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", + extra_features: vec!["in-rust-tree".to_owned()], + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + cargo_args: Vec::new(), + }); // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` // so that r-a can use it. let libexec_path = builder.sysroot(self.compiler).join("libexec"); t!(fs::create_dir_all(&libexec_path)); - builder.copy_link(&path, &libexec_path.join("rust-analyzer-proc-macro-srv")); + builder + .copy_link(&tool_result.tool_path, &libexec_path.join("rust-analyzer-proc-macro-srv")); - Some(path) + Some(tool_result) } } @@ -949,7 +952,7 @@ pub struct LlvmBitcodeLinker { } impl Step for LlvmBitcodeLinker { - type Output = PathBuf; + type Output = ToolBuildResult; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -971,31 +974,34 @@ impl Step for LlvmBitcodeLinker { feature = "tracing", instrument(level = "debug", name = "LlvmBitcodeLinker::run", skip_all) )] - fn run(self, builder: &Builder<'_>) -> PathBuf { - let ToolBuildResult { tool_path, build_compiler: _build_compiler, target_compiler } = - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "llvm-bitcode-linker", - mode: Mode::ToolRustc, - path: "src/tools/llvm-bitcode-linker", - source_type: SourceType::InTree, - extra_features: self.extra_features, - allow_features: "", - cargo_args: Vec::new(), - }); + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { + let tool_result = builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "llvm-bitcode-linker", + mode: Mode::ToolRustc, + path: "src/tools/llvm-bitcode-linker", + source_type: SourceType::InTree, + extra_features: self.extra_features, + allow_features: "", + cargo_args: Vec::new(), + }); - if target_compiler.stage > 0 { + if tool_result.target_compiler.stage > 0 { let bindir_self_contained = builder - .sysroot(target_compiler) + .sysroot(tool_result.target_compiler) .join(format!("lib/rustlib/{}/bin/self-contained", self.target.triple)); t!(fs::create_dir_all(&bindir_self_contained)); - let bin_destination = - bindir_self_contained.join(exe("llvm-bitcode-linker", target_compiler.host)); - builder.copy_link(&tool_path, &bin_destination); - bin_destination + let bin_destination = bindir_self_contained + .join(exe("llvm-bitcode-linker", tool_result.target_compiler.host)); + builder.copy_link(&tool_result.tool_path, &bin_destination); + ToolBuildResult { + tool_path: bin_destination, + build_compiler: tool_result.build_compiler, + target_compiler: tool_result.target_compiler, + } } else { - tool_path + tool_result } } } @@ -1214,7 +1220,7 @@ pub struct TestFloatParse { } impl Step for TestFloatParse { - type Output = (); + type Output = ToolBuildResult; const ONLY_HOSTS: bool = true; const DEFAULT: bool = false; @@ -1222,7 +1228,7 @@ impl Step for TestFloatParse { run.path("src/etc/test-float-parse") } - fn run(self, builder: &Builder<'_>) { + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { let bootstrap_host = builder.config.build; let compiler = builder.compiler(builder.top_stage, bootstrap_host); @@ -1236,7 +1242,7 @@ impl Step for TestFloatParse { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), - }); + }) } } diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index c94bdd0a24684..25fa10e081114 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -1392,7 +1392,7 @@ impl<'a> Builder<'a> { } pub fn rustdoc(&self, compiler: Compiler) -> PathBuf { - self.ensure(tool::Rustdoc { compiler }) + self.ensure(tool::Rustdoc { compiler }).tool_path } pub fn cargo_clippy_cmd(&self, run_compiler: Compiler) -> BootstrapCommand { From 60ade6c1d7c8bbac52d6d1b9ed6703582028f954 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 16:07:42 +0000 Subject: [PATCH 10/22] add test coverage for `tool::get_tool_rustc_compiler` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/builder/tests.rs | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index a828a844f8103..b6aa9e7c8446d 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -1085,3 +1085,33 @@ fn test_is_builder_target() { assert!(!builder.is_builder_target(target2)); } } + +#[test] +fn test_get_tool_rustc_compiler() { + let mut config = configure("build", &[], &[]); + config.download_rustc_commit = None; + let build = Build::new(config); + let builder = Builder::new(&build); + + let target_triple_1 = TargetSelection::from_user(TEST_TRIPLE_1); + + let compiler = Compiler { stage: 2, host: target_triple_1 }; + let expected = Compiler { stage: 1, host: target_triple_1 }; + let actual = tool::get_tool_rustc_compiler(&builder, compiler); + assert_eq!(expected, actual); + + let compiler = Compiler { stage: 1, host: target_triple_1 }; + let expected = Compiler { stage: 0, host: target_triple_1 }; + let actual = tool::get_tool_rustc_compiler(&builder, compiler); + assert_eq!(expected, actual); + + let mut config = configure("build", &[], &[]); + config.download_rustc_commit = Some("".to_owned()); + let build = Build::new(config); + let builder = Builder::new(&build); + + let compiler = Compiler { stage: 1, host: target_triple_1 }; + let expected = Compiler { stage: 1, host: target_triple_1 }; + let actual = tool::get_tool_rustc_compiler(&builder, compiler); + assert_eq!(expected, actual); +} From 26f588bba4e2b2309628ecc1322b6d254253eae7 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 16:28:05 +0000 Subject: [PATCH 11/22] fix cargo tests Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/test.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index c7e9675faa2a1..db2f433bfe15e 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -300,7 +300,9 @@ impl Step for Cargo { fn run(self, builder: &Builder<'_>) { let compiler = builder.compiler(self.stage, self.host); - builder.ensure(tool::Cargo { compiler, target: self.host }); + let cargo = builder.ensure(tool::Cargo { compiler, target: self.host }); + let compiler = cargo.build_compiler; + let cargo = tool::prepare_tool_cargo( builder, compiler, From 5408aaea598793f0803af59f66dfd31376b6c661 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 18 Feb 2025 20:21:50 +0300 Subject: [PATCH 12/22] fix rust-analyzer tests Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/test.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index db2f433bfe15e..8d53ebdca19ef 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -369,6 +369,7 @@ impl Step for RustAnalyzer { let stage = self.stage; let host = self.host; let compiler = builder.compiler(stage, host); + let compiler = tool::get_tool_rustc_compiler(builder, compiler); // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite, // but we do need the standard library to be present. From 26d6ce76a7b38059146a277d15c007b0eeab834e Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 19 Feb 2025 09:03:35 +0300 Subject: [PATCH 13/22] add rustc-dev doc about bootstrap tools Signed-off-by: onur-ozkan --- src/doc/rustc-dev-guide/src/SUMMARY.md | 1 + .../writing-tools-in-bootstrap.md | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 src/doc/rustc-dev-guide/src/building/bootstrapping/writing-tools-in-bootstrap.md diff --git a/src/doc/rustc-dev-guide/src/SUMMARY.md b/src/doc/rustc-dev-guide/src/SUMMARY.md index 91c4aeacbd749..106db508ebbc7 100644 --- a/src/doc/rustc-dev-guide/src/SUMMARY.md +++ b/src/doc/rustc-dev-guide/src/SUMMARY.md @@ -75,6 +75,7 @@ - [Prologue](./building/bootstrapping/intro.md) - [What Bootstrapping does](./building/bootstrapping/what-bootstrapping-does.md) - [How Bootstrap does it](./building/bootstrapping/how-bootstrap-does-it.md) +- [Writing tools in Bootstrap](./building/bootstrapping/writing-tools-in-bootstrap.md) - [Debugging bootstrap](./building/bootstrapping/debugging-bootstrap.md) # High-level Compiler Architecture diff --git a/src/doc/rustc-dev-guide/src/building/bootstrapping/writing-tools-in-bootstrap.md b/src/doc/rustc-dev-guide/src/building/bootstrapping/writing-tools-in-bootstrap.md new file mode 100644 index 0000000000000..6046d5b133d7d --- /dev/null +++ b/src/doc/rustc-dev-guide/src/building/bootstrapping/writing-tools-in-bootstrap.md @@ -0,0 +1,23 @@ +# Writing tools in Bootstrap + +There are three types of tools you can write in bootstrap: + +- **`Mode::ToolBootstrap`** + Use this for tools that don’t need anything from the in-tree compiler and can run with the stage0 `rustc`. + The output is placed in the "stage0-bootstrap-tools" directory. This mode is for general-purpose tools built + entirely with the stage0 compiler, including target libraries and only works for stage 0. + +- **`Mode::ToolStd`** + Use this for tools that rely on the locally built std. The output goes into the "stageN-tools" directory. + This mode is rarely used, mainly for `compiletest` which requires `libtest`. + +- **`Mode::ToolRustc`** + Use this for tools that depend on both the locally built `rustc` and the target `std`. This is more complex than + the other modes because the tool must be built with the same compiler used for `rustc` and placed in the "stageN-tools" + directory. When you choose `Mode::ToolRustc`, `ToolBuild` implementation takes care of this automatically. + If you need to use the builder’s compiler for something specific, you can get it from `ToolBuildResult`, which is + returned by the tool's [`Step`]. + +Regardless of the tool type you must return `ToolBuildResult` from the tool’s [`Step`] implementation and use `ToolBuild` inside it. + +[`Step`]: https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html From b4a3b6484d4a6ca4d5b4561aa0113a8e2ab27695 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 19 Feb 2025 19:16:49 +0300 Subject: [PATCH 14/22] set `build.test-stage = 2` for `tools` profile Signed-off-by: onur-ozkan --- src/bootstrap/defaults/config.tools.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/defaults/config.tools.toml b/src/bootstrap/defaults/config.tools.toml index 64097320caba7..57c2706f60a56 100644 --- a/src/bootstrap/defaults/config.tools.toml +++ b/src/bootstrap/defaults/config.tools.toml @@ -8,6 +8,8 @@ incremental = true download-rustc = "if-unchanged" [build] +# cargo and clippy tests don't pass on stage 1 +test-stage = 2 # Document with the in-tree rustdoc by default, since `download-rustc` makes it quick to compile. doc-stage = 2 # Contributors working on tools will probably expect compiler docs to be generated, so they can figure out how to use the API. From 4b9b5d78963154386d7a0b41b4215bc13ad4dbca Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 19 Feb 2025 19:18:54 +0300 Subject: [PATCH 15/22] add change-entry for tools profile update Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/change_tracker.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index f215c3f6d0b39..8dfe0d3a35ef6 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -355,4 +355,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Info, summary: "It is now possible to configure `jemalloc` for each target", }, + ChangeInfo { + change_id: 137215, + severity: ChangeSeverity::Info, + summary: "Added `build.test-stage = 2` to 'tools' profile defaults", + }, ]; From 76063a683f2ef7530d07301127027488f1ecec8d Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 19 Feb 2025 19:25:17 +0300 Subject: [PATCH 16/22] print warning and help messages Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/test.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 8d53ebdca19ef..40f5259133a5d 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -298,6 +298,11 @@ impl Step for Cargo { /// Runs `cargo test` for `cargo` packaged with Rust. fn run(self, builder: &Builder<'_>) { + if self.stage < 2 { + eprintln!("WARNING: cargo tests on stage {} may not behave well.", self.stage); + eprintln!("HELP: consider using stage 2"); + } + let compiler = builder.compiler(self.stage, self.host); let cargo = builder.ensure(tool::Cargo { compiler, target: self.host }); @@ -743,6 +748,11 @@ impl Step for Clippy { let host = self.host; let compiler = builder.compiler(stage, host); + if stage < 2 { + eprintln!("WARNING: clippy tests on stage {stage} may not behave well."); + eprintln!("HELP: consider using stage 2"); + } + let tool_result = builder.ensure(tool::Clippy { compiler, target: self.host }); let compiler = tool_result.build_compiler; let mut cargo = tool::prepare_tool_cargo( From ba1827a1ff840a4530c1b20a9f811c78bdfd05c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 21 Feb 2025 13:02:41 +0100 Subject: [PATCH 17/22] Create the `OptimizedDist` tool with a macro --- src/bootstrap/src/core/build_steps/tool.rs | 40 +--------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 39acb646dff4d..27cc49d3be7c8 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -445,51 +445,13 @@ bootstrap_tool!( WasmComponentLd, "src/tools/wasm-component-ld", "wasm-component-ld", is_unstable_tool = true, allow_features = "min_specialization"; UnicodeTableGenerator, "src/tools/unicode-table-generator", "unicode-table-generator"; FeaturesStatusDump, "src/tools/features-status-dump", "features-status-dump"; + OptimizedDist, "src/tools/opt-dist", "opt-dist", submodules = &["src/tools/rustc-perf"]; ); /// These are the submodules that are required for rustbook to work due to /// depending on mdbook plugins. pub static SUBMODULES_FOR_RUSTBOOK: &[&str] = &["src/doc/book", "src/doc/reference"]; -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct OptimizedDist { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for OptimizedDist { - type Output = ToolBuildResult; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/opt-dist") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(OptimizedDist { - compiler: run.builder.compiler(0, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> ToolBuildResult { - // We need to ensure the rustc-perf submodule is initialized when building opt-dist since - // the tool requires it to be in place to run. - builder.require_submodule("src/tools/rustc-perf", None); - - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "opt-dist", - mode: Mode::ToolBootstrap, - path: "src/tools/opt-dist", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - } -} - /// The [rustc-perf](https://github.com/rust-lang/rustc-perf) benchmark suite, which is added /// as a submodule at `src/tools/rustc-perf`. #[derive(Debug, Clone, Hash, PartialEq, Eq)] From e0a00d0acbbde4b3697076c9dc40eca3c4239e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 21 Feb 2025 13:20:32 +0100 Subject: [PATCH 18/22] Allow specifying that tools build a library, not a binary --- src/bootstrap/src/core/build_steps/tool.rs | 40 +++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 27cc49d3be7c8..e401a0bdfc842 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -34,6 +34,12 @@ pub enum SourceType { Submodule, } +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum ToolArtifactKind { + Binary, + Library, +} + #[derive(Debug, Clone, Hash, PartialEq, Eq)] struct ToolBuild { compiler: Compiler, @@ -47,6 +53,8 @@ struct ToolBuild { allow_features: &'static str, /// Additional arguments to pass to the `cargo` invocation. cargo_args: Vec, + /// Whether the tool builds a binary or a library. + artifact_kind: ToolArtifactKind, } impl Builder<'_> { @@ -79,7 +87,7 @@ impl Builder<'_> { /// for using this type as `type Output = ToolBuildResult;` #[derive(Clone)] pub struct ToolBuildResult { - /// Executable path of the corresponding tool that was built. + /// Artifact path of the corresponding tool that was built. pub tool_path: PathBuf, /// Compiler used to build the tool. For non-`ToolRustc` tools this is equal to `target_compiler`. /// For `ToolRustc` this is one stage before of the `target_compiler`. @@ -179,8 +187,14 @@ impl Step for ToolBuild { if tool == "tidy" { tool = "rust-tidy"; } - let tool_path = - copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool); + let tool_path = match self.artifact_kind { + ToolArtifactKind::Binary => { + copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool) + } + ToolArtifactKind::Library => builder + .cargo_out(self.compiler, self.mode, self.target) + .join(format!("lib{tool}.rlib")), + }; ToolBuildResult { tool_path, build_compiler: self.compiler, target_compiler } } @@ -330,6 +344,7 @@ macro_rules! bootstrap_tool { $(,is_unstable_tool = $unstable:expr)* $(,allow_features = $allow_features:expr)? $(,submodules = $submodules:expr)? + $(,artifact_kind = $artifact_kind:expr)? ; )+) => { #[derive(PartialEq, Eq, Clone)] @@ -389,6 +404,7 @@ macro_rules! bootstrap_tool { builder.require_submodule(submodule, None); } )* + builder.ensure(ToolBuild { compiler: self.compiler, target: self.target, @@ -407,7 +423,12 @@ macro_rules! bootstrap_tool { }, extra_features: vec![], allow_features: concat!($($allow_features)*), - cargo_args: vec![] + cargo_args: vec![], + artifact_kind: if false $(|| $artifact_kind == ToolArtifactKind::Library)* { + ToolArtifactKind::Library + } else { + ToolArtifactKind::Binary + } }) } } @@ -491,6 +512,7 @@ impl Step for RustcPerf { // Only build the collector package, which is used for benchmarking through // a CLI. cargo_args: vec!["-p".to_string(), "collector".to_string()], + artifact_kind: ToolArtifactKind::Binary, }; let res = builder.ensure(tool.clone()); // We also need to symlink the `rustc-fake` binary to the corresponding directory, @@ -548,6 +570,7 @@ impl Step for ErrorIndex { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }) } } @@ -583,6 +606,7 @@ impl Step for RemoteTestServer { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }) } } @@ -687,6 +711,7 @@ impl Step for Rustdoc { extra_features, allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }); // don't create a stage0-sysroot/bin directory. @@ -741,6 +766,7 @@ impl Step for Cargo { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }) } } @@ -789,6 +815,7 @@ impl Step for LldWrapper { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }); let libdir_bin = builder.sysroot_target_bindir(self.target_compiler, target); @@ -849,6 +876,7 @@ impl Step for RustAnalyzer { source_type: SourceType::InTree, allow_features: RustAnalyzer::ALLOW_FEATURES, cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }) } } @@ -893,6 +921,7 @@ impl Step for RustAnalyzerProcMacroSrv { source_type: SourceType::InTree, allow_features: RustAnalyzer::ALLOW_FEATURES, cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }); // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` @@ -947,6 +976,7 @@ impl Step for LlvmBitcodeLinker { extra_features: self.extra_features, allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }); if tool_result.target_compiler.stage > 0 { @@ -1126,6 +1156,7 @@ fn run_tool_build_step( source_type: SourceType::InTree, allow_features: "", cargo_args: vec![], + artifact_kind: ToolArtifactKind::Binary, }); // FIXME: This should just be an if-let-chain, but those are unstable. @@ -1204,6 +1235,7 @@ impl Step for TestFloatParse { extra_features: Vec::new(), allow_features: "", cargo_args: Vec::new(), + artifact_kind: ToolArtifactKind::Binary, }) } } From 46bfaa283c5c544b988122de045f3f688828499a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 21 Feb 2025 13:36:29 +0100 Subject: [PATCH 19/22] Implement `RunMakeSupport` tool using the `bootstrap_tool!` macro --- src/bootstrap/src/core/build_steps/test.rs | 45 ---------------------- src/bootstrap/src/core/build_steps/tool.rs | 1 + 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 40f5259133a5d..e6a462764fdb1 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1211,51 +1211,6 @@ macro_rules! test { }; } -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] -pub struct RunMakeSupport { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RunMakeSupport { - type Output = PathBuf; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RunMakeSupport { compiler, target: run.build_triple() }); - } - - /// Builds run-make-support and returns the path to the resulting rlib. - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(compile::Std::new(self.compiler, self.target)); - - let cargo = tool::prepare_tool_cargo( - builder, - self.compiler, - Mode::ToolStd, - self.target, - Kind::Build, - "src/tools/run-make-support", - SourceType::InTree, - &[], - ); - - cargo.into_cmd().run(builder); - - let lib_name = "librun_make_support.rlib"; - let lib = builder.tools_dir(self.compiler).join(lib_name); - - let cargo_out = builder.cargo_out(self.compiler, Mode::ToolStd, self.target).join(lib_name); - builder.copy_link(&cargo_out, &lib); - lib - } -} - /// Runs `cargo test` on the `src/tools/run-make-support` crate. /// That crate is used by run-make tests. #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index e401a0bdfc842..0149709f5c51b 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -467,6 +467,7 @@ bootstrap_tool!( UnicodeTableGenerator, "src/tools/unicode-table-generator", "unicode-table-generator"; FeaturesStatusDump, "src/tools/features-status-dump", "features-status-dump"; OptimizedDist, "src/tools/opt-dist", "opt-dist", submodules = &["src/tools/rustc-perf"]; + RunMakeSupport, "src/tools/run-make-support", "run_make_support", artifact_kind = ToolArtifactKind::Library; ); /// These are the submodules that are required for rustbook to work due to From dc8794800d08683dc01c8fb429d121c3c379790a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 21 Feb 2025 13:36:45 +0100 Subject: [PATCH 20/22] Implement `RunMake` test suite using the `test!` macro --- src/bootstrap/src/core/build_steps/test.rs | 38 +++------------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index e6a462764fdb1..424de6e5564b4 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1362,40 +1362,7 @@ test!(Pretty { only_hosts: true, }); -/// Special-handling is needed for `run-make`, so don't use `test!` for defining `RunMake` -/// tests. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RunMake { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RunMake { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path("tests/run-make") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RunMakeSupport { compiler, target: run.build_triple() }); - run.builder.ensure(RunMake { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: "run-make", - suite: "run-make", - path: "tests/run-make", - compare_mode: None, - }); - } -} +test!(RunMake { path: "tests/run-make", mode: "run-make", suite: "run-make", default: true }); test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly", default: true }); @@ -1638,6 +1605,9 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the host: target, }); } + if suite == "run-make" { + builder.tool_exe(Tool::RunMakeSupport); + } // Also provide `rust_test_helpers` for the host. builder.ensure(TestHelpers { target: compiler.host }); From 1f0071946b8b4b6b0fbb3cf50eb2707773b10949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 21 Feb 2025 14:51:36 +0100 Subject: [PATCH 21/22] Compile run-make recipes using the stage0 compiler --- src/tools/compiletest/src/common.rs | 5 + src/tools/compiletest/src/lib.rs | 7 ++ src/tools/compiletest/src/runtest/run_make.rs | 116 ++++++------------ .../src/external_deps/rustc.rs | 8 +- .../src/external_deps/rustdoc.rs | 22 +--- src/tools/run-make-support/src/run.rs | 19 +-- src/tools/run-make-support/src/util.rs | 6 +- 7 files changed, 67 insertions(+), 116 deletions(-) diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index 1614c35cb1ce4..d52431182bc0b 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -205,6 +205,11 @@ pub struct Config { /// The jsondoclint executable. pub jsondoclint_path: Option, + /// Path to a target directory where the run-make-support rlib library was built. + /// We have to pass a path to the directory, because compiletest also resolves its transitive + /// dependencies from there. + pub run_make_support_dir: Option, + /// The LLVM `FileCheck` binary path. pub llvm_filecheck: Option, diff --git a/src/tools/compiletest/src/lib.rs b/src/tools/compiletest/src/lib.rs index d0a83cab9cd99..f416e7350c1f8 100644 --- a/src/tools/compiletest/src/lib.rs +++ b/src/tools/compiletest/src/lib.rs @@ -59,6 +59,12 @@ pub fn parse_config(args: Vec) -> Config { .reqopt("", "python", "path to python to use for doc tests", "PATH") .optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH") .optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH") + .optopt( + "", + "run-make-support-dir", + "path to a directory with built run-make-support library used for run-make tests", + "PATH", + ) .optopt("", "run-clang-based-tests-with", "path to Clang executable", "PATH") .optopt("", "llvm-filecheck", "path to LLVM's FileCheck binary", "DIR") .reqopt("", "src-base", "directory to scan for test files", "PATH") @@ -311,6 +317,7 @@ pub fn parse_config(args: Vec) -> Config { python: matches.opt_str("python").unwrap(), jsondocck_path: matches.opt_str("jsondocck-path"), jsondoclint_path: matches.opt_str("jsondoclint-path"), + run_make_support_dir: matches.opt_str("run-make-support-dir").map(PathBuf::from), run_clang_based_tests_with: matches.opt_str("run-clang-based-tests-with"), llvm_filecheck: matches.opt_str("llvm-filecheck").map(PathBuf::from), llvm_bin_dir: matches.opt_str("llvm-bin-dir").map(PathBuf::from), diff --git a/src/tools/compiletest/src/runtest/run_make.rs b/src/tools/compiletest/src/runtest/run_make.rs index 16c46fc13390f..321c9c8d06f8b 100644 --- a/src/tools/compiletest/src/runtest/run_make.rs +++ b/src/tools/compiletest/src/runtest/run_make.rs @@ -175,7 +175,8 @@ impl TestCx<'_> { fn run_rmake_v2_test(&self) { // For `run-make` V2, we need to perform 2 steps to build and run a `run-make` V2 recipe // (`rmake.rs`) to run the actual tests. The support library is already built as a tool rust - // library and is available under `build/$TARGET/stageN-tools-bin/librun_make_support.rlib`. + // library and is available under + // `build/$TARGET/stage0-bootstrap-tools/$HOST/release/librun_make_support.rlib`. // // 1. We need to build the recipe `rmake.rs` as a binary and link in the `run_make_support` // library. @@ -249,25 +250,21 @@ impl TestCx<'_> { // // ``` // build// - // ├── stageN-tools-bin/ - // │ └── librun_make_support.rlib // <- support rlib itself - // ├── stageN-tools/ - // │ ├── release/deps/ // <- deps of deps - // │ └── /release/deps/ // <- deps + // ├── stage0-bootstrap-tools/ + // │ ├── /release/librun_make_support.rlib // <- support rlib itself + // │ ├── /release/deps/ // <- deps + // │ └── release/deps/ // <- deps of deps // ``` // // FIXME(jieyouxu): there almost certainly is a better way to do this (specifically how the - // support lib and its deps are organized, can't we copy them to the tools-bin dir as - // well?), but this seems to work for now. + // support lib and its deps are organized), but this seems to work for now. - let stage_number = self.config.stage; + let tools_bin = build_root.join("stage0-bootstrap-tools"); + let support_host_path = tools_bin.join(&self.config.host).join("release"); + let support_lib_path = support_host_path.join("librun_make_support.rlib"); - let stage_tools_bin = build_root.join(format!("stage{stage_number}-tools-bin")); - let support_lib_path = stage_tools_bin.join("librun_make_support.rlib"); - - let stage_tools = build_root.join(format!("stage{stage_number}-tools")); - let support_lib_deps = stage_tools.join(&self.config.host).join("release").join("deps"); - let support_lib_deps_deps = stage_tools.join("release").join("deps"); + let support_lib_deps = support_host_path.join("deps"); + let support_lib_deps_deps = tools_bin.join("release").join("deps"); // To compile the recipe with rustc, we need to provide suitable dynamic library search // paths to rustc. This includes both: @@ -278,12 +275,6 @@ impl TestCx<'_> { let base_dylib_search_paths = Vec::from_iter(env::split_paths(&env::var(dylib_env_var()).unwrap())); - let host_dylib_search_paths = { - let mut paths = vec![self.config.compile_lib_path.clone()]; - paths.extend(base_dylib_search_paths.iter().cloned()); - paths - }; - // Calculate the paths of the recipe binary. As previously discussed, this is placed at // `/` with `bin_name` being `rmake` or `rmake.exe` depending on // platform. @@ -293,7 +284,13 @@ impl TestCx<'_> { p }; - let mut rustc = Command::new(&self.config.rustc_path); + // run-make-support and run-make tests are compiled using the bootstrap compiler + let bootstrap_rustc = { + let mut p = build_root.join("stage0").join("bin").join("rustc"); + p.set_extension(env::consts::EXE_EXTENSION); + p + }; + let mut rustc = Command::new(bootstrap_rustc); rustc .arg("-o") .arg(&recipe_bin) @@ -307,34 +304,14 @@ impl TestCx<'_> { .arg(format!("run_make_support={}", &support_lib_path.to_string_lossy())) .arg("--edition=2021") .arg(&self.testpaths.file.join("rmake.rs")) - .arg("-Cprefer-dynamic") - // Provide necessary library search paths for rustc. - .env(dylib_env_var(), &env::join_paths(host_dylib_search_paths).unwrap()); + .arg("-Cprefer-dynamic"); // In test code we want to be very pedantic about values being silently discarded that are // annotated with `#[must_use]`. rustc.arg("-Dunused_must_use"); - // > `cg_clif` uses `COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0` for running the rustc - // > test suite. With the introduction of rmake.rs this broke. `librun_make_support.rlib` is - // > compiled using the bootstrap rustc wrapper which sets `--sysroot - // > build/aarch64-unknown-linux-gnu/stage0-sysroot`, but then compiletest will compile - // > `rmake.rs` using the sysroot of the bootstrap compiler causing it to not find the - // > `libstd.rlib` against which `librun_make_support.rlib` is compiled. - // - // The gist here is that we have to pass the proper stage0 sysroot if we want - // - // ``` - // $ COMPILETEST_FORCE_STAGE0=1 ./x test run-make --stage 0 - // ``` - // - // to work correctly. - // - // See for more background. - let stage0_sysroot = build_root.join("stage0-sysroot"); - if std::env::var_os("COMPILETEST_FORCE_STAGE0").is_some() { - rustc.arg("--sysroot").arg(&stage0_sysroot); - } + // TODO: test COMPILETEST_FORCE_STAGE0=1 ./x test tests/run-make --stage 0 + // TODO: test ./x test tests/run-make // Now run rustc to build the recipe. let res = self.run_command_to_procres(&mut rustc); @@ -345,35 +322,22 @@ impl TestCx<'_> { // To actually run the recipe, we have to provide the recipe with a bunch of information // provided through env vars. - // Compute stage-specific standard library paths. - let stage_std_path = build_root.join(format!("stage{stage_number}")).join("lib"); - // Compute dynamic library search paths for recipes. + // These dylib directories are needed to **execute the recipe**. let recipe_dylib_search_paths = { let mut paths = base_dylib_search_paths.clone(); - - // For stage 0, we need to explicitly include the stage0-sysroot libstd dylib. - // See . - if std::env::var_os("COMPILETEST_FORCE_STAGE0").is_some() { - paths.push( - stage0_sysroot.join("lib").join("rustlib").join(&self.config.host).join("lib"), - ); - } - - paths.push(support_lib_path.parent().unwrap().to_path_buf()); - paths.push(stage_std_path.join("rustlib").join(&self.config.host).join("lib")); - paths - }; - - // Compute runtime library search paths for recipes. This is target-specific. - let target_runtime_dylib_search_paths = { - let mut paths = vec![rmake_out_dir.clone()]; - paths.extend(base_dylib_search_paths.iter().cloned()); + // This is the bootstrap stdlib required to run the rmake recipe itself + paths.push( + build_root + .join("stage0") + .join("lib") + .join("rustlib") + .join(&self.config.host) + .join("lib"), + ); paths }; - // FIXME(jieyouxu): please rename `TARGET_RPATH_ENV`, `HOST_RPATH_DIR` and - // `TARGET_RPATH_DIR`, it is **extremely** confusing! let mut cmd = Command::new(&recipe_bin); cmd.current_dir(&rmake_out_dir) .stdout(Stdio::piped()) @@ -382,9 +346,14 @@ impl TestCx<'_> { // example, this could be `LD_LIBRARY_PATH` on some linux distros but `PATH` on Windows. .env("LD_LIB_PATH_ENVVAR", dylib_env_var()) // Provide the dylib search paths. + // This is required to run the **recipe** itself. .env(dylib_env_var(), &env::join_paths(recipe_dylib_search_paths).unwrap()) - // Provide runtime dylib search paths. - .env("TARGET_RPATH_ENV", &env::join_paths(target_runtime_dylib_search_paths).unwrap()) + // Provide the directory to libraries that are needed to run the *compiler* invoked + // by the recipe. + .env("HOST_RUSTC_DYLIB_PATH", &self.config.compile_lib_path) + // Provide the directory to libraries that might be needed to run binaries created + // by a compiler invoked by the recipe. + .env("TARGET_EXE_DYLIB_PATH", &self.config.run_lib_path) // Provide the target. .env("TARGET", &self.config.target) // Some tests unfortunately still need Python, so provide path to a Python interpreter. @@ -396,13 +365,6 @@ impl TestCx<'_> { .env("BUILD_ROOT", &build_root) // Provide path to stage-corresponding rustc. .env("RUSTC", &self.config.rustc_path) - // Provide the directory to libraries that are needed to run the *compiler*. This is not - // to be confused with `TARGET_RPATH_ENV` or `TARGET_RPATH_DIR`. This is needed if the - // recipe wants to invoke rustc. - .env("HOST_RPATH_DIR", &self.config.compile_lib_path) - // Provide the directory to libraries that might be needed to run compiled binaries - // (further compiled by the recipe!). - .env("TARGET_RPATH_DIR", &self.config.run_lib_path) // Provide which LLVM components are available (e.g. which LLVM components are provided // through a specific CI runner). .env("LLVM_COMPONENTS", &self.config.llvm_components); diff --git a/src/tools/run-make-support/src/external_deps/rustc.rs b/src/tools/run-make-support/src/external_deps/rustc.rs index fd4a20278addd..7dc99f7ec1e0b 100644 --- a/src/tools/run-make-support/src/external_deps/rustc.rs +++ b/src/tools/run-make-support/src/external_deps/rustc.rs @@ -5,7 +5,7 @@ use std::str::FromStr as _; use crate::command::Command; use crate::env::env_var; use crate::path_helpers::cwd; -use crate::util::set_host_rpath; +use crate::util::set_host_compiler_dylib_path; use crate::{is_aix, is_darwin, is_msvc, is_windows, uname}; /// Construct a new `rustc` invocation. This will automatically set the library @@ -15,8 +15,8 @@ pub fn rustc() -> Rustc { Rustc::new() } -/// Construct a plain `rustc` invocation with no flags set. Note that [`set_host_rpath`] -/// still presets the environment variable `HOST_RPATH_DIR` by default. +/// Construct a plain `rustc` invocation with no flags set. Note that [`set_host_compiler_dylib_path`] +/// still presets the environment variable `HOST_RUSTC_DYLIB_PATH` by default. #[track_caller] pub fn bare_rustc() -> Rustc { Rustc::bare() @@ -44,7 +44,7 @@ pub fn rustc_path() -> String { #[track_caller] fn setup_common() -> Command { let mut cmd = Command::new(rustc_path()); - set_host_rpath(&mut cmd); + set_host_compiler_dylib_path(&mut cmd); cmd } diff --git a/src/tools/run-make-support/src/external_deps/rustdoc.rs b/src/tools/run-make-support/src/external_deps/rustdoc.rs index 3c0e9c82f0bbc..e907809b12f03 100644 --- a/src/tools/run-make-support/src/external_deps/rustdoc.rs +++ b/src/tools/run-make-support/src/external_deps/rustdoc.rs @@ -3,15 +3,9 @@ use std::path::Path; use crate::command::Command; use crate::env::{env_var, env_var_os}; -use crate::util::set_host_rpath; +use crate::util::set_host_compiler_dylib_path; -/// Construct a plain `rustdoc` invocation with no flags set. -#[track_caller] -pub fn bare_rustdoc() -> Rustdoc { - Rustdoc::bare() -} - -/// Construct a new `rustdoc` invocation with `-L $(TARGET_RPATH_DIR)` set. +/// Construct a new `rustdoc` invocation. #[track_caller] pub fn rustdoc() -> Rustdoc { Rustdoc::new() @@ -29,23 +23,15 @@ crate::macros::impl_common_helpers!(Rustdoc); fn setup_common() -> Command { let rustdoc = env_var("RUSTDOC"); let mut cmd = Command::new(rustdoc); - set_host_rpath(&mut cmd); + set_host_compiler_dylib_path(&mut cmd); cmd } impl Rustdoc { /// Construct a bare `rustdoc` invocation. #[track_caller] - pub fn bare() -> Self { - let cmd = setup_common(); - Self { cmd } - } - - /// Construct a `rustdoc` invocation with `-L $(TARGET_RPATH_DIR)` set. - #[track_caller] pub fn new() -> Self { - let mut cmd = setup_common(); - cmd.arg("-L").arg(env_var_os("TARGET_RPATH_DIR")); + let cmd = setup_common(); Self { cmd } } diff --git a/src/tools/run-make-support/src/run.rs b/src/tools/run-make-support/src/run.rs index 3eeba6fd5263d..7812863ccc2c8 100644 --- a/src/tools/run-make-support/src/run.rs +++ b/src/tools/run-make-support/src/run.rs @@ -1,10 +1,10 @@ use std::ffi::OsStr; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::{env, panic}; use crate::command::{Command, CompletedProcess}; -use crate::util::{handle_failed_output, set_host_rpath}; -use crate::{cwd, env_var, is_windows}; +use crate::util::handle_failed_output; +use crate::{cwd, env_var}; #[track_caller] fn run_common(name: &str, args: Option<&[&str]>) -> Command { @@ -18,10 +18,11 @@ fn run_common(name: &str, args: Option<&[&str]>) -> Command { cmd.arg(arg); } } + cmd.env(&ld_lib_path_envvar, { let mut paths = vec![]; paths.push(cwd()); - for p in env::split_paths(&env_var("TARGET_RPATH_ENV")) { + for p in env::split_paths(&env_var("TARGET_EXE_DYLIB_PATH")) { paths.push(p.to_path_buf()); } for p in env::split_paths(&env_var(&ld_lib_path_envvar)) { @@ -31,15 +32,6 @@ fn run_common(name: &str, args: Option<&[&str]>) -> Command { }); cmd.env("LC_ALL", "C"); // force english locale - if is_windows() { - let mut paths = vec![]; - for p in env::split_paths(&std::env::var("PATH").unwrap_or(String::new())) { - paths.push(p.to_path_buf()); - } - paths.push(Path::new(&env_var("TARGET_RPATH_DIR")).to_path_buf()); - cmd.env("PATH", env::join_paths(paths.iter()).unwrap()); - } - cmd } @@ -84,7 +76,6 @@ pub fn run_fail(name: &str) -> CompletedProcess { #[track_caller] pub fn cmd>(program: S) -> Command { let mut command = Command::new(program); - set_host_rpath(&mut command); command.env("LC_ALL", "C"); // force english locale command } diff --git a/src/tools/run-make-support/src/util.rs b/src/tools/run-make-support/src/util.rs index 703e3ad1c6cd2..af01758447b93 100644 --- a/src/tools/run-make-support/src/util.rs +++ b/src/tools/run-make-support/src/util.rs @@ -24,13 +24,13 @@ pub(crate) fn handle_failed_output( std::process::exit(1) } -/// Set the runtime library path as needed for running the host rustc/rustdoc/etc. -pub(crate) fn set_host_rpath(cmd: &mut Command) { +/// Set the runtime library paths as needed for running the host compilers (rustc/rustdoc/etc). +pub(crate) fn set_host_compiler_dylib_path(cmd: &mut Command) { let ld_lib_path_envvar = env_var("LD_LIB_PATH_ENVVAR"); cmd.env(&ld_lib_path_envvar, { let mut paths = vec![]; paths.push(cwd()); - paths.push(PathBuf::from(env_var("HOST_RPATH_DIR"))); + paths.push(PathBuf::from(env_var("HOST_RUSTC_DYLIB_PATH"))); for p in std::env::split_paths(&env_var(&ld_lib_path_envvar)) { paths.push(p.to_path_buf()); } From 3f27d09320f4485790fa8ad44bd9a05275db4bc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 21 Feb 2025 14:51:41 +0100 Subject: [PATCH 22/22] Fix run-make tests --- src/tools/run-make-support/src/external_deps/rustdoc.rs | 2 +- src/tools/run-make-support/src/lib.rs | 2 +- tests/run-make/broken-pipe-no-ice/rmake.rs | 2 +- tests/run-make/rustdoc-default-output/rmake.rs | 4 ++-- tests/run-make/version-verbose-commit-hash/rmake.rs | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/tools/run-make-support/src/external_deps/rustdoc.rs b/src/tools/run-make-support/src/external_deps/rustdoc.rs index e907809b12f03..61351ace12c9b 100644 --- a/src/tools/run-make-support/src/external_deps/rustdoc.rs +++ b/src/tools/run-make-support/src/external_deps/rustdoc.rs @@ -2,7 +2,7 @@ use std::ffi::OsStr; use std::path::Path; use crate::command::Command; -use crate::env::{env_var, env_var_os}; +use crate::env::env_var; use crate::util::set_host_compiler_dylib_path; /// Construct a new `rustdoc` invocation. diff --git a/src/tools/run-make-support/src/lib.rs b/src/tools/run-make-support/src/lib.rs index 7e63ab3159ace..d40ec9c4116e7 100644 --- a/src/tools/run-make-support/src/lib.rs +++ b/src/tools/run-make-support/src/lib.rs @@ -67,7 +67,7 @@ pub use llvm::{ }; pub use python::python_command; pub use rustc::{aux_build, bare_rustc, rustc, rustc_path, Rustc}; -pub use rustdoc::{bare_rustdoc, rustdoc, Rustdoc}; +pub use rustdoc::{rustdoc, Rustdoc}; /// [`diff`][mod@diff] is implemented in terms of the [similar] library. /// diff --git a/tests/run-make/broken-pipe-no-ice/rmake.rs b/tests/run-make/broken-pipe-no-ice/rmake.rs index 54d13b62f4a0e..378c3289cb7bc 100644 --- a/tests/run-make/broken-pipe-no-ice/rmake.rs +++ b/tests/run-make/broken-pipe-no-ice/rmake.rs @@ -25,7 +25,7 @@ enum Binary { } fn check_broken_pipe_handled_gracefully(bin: Binary, mut cmd: Command) { - let (reader, writer) = std::io::pipe().unwrap(); + let (reader, writer) = std::pipe::pipe().unwrap(); drop(reader); // close read-end cmd.stdout(writer).stderr(Stdio::piped()); diff --git a/tests/run-make/rustdoc-default-output/rmake.rs b/tests/run-make/rustdoc-default-output/rmake.rs index 06720445a35ce..5f9c501e5286b 100644 --- a/tests/run-make/rustdoc-default-output/rmake.rs +++ b/tests/run-make/rustdoc-default-output/rmake.rs @@ -3,10 +3,10 @@ // ensures the output of rustdoc's help menu is as expected. // See https://github.com/rust-lang/rust/issues/88756 -use run_make_support::{bare_rustdoc, diff}; +use run_make_support::{diff, rustdoc}; fn main() { - let out = bare_rustdoc().run().stdout_utf8(); + let out = rustdoc().run().stdout_utf8(); diff() .expected_file("output-default.stdout") .actual_text("actual", out) diff --git a/tests/run-make/version-verbose-commit-hash/rmake.rs b/tests/run-make/version-verbose-commit-hash/rmake.rs index 733c0e2cdb14f..ae8dda38ab418 100644 --- a/tests/run-make/version-verbose-commit-hash/rmake.rs +++ b/tests/run-make/version-verbose-commit-hash/rmake.rs @@ -5,13 +5,13 @@ //@ needs-git-hash -use run_make_support::{bare_rustc, bare_rustdoc, regex}; +use run_make_support::{bare_rustc, regex, rustdoc}; fn main() { let out_rustc = bare_rustc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase(); let out_rustdoc = - bare_rustdoc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase(); + rustdoc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase(); let re = regex::Regex::new(r#"commit-hash: [0-9a-f]{40}\ncommit-date: [0-9]{4}-[0-9]{2}-[0-9]{2}"#) .unwrap();