Auto merge of #53245 - michaelwoerister:thinlto-rust-llvm, r=<try>

[experimental]: Build LLVM with ThinLTO enabled (2nd attempt)

This is #51207 revived. This time, I'd like to run actual performance tests to see if it improves compile times.

config.toml.example

@@ -21,6 +21,12 @@
 # Indicates whether the LLVM build is a Release or Debug build
 #optimize = true
 
+# Indicates whether LLVM should be built with ThinLTO. Note that this will
+# only succeed if you use clang, lld, llvm-ar, and llvm-ranlib in your C/C++
+# toolchain (see the `cc`, `cxx`, `linker`, `ar`, and `ranlib` options below).
+# More info at: https://clang.llvm.org/docs/ThinLTO.html#clang-bootstrap
+#thin-lto = false
+
 # Indicates whether an LLVM Release build should include debug info
 #release-debuginfo = false
 
@@ -384,6 +390,10 @@
 # Note: an absolute path should be used, otherwise LLVM build will break.
 #ar = "ar"
 
+# Ranlib to be used to assemble static libraries compiled from C/C++ code.
+# Note: an absolute path should be used, otherwise LLVM build will break.
+#ranlib = "ranlib"
+
 # Linker to be used to link Rust code. Note that the
 # default value is platform specific, and if not specified it may also depend on
 # what platform is crossing to what platform.

src/bootstrap/builder.rs

@@ -826,7 +826,7 @@ impl<'a> Builder<'a> {
         if let Some(ref error_format) = self.config.rustc_error_format {
             cargo.env("RUSTC_ERROR_FORMAT", error_format);
         }
-        if cmd != "build" && cmd != "check" && want_rustdoc {
+        if cmd != "build" && cmd != "check" && cmd != "rustc" && want_rustdoc {
             cargo.env("RUSTDOC_LIBDIR", self.sysroot_libdir(compiler, self.config.build));
         }
 
@@ -987,7 +987,7 @@ impl<'a> Builder<'a> {
             }
         }
 
-        if cmd == "build"
+        if (cmd == "build" || cmd == "rustc")
             && mode == Mode::Std
             && self.config.extended
             && compiler.is_final_stage(self)

src/bootstrap/check.rs

@@ -50,6 +50,7 @@ impl Step for Std {
         println!("Checking std artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libstd_stamp(builder, compiler, target),
                   true);
 
@@ -98,6 +99,7 @@ impl Step for Rustc {
         println!("Checking compiler artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &librustc_stamp(builder, compiler, target),
                   true);
 
@@ -149,6 +151,7 @@ impl Step for CodegenBackend {
         let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &codegen_backend_stamp(builder, compiler, target, backend),
                   true);
     }
@@ -187,6 +190,7 @@ impl Step for Test {
         println!("Checking test artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libtest_stamp(builder, compiler, target),
                   true);
 
@@ -236,6 +240,7 @@ impl Step for Rustdoc {
         println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &rustdoc_stamp(builder, compiler, target),
                   true);
 

src/bootstrap/compile.rs

@@ -117,6 +117,7 @@ impl Step for Std {
                 &compiler.host, target));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libstd_stamp(builder, compiler, target),
                   false);
 
@@ -396,6 +397,7 @@ impl Step for Test {
                 &compiler.host, target));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libtest_stamp(builder, compiler, target),
                   false);
 
@@ -529,6 +531,7 @@ impl Step for Rustc {
                  compiler.stage, &compiler.host, target));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &librustc_stamp(builder, compiler, target),
                   false);
 
@@ -673,18 +676,47 @@ impl Step for CodegenBackend {
         let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
         builder.clear_if_dirty(&out_dir, &librustc_stamp(builder, compiler, target));
 
-        let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "build");
+        let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "rustc");
         cargo.arg("--manifest-path")
             .arg(builder.src.join("src/librustc_codegen_llvm/Cargo.toml"));
         rustc_cargo_env(builder, &mut cargo);
 
         let features = build_codegen_backend(&builder, &mut cargo, &compiler, target, backend);
 
+        let mut cargo_tails_args = vec![];
+
+        if builder.config.llvm_thin_lto {
+            cargo_tails_args.push("--".to_string());
+
+            let num_jobs = builder.jobs();
+
+            if !target.contains("msvc") {
+                // Here we assume that the linker is clang. If it's not, there'll
+                // be linker errors.
+                cargo_tails_args.push("-Clink-arg=-fuse-ld=lld".to_string());
+                cargo_tails_args.push("-Clink-arg=-flto=thin".to_string());
+
+                if builder.config.llvm_optimize {
+                    cargo_tails_args.push("-Clink-arg=-O2".to_string());
+                }
+
+                // Let's make LLD respect the `-j` option.
+                let num_jobs_arg = format!("-Clink-arg=-Wl,--thinlto-jobs={}", num_jobs);
+                cargo_tails_args.push(num_jobs_arg);
+            } else {
+                // Here we assume that the linker is lld-link.exe. lld-link.exe
+                // does not need the extra arguments except for num_jobs
+                let num_jobs_arg = format!("-Clink-arg=/opt:lldltojobs={}", num_jobs);
+                cargo_tails_args.push(num_jobs_arg);
+            }
+        }
+
         let tmp_stamp = out_dir.join(".tmp.stamp");
 
         let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
         let files = run_cargo(builder,
                               cargo.arg("--features").arg(features),
+                              cargo_tails_args,
                               &tmp_stamp,
                               false);
         if builder.config.dry_run {
@@ -1045,7 +1077,11 @@ fn stderr_isatty() -> bool {
     }
 }
 
-pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check: bool)
+pub fn run_cargo(builder: &Builder,
+                 cargo: &mut Command,
+                 tail_args: Vec<String>,
+                 stamp: &Path,
+                 is_check: bool)
     -> Vec<PathBuf>
 {
     if builder.config.dry_run {
@@ -1066,7 +1102,7 @@ pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check:
     // files we need to probe for later.
     let mut deps = Vec::new();
     let mut toplevel = Vec::new();
-    let ok = stream_cargo(builder, cargo, &mut |msg| {
+    let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| {
         let filenames = match msg {
             CargoMessage::CompilerArtifact { filenames, .. } => filenames,
             _ => return,
@@ -1191,6 +1227,7 @@ pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check:
 pub fn stream_cargo(
     builder: &Builder,
     cargo: &mut Command,
+    tail_args: Vec<String>,
     cb: &mut dyn FnMut(CargoMessage),
 ) -> bool {
     if builder.config.dry_run {
@@ -1210,6 +1247,10 @@ pub fn stream_cargo(
         cargo.env("RUSTC_COLOR", "1");
     }
 
+    for arg in tail_args {
+        cargo.arg(arg);
+    }
+
     builder.verbose(&format!("running: {:?}", cargo));
     let mut child = match cargo.spawn() {
         Ok(child) => child,

src/bootstrap/config.rs

@@ -77,6 +77,7 @@ pub struct Config {
     pub llvm_enabled: bool,
     pub llvm_assertions: bool,
     pub llvm_optimize: bool,
+    pub llvm_thin_lto: bool,
     pub llvm_release_debuginfo: bool,
     pub llvm_version_check: bool,
     pub llvm_static_stdcpp: bool,
@@ -162,6 +163,7 @@ pub struct Target {
     pub cc: Option<PathBuf>,
     pub cxx: Option<PathBuf>,
     pub ar: Option<PathBuf>,
+    pub ranlib: Option<PathBuf>,
     pub linker: Option<PathBuf>,
     pub ndk: Option<PathBuf>,
     pub crt_static: Option<bool>,
@@ -245,6 +247,7 @@ struct Llvm {
     ninja: Option<bool>,
     assertions: Option<bool>,
     optimize: Option<bool>,
+    thin_lto: Option<bool>,
     release_debuginfo: Option<bool>,
     version_check: Option<bool>,
     static_libstdcpp: Option<bool>,
@@ -325,6 +328,7 @@ struct TomlTarget {
     cc: Option<String>,
     cxx: Option<String>,
     ar: Option<String>,
+    ranlib: Option<String>,
     linker: Option<String>,
     android_ndk: Option<String>,
     crt_static: Option<bool>,
@@ -501,6 +505,7 @@ impl Config {
             set(&mut config.llvm_enabled, llvm.enabled);
             llvm_assertions = llvm.assertions;
             set(&mut config.llvm_optimize, llvm.optimize);
+            set(&mut config.llvm_thin_lto, llvm.thin_lto);
             set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
             set(&mut config.llvm_version_check, llvm.version_check);
             set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
@@ -578,6 +583,7 @@ impl Config {
                 target.cc = cfg.cc.clone().map(PathBuf::from);
                 target.cxx = cfg.cxx.clone().map(PathBuf::from);
                 target.ar = cfg.ar.clone().map(PathBuf::from);
+                target.ranlib = cfg.ranlib.clone().map(PathBuf::from);
                 target.linker = cfg.linker.clone().map(PathBuf::from);
                 target.crt_static = cfg.crt_static.clone();
                 target.musl_root = cfg.musl_root.clone().map(PathBuf::from);

src/bootstrap/lib.rs

@@ -276,6 +276,7 @@ pub struct Build {
     cc: HashMap<Interned<String>, cc::Tool>,
     cxx: HashMap<Interned<String>, cc::Tool>,
     ar: HashMap<Interned<String>, PathBuf>,
+    ranlib: HashMap<Interned<String>, PathBuf>,
     // Misc
     crates: HashMap<Interned<String>, Crate>,
     is_sudo: bool,
@@ -401,6 +402,7 @@ impl Build {
             cc: HashMap::new(),
             cxx: HashMap::new(),
             ar: HashMap::new(),
+            ranlib: HashMap::new(),
             crates: HashMap::new(),
             lldb_version: None,
             lldb_python_dir: None,
@@ -767,6 +769,11 @@ impl Build {
         self.ar.get(&target).map(|p| &**p)
     }
 
+    /// Returns the path to the `ranlib` utility for the target specified.
+    fn ranlib(&self, target: Interned<String>) -> Option<&Path> {
+        self.ranlib.get(&target).map(|p| &**p)
+    }
+
     /// Returns the path to the C++ compiler for the target specified.
     fn cxx(&self, target: Interned<String>) -> Result<&Path, String> {
         match self.cxx.get(&target) {

src/bootstrap/native.rs

@@ -154,6 +154,11 @@ impl Step for Llvm {
            .define("LLVM_TARGET_ARCH", target.split('-').next().unwrap())
            .define("LLVM_DEFAULT_TARGET_TRIPLE", target);
 
+        if builder.config.llvm_thin_lto {
+            cfg.define("LLVM_ENABLE_LTO", "Thin")
+               .define("LLVM_ENABLE_LLD", "ON");
+        }
+
         // By default, LLVM will automatically find OCaml and, if it finds it,
         // install the LLVM bindings in LLVM_OCAML_INSTALL_PATH, which defaults
         // to /usr/bin/ocaml.
@@ -366,6 +371,14 @@ fn configure_cmake(builder: &Builder,
         }
     }
 
+    if let Some(ranlib) = builder.ranlib(target) {
+        if ranlib.is_absolute() {
+            // LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it
+            // tries to resolve this path in the LLVM build directory.
+            cfg.define("CMAKE_RANLIB", sanitize_cc(ranlib));
+        }
+    }
+
     if env::var_os("SCCACHE_ERROR_LOG").is_some() {
         cfg.env("RUST_LOG", "sccache=warn");
     }

src/bootstrap/tool.rs

@@ -136,7 +136,7 @@ impl Step for ToolBuild {
         let _folder = builder.fold_output(|| format!("stage{}-{}", compiler.stage, tool));
         builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
         let mut duplicates = Vec::new();
-        let is_expected = compile::stream_cargo(builder, &mut cargo, &mut |msg| {
+        let is_expected = compile::stream_cargo(builder, &mut cargo, vec![], &mut |msg| {
             // Only care about big things like the RLS/Cargo for now
             match tool {
                 | "rls"

src/ci/docker/dist-x86_64-linux/Dockerfile

@@ -93,7 +93,10 @@ ENV RUST_CONFIGURE_ARGS \
       --enable-sanitizers \
       --enable-profiler \
       --enable-compiler-docs \
-      --set target.x86_64-unknown-linux-gnu.linker=clang
+      --set target.x86_64-unknown-linux-gnu.linker=clang \
+      --set target.x86_64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \
+      --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
+      --set llvm.thin-lto=true
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
 ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang
 

src/ci/docker/dist-x86_64-linux/build-clang.sh

@@ -30,6 +30,12 @@ curl https://releases.llvm.org/$LLVM/cfe-$LLVM.src.tar.xz | \
   xz -d | \
   tar xf - -C tools/clang --strip-components=1
 
+mkdir -p tools/lld
+
+curl https://releases.llvm.org/$LLVM/lld-$LLVM.src.tar.xz | \
+  xz -d | \
+  tar xf - -C tools/lld --strip-components=1
+
 mkdir ../clang-build
 cd ../clang-build