From 43e8ac27d9bf645b66a15f762be6969e9fe16285 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 12 Feb 2018 08:38:46 -0800 Subject: [PATCH] rustc: Persist LLVM's `Linker` in Fat LTO This commit updates our Fat LTO logic to tweak our custom wrapper around LLVM's "link modules" functionality. Previously whenever the `LLVMRustLinkInExternalBitcode` function was called it would call LLVM's `Linker::linkModules` wrapper. Internally this would crate an instance of a `Linker` which internally creates an instance of an `IRMover`. Unfortunately for us the creation of `IRMover` is somewhat O(n) with the input module. This means that every time we linked a module it was O(n) with respect to the entire module we had built up! Now the modules we build up during LTO are quite large, so this quickly started creating an O(n^2) problem for us! Discovered in #48025 it turns out this has always been a problem and we just haven't noticed it. It became particularly worse recently though due to most libraries having 16x more object files than they previously did (1 -> 16). This commit fixes this performance issue by preserving the `Linker` instance across all links into the main LLVM module. This means we only create one `IRMover` and allows LTO to progress much speedier. From the `cargo-cache` project in #48025 a **full build** locally when from 5m15s to 2m24s. Looking at the timing logs each object file was linked in in single-digit millisecond rather than hundreds, clearly being a nice improvement! Closes #48025 --- src/librustc_llvm/build.rs | 1 + src/librustc_llvm/ffi.rs | 10 ++++- src/librustc_trans/back/lto.rs | 40 +++++++++++++++---- src/rustllvm/Linker.cpp | 72 ++++++++++++++++++++++++++++++++++ src/rustllvm/RustWrapper.cpp | 40 ------------------- 5 files changed, 114 insertions(+), 49 deletions(-) create mode 100644 src/rustllvm/Linker.cpp diff --git a/src/librustc_llvm/build.rs b/src/librustc_llvm/build.rs index 49b93f3c7d6a2..54e3f544acb68 100644 --- a/src/librustc_llvm/build.rs +++ b/src/librustc_llvm/build.rs @@ -155,6 +155,7 @@ fn main() { cfg.file("../rustllvm/PassWrapper.cpp") .file("../rustllvm/RustWrapper.cpp") .file("../rustllvm/ArchiveWrapper.cpp") + .file("../rustllvm/Linker.cpp") .cpp(true) .cpp_link_stdlib(None) // we handle this below .compile("rustllvm"); diff --git a/src/librustc_llvm/ffi.rs b/src/librustc_llvm/ffi.rs index 99e43a2ddf98d..e71bef512cf06 100644 --- a/src/librustc_llvm/ffi.rs +++ b/src/librustc_llvm/ffi.rs @@ -444,6 +444,9 @@ pub type RustArchiveMemberRef = *mut RustArchiveMember_opaque; #[allow(missing_copy_implementations)] pub enum OperandBundleDef_opaque {} pub type OperandBundleDefRef = *mut OperandBundleDef_opaque; +#[allow(missing_copy_implementations)] +pub enum Linker_opaque {} +pub type LinkerRef = *mut Linker_opaque; pub type DiagnosticHandler = unsafe extern "C" fn(DiagnosticInfoRef, *mut c_void); pub type InlineAsmDiagHandler = unsafe extern "C" fn(SMDiagnosticRef, *const c_void, c_uint); @@ -1608,7 +1611,6 @@ extern "C" { pub fn LLVMRustPrintPasses(); pub fn LLVMRustSetNormalizedTarget(M: ModuleRef, triple: *const c_char); pub fn LLVMRustAddAlwaysInlinePass(P: PassManagerBuilderRef, AddLifetimes: bool); - pub fn LLVMRustLinkInExternalBitcode(M: ModuleRef, bc: *const c_char, len: size_t) -> bool; pub fn LLVMRustRunRestrictionPass(M: ModuleRef, syms: *const *const c_char, len: size_t); pub fn LLVMRustMarkAllFunctionsNounwind(M: ModuleRef); @@ -1724,4 +1726,10 @@ extern "C" { CU2: *mut *mut c_void); pub fn LLVMRustThinLTOPatchDICompileUnit(M: ModuleRef, CU: *mut c_void); pub fn LLVMRustThinLTORemoveAvailableExternally(M: ModuleRef); + + pub fn LLVMRustLinkerNew(M: ModuleRef) -> LinkerRef; + pub fn LLVMRustLinkerAdd(linker: LinkerRef, + bytecode: *const c_char, + bytecode_len: usize) -> bool; + pub fn LLVMRustLinkerFree(linker: LinkerRef); } diff --git a/src/librustc_trans/back/lto.rs b/src/librustc_trans/back/lto.rs index 9ff5bcf7a33ca..a33270380196f 100644 --- a/src/librustc_trans/back/lto.rs +++ b/src/librustc_trans/back/lto.rs @@ -247,22 +247,20 @@ fn fat_lto(cgcx: &CodegenContext, // know much about the memory management here so we err on the side of being // save and persist everything with the original module. let mut serialized_bitcode = Vec::new(); + let mut linker = Linker::new(llmod); for (bc_decoded, name) in serialized_modules { info!("linking {:?}", name); - time(cgcx.time_passes, &format!("ll link {:?}", name), || unsafe { + time(cgcx.time_passes, &format!("ll link {:?}", name), || { let data = bc_decoded.data(); - if llvm::LLVMRustLinkInExternalBitcode(llmod, - data.as_ptr() as *const libc::c_char, - data.len() as libc::size_t) { - Ok(()) - } else { + linker.add(&data).map_err(|()| { let msg = format!("failed to load bc of {:?}", name); - Err(write::llvm_err(&diag_handler, msg)) - } + write::llvm_err(&diag_handler, msg) + }) })?; timeline.record(&format!("link {:?}", name)); serialized_bitcode.push(bc_decoded); } + drop(linker); cgcx.save_temp_bitcode(&module, "lto.input"); // Internalize everything that *isn't* in our whitelist to help strip out @@ -289,6 +287,32 @@ fn fat_lto(cgcx: &CodegenContext, }]) } +struct Linker(llvm::LinkerRef); + +impl Linker { + fn new(llmod: ModuleRef) -> Linker { + unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) } + } + + fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { + unsafe { + if llvm::LLVMRustLinkerAdd(self.0, + bytecode.as_ptr() as *const libc::c_char, + bytecode.len()) { + Ok(()) + } else { + Err(()) + } + } + } +} + +impl Drop for Linker { + fn drop(&mut self) { + unsafe { llvm::LLVMRustLinkerFree(self.0); } + } +} + /// Prepare "thin" LTO to get run on these modules. /// /// The general structure of ThinLTO is quite different from the structure of diff --git a/src/rustllvm/Linker.cpp b/src/rustllvm/Linker.cpp new file mode 100644 index 0000000000000..534e4b910902e --- /dev/null +++ b/src/rustllvm/Linker.cpp @@ -0,0 +1,72 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#include "llvm/Linker/Linker.h" + +#include "rustllvm.h" + +using namespace llvm; + +struct RustLinker { + Linker L; + LLVMContext &Ctx; + + RustLinker(Module &M) : + L(M), + Ctx(M.getContext()) + {} +}; + +extern "C" RustLinker* +LLVMRustLinkerNew(LLVMModuleRef DstRef) { + Module *Dst = unwrap(DstRef); + + auto Ret = llvm::make_unique(*Dst); + return Ret.release(); +} + +extern "C" void +LLVMRustLinkerFree(RustLinker *L) { + delete L; +} + +extern "C" bool +LLVMRustLinkerAdd(RustLinker *L, char *BC, size_t Len) { + std::unique_ptr Buf = + MemoryBuffer::getMemBufferCopy(StringRef(BC, Len)); + +#if LLVM_VERSION_GE(4, 0) + Expected> SrcOrError = + llvm::getLazyBitcodeModule(Buf->getMemBufferRef(), L->Ctx); + if (!SrcOrError) { + LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); + return false; + } + + auto Src = std::move(*SrcOrError); +#else + ErrorOr> Src = + llvm::getLazyBitcodeModule(std::move(Buf), L->Ctx); + if (!Src) { + LLVMRustSetLastError(Src.getError().message().c_str()); + return false; + } +#endif + +#if LLVM_VERSION_GE(4, 0) + if (L->L.linkInModule(std::move(Src))) { +#else + if (L->L.linkInModule(std::move(Src.get()))) { +#endif + LLVMRustSetLastError(""); + return false; + } + return true; +} diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp index 4dfc4029d75dc..27d5496f57628 100644 --- a/src/rustllvm/RustWrapper.cpp +++ b/src/rustllvm/RustWrapper.cpp @@ -916,46 +916,6 @@ extern "C" void LLVMRustWriteValueToString(LLVMValueRef V, } } -extern "C" bool LLVMRustLinkInExternalBitcode(LLVMModuleRef DstRef, char *BC, - size_t Len) { - Module *Dst = unwrap(DstRef); - - std::unique_ptr Buf = - MemoryBuffer::getMemBufferCopy(StringRef(BC, Len)); - -#if LLVM_VERSION_GE(4, 0) - Expected> SrcOrError = - llvm::getLazyBitcodeModule(Buf->getMemBufferRef(), Dst->getContext()); - if (!SrcOrError) { - LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); - return false; - } - - auto Src = std::move(*SrcOrError); -#else - ErrorOr> Src = - llvm::getLazyBitcodeModule(std::move(Buf), Dst->getContext()); - if (!Src) { - LLVMRustSetLastError(Src.getError().message().c_str()); - return false; - } -#endif - - std::string Err; - - raw_string_ostream Stream(Err); - DiagnosticPrinterRawOStream DP(Stream); -#if LLVM_VERSION_GE(4, 0) - if (Linker::linkModules(*Dst, std::move(Src))) { -#else - if (Linker::linkModules(*Dst, std::move(Src.get()))) { -#endif - LLVMRustSetLastError(Err.c_str()); - return false; - } - return true; -} - // Note that the two following functions look quite similar to the // LLVMGetSectionName function. Sadly, it appears that this function only // returns a char* pointer, which isn't guaranteed to be null-terminated. The