Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[perf] don't use temporary files for archive builder #104191

Closed
wants to merge 2 commits into from
Closed

[perf] don't use temporary files for archive builder #104191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
19c3e8f
[perf]
belovdv Nov 9, 2022
8b66395
increase add_buffer using
belovdv Nov 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions compiler/rustc_codegen_llvm/src/back/archive.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ use rustc_session::Session;
pub struct LlvmArchiveBuilder<'a> {
sess: &'a Session,
additions: Vec<Addition>,

buffers: Vec<(String, Vec<u8>)>,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this PR goes towards merging, then this is better added as a new variant to enum Addition instead of a separate vector.

}

enum Addition {
Expand Down Expand Up @@ -139,6 +141,10 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
Ok(())
}

fn add_buffer(&mut self, buffer: Vec<u8>, name: &str) {
self.buffers.push((name.to_string(), buffer))
}

/// Adds an arbitrary file to this archive
fn add_file(&mut self, file: &Path) {
let name = file.file_name().unwrap().to_str().unwrap();
Expand All @@ -160,7 +166,7 @@ pub struct LlvmArchiveBuilderBuilder;

impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> {
Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() })
Box::new(LlvmArchiveBuilder { sess, additions: Vec::new(), buffers: Vec::new() })
}

fn create_dll_import_lib(
Expand Down Expand Up @@ -320,6 +326,10 @@ impl<'a> LlvmArchiveBuilder<'a> {
let mut additions = mem::take(&mut self.additions);
let mut strings = Vec::new();
let mut members = Vec::new();
let addition_buffers = mem::take(&mut self.buffers);
let mut member_buffers = Vec::new();
let mut string_buffers = Vec::new();
let mut buffer_buffers = Vec::new();

let dst = CString::new(output.to_str().unwrap())?;

Expand All @@ -329,7 +339,7 @@ impl<'a> LlvmArchiveBuilder<'a> {
Addition::File { path, name_in_archive } => {
let path = CString::new(path.to_str().unwrap())?;
let name = CString::new(name_in_archive.clone())?;
members.push(llvm::LLVMRustArchiveMemberNew(
members.push(llvm::LLVMRustArchiveMemberNewFile(
path.as_ptr(),
name.as_ptr(),
None,
Expand Down Expand Up @@ -357,7 +367,7 @@ impl<'a> LlvmArchiveBuilder<'a> {
let child_name =
Path::new(child_name).file_name().unwrap().to_str().unwrap();
let name = CString::new(child_name)?;
let m = llvm::LLVMRustArchiveMemberNew(
let m = llvm::LLVMRustArchiveMemberNewFile(
ptr::null(),
name.as_ptr(),
Some(child.raw),
Expand All @@ -368,11 +378,24 @@ impl<'a> LlvmArchiveBuilder<'a> {
}
}
}
for addition in addition_buffers {
let name = CString::new(addition.0)?;
let buffer = addition.1;
member_buffers.push(llvm::LLVMRustArchiveMemberNewBuffer(
name.as_ptr(),
buffer.as_ptr() as *const i8,
buffer.len(),
));
string_buffers.push(name);
buffer_buffers.push(buffer);
}

let r = llvm::LLVMRustWriteArchive(
dst.as_ptr(),
members.len() as libc::size_t,
members.as_ptr() as *const &_,
member_buffers.len() as libc::size_t,
member_buffers.as_ptr() as *const &_,
true,
kind,
);
Expand Down
12 changes: 11 additions & 1 deletion compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -664,6 +664,8 @@ extern "C" {
#[repr(C)]
pub struct RustArchiveMember<'a>(InvariantOpaque<'a>);
#[repr(C)]
pub struct RustArchiveMemberBuffer<'a>(InvariantOpaque<'a>);
#[repr(C)]
pub struct OperandBundleDef<'a>(InvariantOpaque<'a>);
#[repr(C)]
pub struct Linker<'a>(InvariantOpaque<'a>);
Expand Down Expand Up @@ -2360,15 +2362,23 @@ extern "C" {
Dst: *const c_char,
NumMembers: size_t,
Members: *const &RustArchiveMember<'_>,
NumMemberBuffers: size_t,
Members: *const &RustArchiveMemberBuffer<'_>,
WriteSymbtab: bool,
Kind: ArchiveKind,
) -> LLVMRustResult;
pub fn LLVMRustArchiveMemberNew<'a>(
pub fn LLVMRustArchiveMemberNewFile<'a>(
Filename: *const c_char,
Name: *const c_char,
Child: Option<&ArchiveChild<'a>>,
) -> &'a mut RustArchiveMember<'a>;
pub fn LLVMRustArchiveMemberNewBuffer<'a>(
Name: *const c_char,
BufferStart: *const c_char,
BufferSize: size_t,
) -> &'a mut RustArchiveMember<'a>;
pub fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>);
pub fn LLVMRustArchiveMemberBufferFree<'a>(Member: &'a mut RustArchiveMemberBuffer<'a>);

pub fn LLVMRustWriteImportLibrary(
ImportName: *const c_char,
Expand Down
4 changes: 4 additions & 0 deletions compiler/rustc_codegen_ssa/src/back/archive.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ pub trait ArchiveBuilderBuilder {
pub trait ArchiveBuilder<'a> {
fn add_file(&mut self, path: &Path);

fn add_buffer(&mut self, _buffer: Vec<u8>, _name: &str) {
unimplemented!();
}

fn add_archive(
&mut self,
archive: &Path,
Expand Down
15 changes: 8 additions & 7 deletions compiler/rustc_codegen_ssa/src/back/link.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use rustc_errors::{ErrorGuaranteed, Handler};
use rustc_fs_util::fix_windows_verbatim_for_gcc;
use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
use rustc_metadata::find_native_static_library;
use rustc_metadata::fs::{emit_wrapper_file, METADATA_FILENAME};
use rustc_metadata::fs::METADATA_FILENAME;
use rustc_middle::middle::dependency_format::Linkage;
use rustc_middle::middle::exported_symbols::SymbolExportKind;
use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Lto, Strip};
Expand Down Expand Up @@ -293,15 +293,15 @@ fn link_rlib<'a>(
RlibFlavor::Normal => {
let (metadata, metadata_position) =
create_wrapper_file(sess, b".rmeta".to_vec(), codegen_results.metadata.raw_data());
let metadata = emit_wrapper_file(sess, &metadata, tmpdir, METADATA_FILENAME);
match metadata_position {
MetadataPosition::First => {
// Most of the time metadata in rlib files is wrapped in a "dummy" object
// file for the target platform so the rlib can be processed entirely by
// normal linkers for the platform. Sometimes this is not possible however.
// If it is possible however, placing the metadata object first improves
// performance of getting metadata from rlibs.
ab.add_file(&metadata);
ab.add_buffer(metadata, METADATA_FILENAME);
// ab.add_file(&metadata);
None
}
MetadataPosition::Last => Some(metadata),
Expand Down Expand Up @@ -386,8 +386,8 @@ fn link_rlib<'a>(
let src = read(lib_path)
.map_err(|e| sess.emit_fatal(errors::ReadFileError { message: e }))?;
let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
packed_bundled_libs.push(wrapper_file);
// let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
packed_bundled_libs.push((data, filename));
continue;
}
ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|error| {
Expand Down Expand Up @@ -438,13 +438,14 @@ fn link_rlib<'a>(
//
// Basically, all this means is that this code should not move above the
// code above.
ab.add_file(&trailing_metadata);
ab.add_buffer(trailing_metadata, METADATA_FILENAME);
// ab.add_file(&trailing_metadata);
}

// Add all bundled static native library dependencies.
// Archives added to the end of .rlib archive, see comment above for the reason.
for lib in packed_bundled_libs {
ab.add_file(&lib)
ab.add_buffer(lib.0, lib.1.as_str())
}

return Ok(ab);
Expand Down
30 changes: 29 additions & 1 deletion compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ struct RustArchiveMember {
~RustArchiveMember() {}
};

struct RustArchiveMemberBuffer {
MemoryBufferRef Buffer;

RustArchiveMemberBuffer() {}
~RustArchiveMemberBuffer() {}
};

struct RustArchiveIterator {
bool First;
Archive::child_iterator Cur;
Expand Down Expand Up @@ -58,6 +65,7 @@ static Archive::Kind fromRust(LLVMRustArchiveKind Kind) {

typedef OwningBinary<Archive> *LLVMRustArchiveRef;
typedef RustArchiveMember *LLVMRustArchiveMemberRef;
typedef RustArchiveMemberBuffer *LLVMRustArchiveMemberBufferRef;
typedef Archive::Child *LLVMRustArchiveChildRef;
typedef Archive::Child const *LLVMRustArchiveChildConstRef;
typedef RustArchiveIterator *LLVMRustArchiveIteratorRef;
Expand Down Expand Up @@ -155,7 +163,7 @@ LLVMRustArchiveChildName(LLVMRustArchiveChildConstRef Child, size_t *Size) {
}

extern "C" LLVMRustArchiveMemberRef
LLVMRustArchiveMemberNew(char *Filename, char *Name,
LLVMRustArchiveMemberNewFile(char *Filename, char *Name,
LLVMRustArchiveChildRef Child) {
RustArchiveMember *Member = new RustArchiveMember;
Member->Filename = Filename;
Expand All @@ -165,13 +173,26 @@ LLVMRustArchiveMemberNew(char *Filename, char *Name,
return Member;
}

extern "C" LLVMRustArchiveMemberBufferRef
LLVMRustArchiveMemberNewBuffer(const char* Name, const char* BufferStart, size_t BufferSize) {
RustArchiveMemberBuffer *Member = new RustArchiveMemberBuffer;
Member->Buffer = MemoryBufferRef(StringRef(BufferStart, BufferSize), StringRef(Name));
return Member;
}

extern "C" void LLVMRustArchiveMemberFree(LLVMRustArchiveMemberRef Member) {
delete Member;
}

extern "C" void LLVMRustArchiveMemberBufferFree(LLVMRustArchiveMemberBufferRef Member) {
delete Member;
}

extern "C" LLVMRustResult
LLVMRustWriteArchive(char *Dst, size_t NumMembers,
const LLVMRustArchiveMemberRef *NewMembers,
size_t NumMemberBuffers,
const LLVMRustArchiveMemberBufferRef *NewMemberBuffers,
bool WriteSymbtab, LLVMRustArchiveKind RustKind) {

std::vector<NewArchiveMember> Members;
Expand All @@ -181,6 +202,7 @@ LLVMRustWriteArchive(char *Dst, size_t NumMembers,
auto Member = NewMembers[I];
assert(Member->Name);
if (Member->Filename) {
// file
Expected<NewArchiveMember> MOrErr =
NewArchiveMember::getFile(Member->Filename, true);
if (!MOrErr) {
Expand All @@ -190,6 +212,7 @@ LLVMRustWriteArchive(char *Dst, size_t NumMembers,
MOrErr->MemberName = sys::path::filename(MOrErr->MemberName);
Members.push_back(std::move(*MOrErr));
} else {
// archive
Expected<NewArchiveMember> MOrErr =
NewArchiveMember::getOldMember(Member->Child, true);
if (!MOrErr) {
Expand All @@ -199,6 +222,11 @@ LLVMRustWriteArchive(char *Dst, size_t NumMembers,
Members.push_back(std::move(*MOrErr));
}
}
for (size_t I = 0; I < NumMemberBuffers; I++) {
auto Member = NewMemberBuffers[I];
auto M = NewArchiveMember(Member->Buffer);
Members.push_back(std::move(M));
}

auto Result = writeArchive(Dst, Members, WriteSymbtab, Kind, true, false);
if (!Result)
Expand Down