Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Emit metadata files earlier #60385

Merged
merged 3 commits into from
May 2, 2019
Merged

Emit metadata files earlier #60385

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
faf5eac
Move metadata encoding earlier.
nnethercote Apr 26, 2019
3da5d4a
Inline and remove `link_binary_output`.
nnethercote Apr 30, 2019
38dffeb
Move metadata writing earlier.
nnethercote Apr 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2802,6 +2802,7 @@ dependencies = [
"rustc-rayon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_allocator 0.0.0",
"rustc_borrowck 0.0.0",
"rustc_codegen_ssa 0.0.0",
"rustc_codegen_utils 0.0.0",
"rustc_data_structures 0.0.0",
"rustc_errors 0.0.0",
Expand All @@ -2821,6 +2822,7 @@ dependencies = [
"syntax 0.0.0",
"syntax_ext 0.0.0",
"syntax_pos 0.0.0",
"tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
Expand Down
40 changes: 4 additions & 36 deletions src/librustc_codegen_llvm/base.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ use rustc::mir::mono::{Linkage, Visibility, Stats};
use rustc::middle::cstore::{EncodedMetadata};
use rustc::ty::TyCtxt;
use rustc::middle::exported_symbols;
use rustc::session::config::{self, DebugInfo};
use rustc::session::config::DebugInfo;
use rustc_codegen_ssa::mono_item::MonoItemExt;
use rustc_data_structures::small_c_str::SmallCStr;

Expand All @@ -42,47 +42,16 @@ use rustc::hir::CodegenFnAttrs;

use crate::value::Value;


pub fn write_metadata<'a, 'gcx>(
pub fn write_compressed_metadata<'a, 'gcx>(
tcx: TyCtxt<'a, 'gcx, 'gcx>,
metadata: &EncodedMetadata,
llvm_module: &mut ModuleLlvm
) -> EncodedMetadata {
) {
use std::io::Write;
use flate2::Compression;
use flate2::write::DeflateEncoder;

let (metadata_llcx, metadata_llmod) = (&*llvm_module.llcx, llvm_module.llmod());

#[derive(PartialEq, Eq, PartialOrd, Ord)]
enum MetadataKind {
None,
Uncompressed,
Compressed
}

let kind = tcx.sess.crate_types.borrow().iter().map(|ty| {
match *ty {
config::CrateType::Executable |
config::CrateType::Staticlib |
config::CrateType::Cdylib => MetadataKind::None,

config::CrateType::Rlib => MetadataKind::Uncompressed,

config::CrateType::Dylib |
config::CrateType::ProcMacro => MetadataKind::Compressed,
}
}).max().unwrap_or(MetadataKind::None);

if kind == MetadataKind::None {
return EncodedMetadata::new();
}

let metadata = tcx.encode_metadata();
if kind == MetadataKind::Uncompressed {
return metadata;
}

assert!(kind == MetadataKind::Compressed);
let mut compressed = tcx.metadata_encoding_version();
DeflateEncoder::new(&mut compressed, Compression::fast())
.write_all(&metadata.raw_data).unwrap();
Expand All @@ -107,7 +76,6 @@ pub fn write_metadata<'a, 'gcx>(
let directive = CString::new(directive).unwrap();
llvm::LLVMSetModuleInlineAsm(metadata_llmod, directive.as_ptr())
}
return metadata;
}

pub struct ValueIter<'ll> {
Expand Down
14 changes: 9 additions & 5 deletions src/librustc_codegen_llvm/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,13 @@ impl ExtraBackendMethods for LlvmCodegenBackend {
ModuleLlvm::new_metadata(tcx, mod_name)
}

fn write_metadata<'b, 'gcx>(
fn write_compressed_metadata<'b, 'gcx>(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this name is misleading, it should probably be {inject,embed,include,etc.}_compressed_metadata.

&self,
tcx: TyCtxt<'b, 'gcx, 'gcx>,
metadata: &mut ModuleLlvm
) -> EncodedMetadata {
base::write_metadata(tcx, metadata)
metadata: &EncodedMetadata,
llvm_module: &mut ModuleLlvm
) {
base::write_compressed_metadata(tcx, metadata, llvm_module)
}
fn codegen_allocator<'b, 'gcx>(
&self,
Expand Down Expand Up @@ -289,9 +290,12 @@ impl CodegenBackend for LlvmCodegenBackend {
fn codegen_crate<'b, 'tcx>(
&self,
tcx: TyCtxt<'b, 'tcx, 'tcx>,
metadata: EncodedMetadata,
need_metadata_module: bool,
rx: mpsc::Receiver<Box<dyn Any + Send>>
) -> Box<dyn Any> {
box rustc_codegen_ssa::base::codegen_crate(LlvmCodegenBackend(()), tcx, rx)
box rustc_codegen_ssa::base::codegen_crate(
LlvmCodegenBackend(()), tcx, metadata, need_metadata_module, rx)
}

fn join_codegen_and_link(
Expand Down
124 changes: 44 additions & 80 deletions src/librustc_codegen_ssa/back/link.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use rustc::session::config::{
};
use rustc::session::search_paths::PathKind;
use rustc::middle::dependency_format::Linkage;
use rustc::middle::cstore::{LibSource, NativeLibrary, NativeLibraryKind};
use rustc::middle::cstore::{EncodedMetadata, LibSource, NativeLibrary, NativeLibraryKind};
use rustc::util::common::{time, time_ext};
use rustc::hir::def_id::CrateNum;
use rustc_data_structures::fx::FxHashSet;
Expand Down Expand Up @@ -50,9 +50,9 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
outputs: &OutputFilenames,
crate_name: &str,
target_cpu: &str) {
let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata);
for &crate_type in sess.crate_types.borrow().iter() {
// Ignore executable crates if we have -Z no-codegen, as they will error.
let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata);
if (sess.opts.debugging_opts.no_codegen || !sess.opts.output_types.should_codegen()) &&
!output_metadata &&
crate_type == config::CrateType::Executable {
Expand All @@ -63,12 +63,43 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
bug!("invalid output type `{:?}` for target os `{}`",
crate_type, sess.opts.target_triple);
}
link_binary_output::<B>(sess,
codegen_results,
crate_type,
outputs,
crate_name,
target_cpu);

for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) {
check_file_is_writeable(obj, sess);
}

let tmpdir = TempFileBuilder::new().prefix("rustc").tempdir().unwrap_or_else(|err|
sess.fatal(&format!("couldn't create a temp dir: {}", err)));

if outputs.outputs.should_codegen() {
let out_filename = out_filename(sess, crate_type, outputs, crate_name);
match crate_type {
config::CrateType::Rlib => {
link_rlib::<B>(sess,
codegen_results,
RlibFlavor::Normal,
&out_filename,
&tmpdir).build();
}
config::CrateType::Staticlib => {
link_staticlib::<B>(sess, codegen_results, &out_filename, &tmpdir);
}
_ => {
link_natively::<B>(
sess,
crate_type,
&out_filename,
codegen_results,
tmpdir.path(),
target_cpu,
);
}
}
}

if sess.opts.cg.save_temps {
let _ = tmpdir.into_path();
}
}

// Remove the temporary object file and metadata if we aren't saving temps
Expand All @@ -85,7 +116,7 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
if let Some(ref obj) = metadata_module.object {
remove(sess, obj);
}
}
}
if let Some(ref allocator_module) = codegen_results.allocator_module {
if let Some(ref obj) = allocator_module.object {
remove(sess, obj);
Expand All @@ -97,73 +128,6 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
}
}

fn link_binary_output<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
codegen_results: &CodegenResults,
crate_type: config::CrateType,
outputs: &OutputFilenames,
crate_name: &str,
target_cpu: &str) {
for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) {
check_file_is_writeable(obj, sess);
}

if outputs.outputs.contains_key(&OutputType::Metadata) {
let out_filename = filename_for_metadata(sess, crate_name, outputs);
// To avoid races with another rustc process scanning the output directory,
// we need to write the file somewhere else and atomically move it to its
// final destination, with a `fs::rename` call. In order for the rename to
// always succeed, the temporary file needs to be on the same filesystem,
// which is why we create it inside the output directory specifically.
let metadata_tmpdir = TempFileBuilder::new()
.prefix("rmeta")
.tempdir_in(out_filename.parent().unwrap())
.unwrap_or_else(|err| sess.fatal(&format!("couldn't create a temp dir: {}", err)));
let metadata = emit_metadata(sess, codegen_results, &metadata_tmpdir);
match fs::rename(&metadata, &out_filename) {
Ok(_) => {
if sess.opts.debugging_opts.emit_directives {
sess.parse_sess.span_diagnostic.maybe_emit_json_directive(
format!("metadata file written: {}", out_filename.display()));
}
}
Err(e) => sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)),
}
}

let tmpdir = TempFileBuilder::new().prefix("rustc").tempdir().unwrap_or_else(|err|
sess.fatal(&format!("couldn't create a temp dir: {}", err)));

if outputs.outputs.should_codegen() {
let out_filename = out_filename(sess, crate_type, outputs, crate_name);
match crate_type {
config::CrateType::Rlib => {
link_rlib::<B>(sess,
codegen_results,
RlibFlavor::Normal,
&out_filename,
&tmpdir).build();
}
config::CrateType::Staticlib => {
link_staticlib::<B>(sess, codegen_results, &out_filename, &tmpdir);
}
_ => {
link_natively::<B>(
sess,
crate_type,
&out_filename,
codegen_results,
tmpdir.path(),
target_cpu,
);
}
}
}

if sess.opts.cg.save_temps {
let _ = tmpdir.into_path();
}
}

// The third parameter is for env vars, used on windows to set up the
// path for MSVC to find its DLLs, and gcc to find its bundled
// toolchain
Expand Down Expand Up @@ -261,13 +225,13 @@ pub fn each_linked_rlib(sess: &Session,
/// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a
/// directory being searched for `extern crate` (observing an incomplete file).
/// The returned path is the temporary file containing the complete metadata.
fn emit_metadata<'a>(
pub fn emit_metadata<'a>(
sess: &'a Session,
codegen_results: &CodegenResults,
metadata: &EncodedMetadata,
tmpdir: &TempDir
) -> PathBuf {
let out_filename = tmpdir.path().join(METADATA_FILENAME);
let result = fs::write(&out_filename, &codegen_results.metadata.raw_data);
let result = fs::write(&out_filename, &metadata.raw_data);

if let Err(e) = result {
sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
Expand Down Expand Up @@ -351,7 +315,7 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
RlibFlavor::Normal => {
// Instead of putting the metadata in an object file section, rlibs
// contain the metadata in a separate file.
ab.add_file(&emit_metadata(sess, codegen_results, tmpdir));
ab.add_file(&emit_metadata(sess, &codegen_results.metadata, tmpdir));

// For LTO purposes, the bytecode of this library is also inserted
// into the archive.
Expand Down
48 changes: 21 additions & 27 deletions src/librustc_codegen_ssa/base.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ use crate::{ModuleCodegen, ModuleKind, CachedModuleCodegen};

use rustc::dep_graph::cgu_reuse_tracker::CguReuse;
use rustc::hir::def_id::{DefId, LOCAL_CRATE};
use rustc::middle::cstore::EncodedMetadata;
use rustc::middle::lang_items::StartFnLangItem;
use rustc::middle::weak_lang_items;
use rustc::mir::mono::{Stats, CodegenUnitNameBuilder};
Expand All @@ -25,7 +26,7 @@ use rustc::ty::layout::{self, Align, TyLayout, LayoutOf, VariantIdx, HasTyCtxt};
use rustc::ty::query::Providers;
use rustc::middle::cstore::{self, LinkagePreference};
use rustc::util::common::{time, print_time_passes_entry};
use rustc::session::config::{self, CrateType, EntryFnType, Lto};
use rustc::session::config::{self, EntryFnType, Lto};
use rustc::session::Session;
use rustc_mir::monomorphize::item::DefPathBasedNames;
use rustc_mir::monomorphize::Instance;
Expand Down Expand Up @@ -530,26 +531,13 @@ pub const CODEGEN_WORKER_ID: usize = ::std::usize::MAX;
pub fn codegen_crate<B: ExtraBackendMethods>(
backend: B,
tcx: TyCtxt<'a, 'tcx, 'tcx>,
metadata: EncodedMetadata,
need_metadata_module: bool,
rx: mpsc::Receiver<Box<dyn Any + Send>>
) -> OngoingCodegen<B> {

check_for_rustc_errors_attr(tcx);

let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);

// Codegen the metadata.
tcx.sess.profiler(|p| p.start_activity("codegen crate metadata"));

let metadata_cgu_name = cgu_name_builder.build_cgu_name(LOCAL_CRATE,
&["crate"],
Some("metadata")).as_str()
.to_string();
let mut metadata_llvm_module = backend.new_metadata(tcx, &metadata_cgu_name);
let metadata = time(tcx.sess, "write metadata", || {
backend.write_metadata(tcx, &mut metadata_llvm_module)
});
tcx.sess.profiler(|p| p.end_activity("codegen crate metadata"));

// Skip crate items and just output metadata in -Z no-codegen mode.
if tcx.sess.opts.debugging_opts.no_codegen ||
!tcx.sess.opts.output_types.should_codegen() {
Expand All @@ -569,6 +557,8 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
return ongoing_codegen;
}

let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);

// Run the monomorphization collector and partition the collected items into
// codegen units.
let codegen_units = tcx.collect_and_partition_mono_items(LOCAL_CRATE).1;
Expand Down Expand Up @@ -632,17 +622,21 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
ongoing_codegen.submit_pre_codegened_module_to_llvm(tcx, allocator_module);
}

let needs_metadata_module = tcx.sess.crate_types.borrow().iter().any(|ct| {
match *ct {
CrateType::Dylib |
CrateType::ProcMacro => true,
CrateType::Executable |
CrateType::Rlib |
CrateType::Staticlib |
CrateType::Cdylib => false,
}
});
if needs_metadata_module {
if need_metadata_module {
// Codegen the encoded metadata.
tcx.sess.profiler(|p| p.start_activity("codegen crate metadata"));

let metadata_cgu_name = cgu_name_builder.build_cgu_name(LOCAL_CRATE,
&["crate"],
Some("metadata")).as_str()
.to_string();
let mut metadata_llvm_module = backend.new_metadata(tcx, &metadata_cgu_name);
time(tcx.sess, "write compressed metadata", || {
backend.write_compressed_metadata(tcx, &ongoing_codegen.metadata,
&mut metadata_llvm_module);
});
tcx.sess.profiler(|p| p.end_activity("codegen crate metadata"));

let metadata_module = ModuleCodegen {
name: metadata_cgu_name,
module_llvm: metadata_llvm_module,
Expand Down
7 changes: 4 additions & 3 deletions src/librustc_codegen_ssa/traits/backend.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,12 @@ impl<'tcx, T> Backend<'tcx> for T where

pub trait ExtraBackendMethods: CodegenBackend + WriteBackendMethods + Sized + Send {
fn new_metadata(&self, sess: TyCtxt<'_, '_, '_>, mod_name: &str) -> Self::Module;
fn write_metadata<'b, 'gcx>(
fn write_compressed_metadata<'b, 'gcx>(
&self,
tcx: TyCtxt<'b, 'gcx, 'gcx>,
metadata: &mut Self::Module,
) -> EncodedMetadata;
metadata: &EncodedMetadata,
llvm_module: &mut Self::Module,
);
fn codegen_allocator<'b, 'gcx>(
&self,
tcx: TyCtxt<'b, 'gcx, 'gcx>,
Expand Down
Loading