Skip to content

Commit

Permalink
Merge pull request #645 from eddyb/build-std
Browse files Browse the repository at this point in the history 
Build the stdlib from rust-src sources.
  • Loading branch information
@Mark-Simulacrum
Mark-Simulacrum authored Apr 16, 2020
2 parents f636e7c + e8e8983 commit 90d06dc
Show file tree
Hide file tree
Showing 2 changed files with 152 additions and 72 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
target
*.pyc
/cache/
/rust.git/
222 changes: 150 additions & 72 deletions collector/src/bin/rustc-perf-collector/sysroot.rs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
use anyhow::{anyhow, Context};
use chrono::{DateTime, Utc};
use collector::Sha;
use std::ffi::OsStr;
use std::fmt;
use std::fs::{self, File};
use std::io::{BufReader, Read};
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use tar::Archive;
use xz2::bufread::XzDecoder;

Expand Down Expand Up @@ -40,10 +39,116 @@ impl Sysroot {
};

download.get_and_extract(ModuleVariant::Rustc)?;
download.get_and_extract(ModuleVariant::Std)?;
// HACK(eddyb) commented out because we build our own stdlib
// (see `fn build_std` below).
// download.get_and_extract(ModuleVariant::Std)?;
download.get_and_extract(ModuleVariant::Cargo)?;
download.get_and_extract(ModuleVariant::RustSrc)?;

download.into_sysroot()
let sysroot_dir = download.directory.join(&download.rust_sha);
let sysroot = download.into_sysroot()?;

// FIXME(eddyb) remove this once we no longer need to
// build our own stdlib (see `fn build_std` below).
sysroot.build_std(sysroot_dir)?;

Ok(sysroot)
}

/// Build `std`+`test`+`proc_macro` in a similar way to Cargo's `-Zbuild-std`
/// feature, but only once, and move the resulting libraries into the sysroot.
///
/// We only need this until https://github.com/rust-lang/cargo/pull/8073
/// reaches beta, because then `rust-lang/rust` builds will have that
/// treatment. For now, we only have access to that Cargo change here,
/// using the newly built Cargo.
///
/// For more background on why we need this, see this comment:
/// https://github.com/rust-lang/rust/issues/69060#issuecomment-604928032
/// (in short, Cargo used to include `rustc -vV` output, which contains
/// the commit hash, into `-Cmetadata`, producing different `std`s,
/// and making the perf runs incomparable, up to several % of difference).
fn build_std(&self, sysroot_dir: PathBuf) -> anyhow::Result<()> {
// Make sure everything below gets absolute directories.
let sysroot_dir = sysroot_dir.canonicalize()?;

let sysroot_rustlib_dir = sysroot_dir.join("lib/rustlib");
let rust_src_dir = sysroot_rustlib_dir.join("src/rust");

// HACK(eddyb) add a top-level `Cargo.toml` that has the necessary
// `patch.crates-io` entries for `rustc-std-workspace-{core,alloc,std}`.
// (maybe `rust-src` should include such a `Cargo.toml`?)
fs::write(
rust_src_dir.join("Cargo.toml"),
"\
[workspace]
members = ['src/libtest']
[patch.crates-io]
# See comments in `tools/rustc-std-workspace-core/README.md` for what's going on
# here
rustc-std-workspace-core = { path = 'src/tools/rustc-std-workspace-core' }
rustc-std-workspace-alloc = { path = 'src/tools/rustc-std-workspace-alloc' }
rustc-std-workspace-std = { path = 'src/tools/rustc-std-workspace-std' }
",
)?;

// HACK(eddyb) we need `std` to run the build scripts to build `std`.
let vanilla_sysroot_dir = {
let vanilla_download = SysrootDownload {
directory: sysroot_dir.join("vanilla-sysroot"),
rust_sha: self.sha.clone(),
triple: self.triple.clone(),
};
vanilla_download.get_and_extract(ModuleVariant::Std)?;
vanilla_download.directory.join(vanilla_download.rust_sha)
};

let rustflags = format!(
"--sysroot={sysroot} --remap-path-prefix={remap_from}={remap_to}",
sysroot = vanilla_sysroot_dir.display(),
remap_from = rust_src_dir.display(),
remap_to = "/rustc/REDACTED_SHA_HASH/"
);

// Run Cargo to produce `$local_build_target_dir/release/deps/lib*.rlib`.
let local_build_target_dir = sysroot_dir.join("build-std-target");
let cargo_status = std::process::Command::new(&self.cargo)
.env("RUSTC", &self.rustc)
.env("RUSTFLAGS", rustflags)
.env("__CARGO_DEFAULT_LIB_METADATA", "rustc-perf-std")
.args(&["build", "--release"])
.arg("--target-dir")
.arg(&local_build_target_dir)
.args(&["--features", "panic-unwind", "--features", "backtrace"])
.arg("--manifest-path")
.arg(rust_src_dir.join("src/libtest/Cargo.toml"))
.status()?;
if !cargo_status.success() {
return Err(anyhow!(
"unable to build stdlib for {} triple {}",
self.sha,
self.triple
));
}

// Move all of the `rlib` files into the main sysroot.
let sysroot_target_lib_dir = sysroot_rustlib_dir.join(&self.triple).join("lib");
for entry in fs::read_dir(local_build_target_dir.join("release/deps"))? {
let entry = entry?;
let path = entry.path();
if let (Some(name), Some(ext)) = (path.file_name(), path.extension()) {
if ext == "rlib" {
fs::rename(&path, sysroot_target_lib_dir.join(name))?;
}
}
}

// Clean up, to avoid accidental usage of these directories.
fs::remove_dir_all(vanilla_sysroot_dir)?;
fs::remove_dir_all(local_build_target_dir)?;

Ok(())
}
}

Expand All @@ -66,14 +171,15 @@ struct SysrootDownload {
triple: String,
}

const MODULE_URL: &str =
"https://rust-lang-ci2.s3.amazonaws.com/rustc-builds/@SHA@/@MODULE@-nightly-@TRIPLE@.tar.xz";
const BASE_URL: &str = "https://rust-lang-ci2.s3.amazonaws.com/rustc-builds";

// FIXME(eddyb) rename to just `Component`.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum ModuleVariant {
Cargo,
Rustc,
Std,
RustSrc,
}

impl fmt::Display for ModuleVariant {
Expand All @@ -82,47 +188,45 @@ impl fmt::Display for ModuleVariant {
ModuleVariant::Cargo => write!(f, "cargo"),
ModuleVariant::Rustc => write!(f, "rustc"),
ModuleVariant::Std => write!(f, "rust-std"),
ModuleVariant::RustSrc => write!(f, "rust-src"),
}
}
}

impl ModuleVariant {
fn url(&self, sysroot: &SysrootDownload, triple: &str) -> String {
MODULE_URL
.replace("@MODULE@", &self.to_string())
.replace("@SHA@", &sysroot.rust_sha)
.replace("@TRIPLE@", triple)
let suffix = if *self == ModuleVariant::RustSrc {
String::new()
} else {
format!("-{}", triple)
};
format!(
"{base}/{sha}/{module}-nightly{suffix}.tar.xz",
base = BASE_URL,
module = self,
sha = sysroot.rust_sha,
suffix = suffix,
)
}
}

impl SysrootDownload {
fn into_sysroot(self) -> anyhow::Result<Sysroot> {
let sysroot_bin_dir = self.directory.join(&self.rust_sha).join("bin");
let sysroot_bin = |name| {
let path = sysroot_bin_dir.join(name);
path.canonicalize().with_context(|| {
format!(
"failed to canonicalize {} path for {}: {:?}",
name, self.rust_sha, path
)
})
};

Ok(Sysroot {
rustc: self
.directory
.join(&self.rust_sha)
.join("rustc/bin/rustc")
.canonicalize()
.with_context(|| {
format!("failed to canonicalize rustc path for {}", self.rust_sha)
})?,
rustdoc: self
.directory
.join(&self.rust_sha)
.join("rustc/bin/rustdoc")
.canonicalize()
.with_context(|| {
format!("failed to canonicalize rustdoc path for {}", self.rust_sha)
})?,
cargo: {
let path = self.directory.join(&self.rust_sha).join("cargo/bin/cargo");
path.canonicalize().with_context(|| {
format!(
"failed to canonicalize cargo path for {}: {:?}",
self.rust_sha, path
)
})?
},
rustc: sysroot_bin("rustc")?,
rustdoc: sysroot_bin("rustdoc")?,
cargo: sysroot_bin("cargo")?,
sha: self.rust_sha,
triple: self.triple,
})
Expand Down Expand Up @@ -161,19 +265,21 @@ impl SysrootDownload {
}

return Err(anyhow!(
"unable to download sha {} triple {} module {}",
"unable to download sha {} triple {} module {} from {}",
self.rust_sha,
self.triple,
variant
variant,
url
));
}

fn extract<T: Read>(&self, variant: ModuleVariant, reader: T) -> anyhow::Result<()> {
let is_std = variant == ModuleVariant::Std;
let mut archive = Archive::new(reader);
let std_prefix = format!("rust-std-{}/lib/rustlib", self.triple);

let mut to_link = Vec::new();
let prefix = if variant == ModuleVariant::Std {
format!("rust-std-{}", self.triple)
} else {
variant.to_string()
};

let unpack_into = self.directory.join(&self.rust_sha);

Expand All @@ -184,21 +290,11 @@ impl SysrootDownload {
assert!(components.next().is_some(), "strip container directory");
let path = components.as_path();

let path = if is_std {
if let Ok(path) = path.strip_prefix(&std_prefix) {
if path.extension() == Some(OsStr::new("dylib")) {
to_link.push(path.to_owned());
continue;
} else {
Path::new("rustc/lib/rustlib").join(path)
}
} else {
continue;
}
let path = if let Ok(path) = path.strip_prefix(&prefix) {
unpack_into.join(path)
} else {
path.into()
continue;
};
let path = unpack_into.join(path);
fs::create_dir_all(&path.parent().unwrap()).with_context(|| {
format!(
"could not create intermediate directories for {}",
Expand All @@ -208,24 +304,6 @@ impl SysrootDownload {
entry.unpack(path)?;
}

let link_dst_prefix = unpack_into.join(format!("rustc/lib/rustlib/{}/lib", self.triple));
let link_src_prefix = format!("{}/lib", self.triple);
for path in to_link {
let src = unpack_into.join("rustc/lib").join(
path.strip_prefix(&link_src_prefix)
.with_context(|| format!("stripping prefix from: {:?}", path))?,
);
let dst = link_dst_prefix.join(&path);
fs::create_dir_all(&dst.parent().unwrap()).with_context(|| {
format!(
"could not create intermediate directories for {}",
dst.display()
)
})?;
log::trace!("linking {} to {}", src.display(), dst.display());
fs::hard_link(src, dst)?;
}

Ok(())
}
}

0 comments on commit 90d06dc

Please sign in to comment.