From b5875a3b0d5baa275544183714e248b050f58227 Mon Sep 17 00:00:00 2001 From: Eduard Burtescu Date: Wed, 15 Apr 2020 21:47:27 +0000 Subject: [PATCH 1/3] Add temporary directories to .gitignore. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index ec9171812..7810e8534 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ target *.pyc +/cache/ +/rust.git/ From 090817d3c46db4b17fa37d160e6eb58d3be56fe1 Mon Sep 17 00:00:00 2001 From: Eduard Burtescu Date: Thu, 9 Apr 2020 18:03:57 +0000 Subject: [PATCH 2/3] Download rust-src component. --- .../src/bin/rustc-perf-collector/sysroot.rs | 113 +++++++----------- 1 file changed, 43 insertions(+), 70 deletions(-) diff --git a/collector/src/bin/rustc-perf-collector/sysroot.rs b/collector/src/bin/rustc-perf-collector/sysroot.rs index 3b629f038..c2b233562 100644 --- a/collector/src/bin/rustc-perf-collector/sysroot.rs +++ b/collector/src/bin/rustc-perf-collector/sysroot.rs @@ -1,11 +1,10 @@ use anyhow::{anyhow, Context}; use chrono::{DateTime, Utc}; use collector::Sha; -use std::ffi::OsStr; use std::fmt; use std::fs::{self, File}; use std::io::{BufReader, Read}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use tar::Archive; use xz2::bufread::XzDecoder; @@ -42,6 +41,7 @@ impl Sysroot { download.get_and_extract(ModuleVariant::Rustc)?; download.get_and_extract(ModuleVariant::Std)?; download.get_and_extract(ModuleVariant::Cargo)?; + download.get_and_extract(ModuleVariant::RustSrc)?; download.into_sysroot() } @@ -66,14 +66,15 @@ struct SysrootDownload { triple: String, } -const MODULE_URL: &str = - "https://rust-lang-ci2.s3.amazonaws.com/rustc-builds/@SHA@/@MODULE@-nightly-@TRIPLE@.tar.xz"; +const BASE_URL: &str = "https://rust-lang-ci2.s3.amazonaws.com/rustc-builds"; +// FIXME(eddyb) rename to just `Component`. #[derive(Debug, Copy, Clone, PartialEq, Eq)] enum ModuleVariant { Cargo, Rustc, Std, + RustSrc, } impl fmt::Display for ModuleVariant { @@ -82,47 +83,45 @@ impl fmt::Display for ModuleVariant { ModuleVariant::Cargo => write!(f, "cargo"), ModuleVariant::Rustc => write!(f, "rustc"), ModuleVariant::Std => write!(f, "rust-std"), + ModuleVariant::RustSrc => write!(f, "rust-src"), } } } impl ModuleVariant { fn url(&self, sysroot: &SysrootDownload, triple: &str) -> String { - MODULE_URL - .replace("@MODULE@", &self.to_string()) - .replace("@SHA@", &sysroot.rust_sha) - .replace("@TRIPLE@", triple) + let suffix = if *self == ModuleVariant::RustSrc { + String::new() + } else { + format!("-{}", triple) + }; + format!( + "{base}/{sha}/{module}-nightly{suffix}.tar.xz", + base = BASE_URL, + module = self, + sha = sysroot.rust_sha, + suffix = suffix, + ) } } impl SysrootDownload { fn into_sysroot(self) -> anyhow::Result { + let sysroot_bin_dir = self.directory.join(&self.rust_sha).join("bin"); + let sysroot_bin = |name| { + let path = sysroot_bin_dir.join(name); + path.canonicalize().with_context(|| { + format!( + "failed to canonicalize {} path for {}: {:?}", + name, self.rust_sha, path + ) + }) + }; + Ok(Sysroot { - rustc: self - .directory - .join(&self.rust_sha) - .join("rustc/bin/rustc") - .canonicalize() - .with_context(|| { - format!("failed to canonicalize rustc path for {}", self.rust_sha) - })?, - rustdoc: self - .directory - .join(&self.rust_sha) - .join("rustc/bin/rustdoc") - .canonicalize() - .with_context(|| { - format!("failed to canonicalize rustdoc path for {}", self.rust_sha) - })?, - cargo: { - let path = self.directory.join(&self.rust_sha).join("cargo/bin/cargo"); - path.canonicalize().with_context(|| { - format!( - "failed to canonicalize cargo path for {}: {:?}", - self.rust_sha, path - ) - })? - }, + rustc: sysroot_bin("rustc")?, + rustdoc: sysroot_bin("rustdoc")?, + cargo: sysroot_bin("cargo")?, sha: self.rust_sha, triple: self.triple, }) @@ -161,19 +160,21 @@ impl SysrootDownload { } return Err(anyhow!( - "unable to download sha {} triple {} module {}", + "unable to download sha {} triple {} module {} from {}", self.rust_sha, self.triple, - variant + variant, + url )); } fn extract(&self, variant: ModuleVariant, reader: T) -> anyhow::Result<()> { - let is_std = variant == ModuleVariant::Std; let mut archive = Archive::new(reader); - let std_prefix = format!("rust-std-{}/lib/rustlib", self.triple); - - let mut to_link = Vec::new(); + let prefix = if variant == ModuleVariant::Std { + format!("rust-std-{}", self.triple) + } else { + variant.to_string() + }; let unpack_into = self.directory.join(&self.rust_sha); @@ -184,21 +185,11 @@ impl SysrootDownload { assert!(components.next().is_some(), "strip container directory"); let path = components.as_path(); - let path = if is_std { - if let Ok(path) = path.strip_prefix(&std_prefix) { - if path.extension() == Some(OsStr::new("dylib")) { - to_link.push(path.to_owned()); - continue; - } else { - Path::new("rustc/lib/rustlib").join(path) - } - } else { - continue; - } + let path = if let Ok(path) = path.strip_prefix(&prefix) { + unpack_into.join(path) } else { - path.into() + continue; }; - let path = unpack_into.join(path); fs::create_dir_all(&path.parent().unwrap()).with_context(|| { format!( "could not create intermediate directories for {}", @@ -208,24 +199,6 @@ impl SysrootDownload { entry.unpack(path)?; } - let link_dst_prefix = unpack_into.join(format!("rustc/lib/rustlib/{}/lib", self.triple)); - let link_src_prefix = format!("{}/lib", self.triple); - for path in to_link { - let src = unpack_into.join("rustc/lib").join( - path.strip_prefix(&link_src_prefix) - .with_context(|| format!("stripping prefix from: {:?}", path))?, - ); - let dst = link_dst_prefix.join(&path); - fs::create_dir_all(&dst.parent().unwrap()).with_context(|| { - format!( - "could not create intermediate directories for {}", - dst.display() - ) - })?; - log::trace!("linking {} to {}", src.display(), dst.display()); - fs::hard_link(src, dst)?; - } - Ok(()) } } From e8e898305df54f3b1a2518c371abb5d353aedf8d Mon Sep 17 00:00:00 2001 From: Eduard Burtescu Date: Wed, 15 Apr 2020 21:45:14 +0000 Subject: [PATCH 3/3] Build the stdlib from rust-src sources. --- .../src/bin/rustc-perf-collector/sysroot.rs | 109 +++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/collector/src/bin/rustc-perf-collector/sysroot.rs b/collector/src/bin/rustc-perf-collector/sysroot.rs index c2b233562..d8503273b 100644 --- a/collector/src/bin/rustc-perf-collector/sysroot.rs +++ b/collector/src/bin/rustc-perf-collector/sysroot.rs @@ -39,11 +39,116 @@ impl Sysroot { }; download.get_and_extract(ModuleVariant::Rustc)?; - download.get_and_extract(ModuleVariant::Std)?; + // HACK(eddyb) commented out because we build our own stdlib + // (see `fn build_std` below). + // download.get_and_extract(ModuleVariant::Std)?; download.get_and_extract(ModuleVariant::Cargo)?; download.get_and_extract(ModuleVariant::RustSrc)?; - download.into_sysroot() + let sysroot_dir = download.directory.join(&download.rust_sha); + let sysroot = download.into_sysroot()?; + + // FIXME(eddyb) remove this once we no longer need to + // build our own stdlib (see `fn build_std` below). + sysroot.build_std(sysroot_dir)?; + + Ok(sysroot) + } + + /// Build `std`+`test`+`proc_macro` in a similar way to Cargo's `-Zbuild-std` + /// feature, but only once, and move the resulting libraries into the sysroot. + /// + /// We only need this until https://github.com/rust-lang/cargo/pull/8073 + /// reaches beta, because then `rust-lang/rust` builds will have that + /// treatment. For now, we only have access to that Cargo change here, + /// using the newly built Cargo. + /// + /// For more background on why we need this, see this comment: + /// https://github.com/rust-lang/rust/issues/69060#issuecomment-604928032 + /// (in short, Cargo used to include `rustc -vV` output, which contains + /// the commit hash, into `-Cmetadata`, producing different `std`s, + /// and making the perf runs incomparable, up to several % of difference). + fn build_std(&self, sysroot_dir: PathBuf) -> anyhow::Result<()> { + // Make sure everything below gets absolute directories. + let sysroot_dir = sysroot_dir.canonicalize()?; + + let sysroot_rustlib_dir = sysroot_dir.join("lib/rustlib"); + let rust_src_dir = sysroot_rustlib_dir.join("src/rust"); + + // HACK(eddyb) add a top-level `Cargo.toml` that has the necessary + // `patch.crates-io` entries for `rustc-std-workspace-{core,alloc,std}`. + // (maybe `rust-src` should include such a `Cargo.toml`?) + fs::write( + rust_src_dir.join("Cargo.toml"), + "\ +[workspace] +members = ['src/libtest'] + +[patch.crates-io] +# See comments in `tools/rustc-std-workspace-core/README.md` for what's going on +# here +rustc-std-workspace-core = { path = 'src/tools/rustc-std-workspace-core' } +rustc-std-workspace-alloc = { path = 'src/tools/rustc-std-workspace-alloc' } +rustc-std-workspace-std = { path = 'src/tools/rustc-std-workspace-std' } +", + )?; + + // HACK(eddyb) we need `std` to run the build scripts to build `std`. + let vanilla_sysroot_dir = { + let vanilla_download = SysrootDownload { + directory: sysroot_dir.join("vanilla-sysroot"), + rust_sha: self.sha.clone(), + triple: self.triple.clone(), + }; + vanilla_download.get_and_extract(ModuleVariant::Std)?; + vanilla_download.directory.join(vanilla_download.rust_sha) + }; + + let rustflags = format!( + "--sysroot={sysroot} --remap-path-prefix={remap_from}={remap_to}", + sysroot = vanilla_sysroot_dir.display(), + remap_from = rust_src_dir.display(), + remap_to = "/rustc/REDACTED_SHA_HASH/" + ); + + // Run Cargo to produce `$local_build_target_dir/release/deps/lib*.rlib`. + let local_build_target_dir = sysroot_dir.join("build-std-target"); + let cargo_status = std::process::Command::new(&self.cargo) + .env("RUSTC", &self.rustc) + .env("RUSTFLAGS", rustflags) + .env("__CARGO_DEFAULT_LIB_METADATA", "rustc-perf-std") + .args(&["build", "--release"]) + .arg("--target-dir") + .arg(&local_build_target_dir) + .args(&["--features", "panic-unwind", "--features", "backtrace"]) + .arg("--manifest-path") + .arg(rust_src_dir.join("src/libtest/Cargo.toml")) + .status()?; + if !cargo_status.success() { + return Err(anyhow!( + "unable to build stdlib for {} triple {}", + self.sha, + self.triple + )); + } + + // Move all of the `rlib` files into the main sysroot. + let sysroot_target_lib_dir = sysroot_rustlib_dir.join(&self.triple).join("lib"); + for entry in fs::read_dir(local_build_target_dir.join("release/deps"))? { + let entry = entry?; + let path = entry.path(); + if let (Some(name), Some(ext)) = (path.file_name(), path.extension()) { + if ext == "rlib" { + fs::rename(&path, sysroot_target_lib_dir.join(name))?; + } + } + } + + // Clean up, to avoid accidental usage of these directories. + fs::remove_dir_all(vanilla_sysroot_dir)?; + fs::remove_dir_all(local_build_target_dir)?; + + Ok(()) } }