Auto merge of rust-lang#8834 - Gankra:rust-src-vendor, r=ehuss

Check if rust-src contains a vendor dir, and patch it in

This is the cargo side of rust-lang/wg-cargo-std-aware#23

Note that this design naively assumes there is only one version of each package. It does not robustly verify this, and will presumably just cryptically fail to resolve dependencies.

See rust-lang/rust#78790 for the other half of this change.

src/cargo/core/compiler/standard_lib.rs

@@ -7,9 +7,10 @@ use crate::core::resolver::features::{FeaturesFor, ResolvedFeatures};
 use crate::core::resolver::{HasDevUnits, ResolveOpts};
 use crate::core::{Dependency, PackageId, PackageSet, Resolve, SourceId, Workspace};
 use crate::ops::{self, Packages};
-use crate::util::errors::CargoResult;
+use crate::util::errors::{CargoResult, CargoResultExt};
 use std::collections::{HashMap, HashSet};
 use std::env;
+use std::fs;
 use std::path::PathBuf;
 
 /// Parse the `-Zbuild-std` flag.
@@ -38,28 +39,48 @@ pub fn resolve_std<'cfg>(
     crates: &[String],
 ) -> CargoResult<(PackageSet<'cfg>, Resolve, ResolvedFeatures)> {
     let src_path = detect_sysroot_src_path(target_data)?;
-    let to_patch = [
-        "rustc-std-workspace-core",
-        "rustc-std-workspace-alloc",
-        "rustc-std-workspace-std",
-    ];
-    let patches = to_patch
-        .iter()
-        .map(|&name| {
-            let source_path = SourceId::for_path(&src_path.join("library").join(name))?;
-            let dep = Dependency::parse_no_deprecated(name, None, source_path)?;
+
+    // Special std packages should be pulled from `library/` and should be
+    // prefixed with `rustc-std-workspace-` in certain places.
+    let libs_prefix = "library/";
+    let special_std_prefix = "rustc-std-workspace-";
+    let libs_path = src_path.join(libs_prefix);
+
+    // Crates in rust-src to build. libtest is in some sense the "root" package
+    // of std, as nothing else depends on it, so it must be explicitly added.
+    let mut members = vec![format!("{}test", libs_prefix)];
+
+    // If rust-src contains a "vendor" directory, then patch in all the crates it contains.
+    let vendor_path = src_path.join("vendor");
+    let vendor_dir = fs::read_dir(&vendor_path)
+        .chain_err(|| format!("could not read vendor path {}", vendor_path.display()))?;
+    let patches = vendor_dir
+        .into_iter()
+        .map(|entry| {
+            let entry = entry?;
+            let name = entry
+                .file_name()
+                .into_string()
+                .map_err(|_| anyhow::anyhow!("package name wasn't utf8"))?;
+
+            // Remap the rustc-std-workspace crates to the actual rust-src libraries
+            let path = if let Some(real_name) = name.strip_prefix(special_std_prefix) {
+                // Record this crate as something to build in the workspace
+                members.push(format!("{}{}", libs_prefix, real_name));
+                libs_path.join(&name)
+            } else {
+                entry.path()
+            };
+            let source_path = SourceId::for_path(&path)?;
+            let dep = Dependency::parse_no_deprecated(&name, None, source_path)?;
             Ok(dep)
         })
-        .collect::<CargoResult<Vec<_>>>()?;
+        .collect::<CargoResult<Vec<_>>>()
+        .chain_err(|| "failed to generate vendor patches")?;
+
     let crates_io_url = crate::sources::CRATES_IO_INDEX.parse().unwrap();
     let mut patch = HashMap::new();
     patch.insert(crates_io_url, patches);
-    let members = vec![
-        String::from("library/std"),
-        String::from("library/core"),
-        String::from("library/alloc"),
-        String::from("library/test"),
-    ];
     let ws_config = crate::core::WorkspaceConfig::Root(crate::core::WorkspaceRootConfig::new(
         &src_path,
         &Some(members),

tests/testsuite/mock-std/library/test/Cargo.toml

@@ -10,6 +10,7 @@ std = { path = "../std" }
 panic_unwind = { path = "../panic_unwind" }
 compiler_builtins = { path = "../compiler_builtins" }
 registry-dep-using-std = { version = "*", features = ['mockbuild'] }
+registry-dep-only-used-by-test = { version = "*" }
 
 [features]
 panic-unwind = []

tests/testsuite/mock-std/library/test/src/lib.rs

@@ -7,4 +7,5 @@ extern crate test;
 pub use test::*;
 
 pub fn custom_api() {
+    registry_dep_only_used_by_test::wow_testing_is_so_easy();
 }

tests/testsuite/mock-std/vendor/registry-dep-only-used-by-test/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "registry-dep-only-used-by-test"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+
+[features]

tests/testsuite/mock-std/vendor/registry-dep-only-used-by-test/src/lib.rs

@@ -0,0 +1,2 @@
+pub fn wow_testing_is_so_easy() {
+}

tests/testsuite/mock-std/vendor/registry-dep-using-alloc/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "registry-dep-using-alloc"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+rustc-std-workspace-alloc = { version = "*", optional = true }
+rustc-std-workspace-core = { version = "*", optional = true }
+
+[features]
+mockbuild = ["rustc-std-workspace-alloc", "rustc-std-workspace-core"]

tests/testsuite/mock-std/vendor/registry-dep-using-alloc/src/lib.rs

@@ -0,0 +1,9 @@
+#[cfg(feature = "mockbuild")]
+pub fn custom_api() {
+}
+
+#[cfg(not(feature = "mockbuild"))]
+pub fn non_sysroot_api() {
+    core::custom_api();
+    alloc::custom_api();
+}

tests/testsuite/mock-std/vendor/registry-dep-using-core/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "registry-dep-using-core"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+rustc-std-workspace-core = { version = "*", optional = true }
+
+[features]
+mockbuild = ["rustc-std-workspace-core"]

tests/testsuite/mock-std/vendor/registry-dep-using-core/src/lib.rs

@@ -0,0 +1,8 @@
+#[cfg(feature = "mockbuild")]
+pub fn custom_api() {
+}
+
+#[cfg(not(feature = "mockbuild"))]
+pub fn non_sysroot_api() {
+    core::custom_api();
+}

tests/testsuite/mock-std/vendor/registry-dep-using-std/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "registry-dep-using-std"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+rustc-std-workspace-std = { version = "*", optional = true }
+
+[features]
+mockbuild = ["rustc-std-workspace-std"]

tests/testsuite/mock-std/vendor/registry-dep-using-std/src/lib.rs

@@ -0,0 +1,8 @@
+#[cfg(feature = "mockbuild")]
+pub fn custom_api() {
+}
+
+#[cfg(not(feature = "mockbuild"))]
+pub fn non_sysroot_api() {
+    std::custom_api();
+}

tests/testsuite/mock-std/vendor/rustc-std-workspace-alloc/Cargo.toml

@@ -0,0 +1 @@
+this file shouldn't be read 

tests/testsuite/mock-std/vendor/rustc-std-workspace-core/Cargo.toml

@@ -0,0 +1 @@
+this file shouldn't be read 

tests/testsuite/mock-std/vendor/rustc-std-workspace-std/Cargo.toml

@@ -0,0 +1 @@
+this file shouldn't be read 

tests/testsuite/standard_lib.rs

@@ -27,71 +27,18 @@ fn setup() -> Option<Setup> {
         return None;
     }
 
-    // Our mock sysroot requires a few packages from crates.io, so make sure
-    // they're "published" to crates.io. Also edit their code a bit to make sure
-    // that they have access to our custom crates with custom apis.
+    // Register a version of one of the std dependencies that doesn't compile.
+    // This ensures that the mock-std's vendor is actually being used.
     Package::new("registry-dep-using-core", "1.0.0")
         .file(
             "src/lib.rs",
             "
-                #![no_std]
-
-                #[cfg(feature = \"mockbuild\")]
-                pub fn custom_api() {
-                }
-
-                #[cfg(not(feature = \"mockbuild\"))]
-                pub fn non_sysroot_api() {
-                    core::custom_api();
-                }
+               don't compile me bro!!
             ",
         )
         .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
         .feature("mockbuild", &["rustc-std-workspace-core"])
         .publish();
-    Package::new("registry-dep-using-alloc", "1.0.0")
-        .file(
-            "src/lib.rs",
-            "
-                #![no_std]
-
-                extern crate alloc;
-
-                #[cfg(feature = \"mockbuild\")]
-                pub fn custom_api() {
-                }
-
-                #[cfg(not(feature = \"mockbuild\"))]
-                pub fn non_sysroot_api() {
-                    core::custom_api();
-                    alloc::custom_api();
-                }
-            ",
-        )
-        .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
-        .add_dep(Dependency::new("rustc-std-workspace-alloc", "*").optional(true))
-        .feature(
-            "mockbuild",
-            &["rustc-std-workspace-core", "rustc-std-workspace-alloc"],
-        )
-        .publish();
-    Package::new("registry-dep-using-std", "1.0.0")
-        .file(
-            "src/lib.rs",
-            "
-                #[cfg(feature = \"mockbuild\")]
-                pub fn custom_api() {
-                }
-
-                #[cfg(not(feature = \"mockbuild\"))]
-                pub fn non_sysroot_api() {
-                    std::custom_api();
-                }
-            ",
-        )
-        .add_dep(Dependency::new("rustc-std-workspace-std", "*").optional(true))
-        .feature("mockbuild", &["rustc-std-workspace-std"])
-        .publish();
 
     let p = ProjectBuilder::new(paths::root().join("rustc-wrapper"))
         .file(
@@ -335,6 +282,81 @@ fn depend_same_as_std() {
         None => return,
     };
 
+    // Our mock sysroot requires a few packages from crates.io, so make sure
+    // they're "published" to crates.io. Also edit their code a bit to make sure
+    // that they have access to our custom crates with custom apis.
+    Package::new("registry-dep-using-core", "1.0.0")
+        .file(
+            "src/lib.rs",
+            "
+                #![no_std]
+
+                #[cfg(feature = \"mockbuild\")]
+                pub fn custom_api() {
+                }
+
+                #[cfg(not(feature = \"mockbuild\"))]
+                pub fn non_sysroot_api() {
+                    core::custom_api();
+                }
+            ",
+        )
+        .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
+        .feature("mockbuild", &["rustc-std-workspace-core"])
+        .publish();
+    Package::new("registry-dep-using-alloc", "1.0.0")
+        .file(
+            "src/lib.rs",
+            "
+                #![no_std]
+
+                extern crate alloc;
+
+                #[cfg(feature = \"mockbuild\")]
+                pub fn custom_api() {
+                }
+
+                #[cfg(not(feature = \"mockbuild\"))]
+                pub fn non_sysroot_api() {
+                    core::custom_api();
+                    alloc::custom_api();
+                }
+            ",
+        )
+        .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
+        .add_dep(Dependency::new("rustc-std-workspace-alloc", "*").optional(true))
+        .feature(
+            "mockbuild",
+            &["rustc-std-workspace-core", "rustc-std-workspace-alloc"],
+        )
+        .publish();
+    Package::new("registry-dep-using-std", "1.0.0")
+        .file(
+            "src/lib.rs",
+            "
+                #[cfg(feature = \"mockbuild\")]
+                pub fn custom_api() {
+                }
+
+                #[cfg(not(feature = \"mockbuild\"))]
+                pub fn non_sysroot_api() {
+                    std::custom_api();
+                }
+            ",
+        )
+        .add_dep(Dependency::new("rustc-std-workspace-std", "*").optional(true))
+        .feature("mockbuild", &["rustc-std-workspace-std"])
+        .publish();
+    Package::new("registry-dep-only-used-by-test", "1.0.0")
+        .file(
+            "src/lib.rs",
+            "
+                pub fn wow_testing_is_so_easy() {
+                }
+            ",
+        )
+        .publish();
+
     let p = project()
         .file(
             "src/lib.rs",