[beta] Rollup backports

Cargo.lock

@@ -243,6 +243,7 @@ dependencies = [
  "anyhow",
  "flate2",
  "hex 0.4.2",
+ "num_cpus",
  "rayon",
  "serde",
  "serde_json",
@@ -728,12 +729,12 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09ee0cc8804d5393478d743b035099520087a5186f3b93fa58cec08fa62407b6"
+checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87"
 dependencies = [
- "cfg-if",
  "crossbeam-utils 0.7.2",
+ "maybe-uninit",
 ]
 
 [[package]]

compiler/rustc_parse/src/parser/expr.rs

@@ -16,6 +16,7 @@ use rustc_ast_pretty::pprust;
 use rustc_errors::{Applicability, DiagnosticBuilder, PResult};
 use rustc_span::source_map::{self, Span, Spanned};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
+use rustc_span::{BytePos, Pos};
 use std::mem;
 use tracing::debug;
 
@@ -839,9 +840,10 @@ impl<'a> Parser<'a> {
         }
         use FloatComponent::*;
 
+        let float_str = float.as_str();
         let mut components = Vec::new();
         let mut ident_like = String::new();
-        for c in float.as_str().chars() {
+        for c in float_str.chars() {
             if c == '_' || c.is_ascii_alphanumeric() {
                 ident_like.push(c);
             } else if matches!(c, '.' | '+' | '-') {
@@ -857,30 +859,54 @@ impl<'a> Parser<'a> {
             components.push(IdentLike(ident_like));
         }
 
-        // FIXME: Make the span more precise.
+        // With proc macros the span can refer to anything, the source may be too short,
+        // or too long, or non-ASCII. It only makes sense to break our span into components
+        // if its underlying text is identical to our float literal.
         let span = self.token.span;
+        let can_take_span_apart =
+            || self.span_to_snippet(span).as_deref() == Ok(float_str).as_deref();
+
         match &*components {
             // 1e2
             [IdentLike(i)] => {
                 self.parse_tuple_field_access_expr(lo, base, Symbol::intern(&i), suffix, None)
             }
             // 1.
             [IdentLike(i), Punct('.')] => {
+                let (ident_span, dot_span) = if can_take_span_apart() {
+                    let (span, ident_len) = (span.data(), BytePos::from_usize(i.len()));
+                    let ident_span = span.with_hi(span.lo + ident_len);
+                    let dot_span = span.with_lo(span.lo + ident_len);
+                    (ident_span, dot_span)
+                } else {
+                    (span, span)
+                };
                 assert!(suffix.is_none());
                 let symbol = Symbol::intern(&i);
-                self.token = Token::new(token::Ident(symbol, false), span);
-                let next_token = Token::new(token::Dot, span);
+                self.token = Token::new(token::Ident(symbol, false), ident_span);
+                let next_token = Token::new(token::Dot, dot_span);
                 self.parse_tuple_field_access_expr(lo, base, symbol, None, Some(next_token))
             }
             // 1.2 | 1.2e3
             [IdentLike(i1), Punct('.'), IdentLike(i2)] => {
+                let (ident1_span, dot_span, ident2_span) = if can_take_span_apart() {
+                    let (span, ident1_len) = (span.data(), BytePos::from_usize(i1.len()));
+                    let ident1_span = span.with_hi(span.lo + ident1_len);
+                    let dot_span = span
+                        .with_lo(span.lo + ident1_len)
+                        .with_hi(span.lo + ident1_len + BytePos(1));
+                    let ident2_span = self.token.span.with_lo(span.lo + ident1_len + BytePos(1));
+                    (ident1_span, dot_span, ident2_span)
+                } else {
+                    (span, span, span)
+                };
                 let symbol1 = Symbol::intern(&i1);
-                self.token = Token::new(token::Ident(symbol1, false), span);
-                let next_token1 = Token::new(token::Dot, span);
+                self.token = Token::new(token::Ident(symbol1, false), ident1_span);
+                let next_token1 = Token::new(token::Dot, dot_span);
                 let base1 =
                     self.parse_tuple_field_access_expr(lo, base, symbol1, None, Some(next_token1));
                 let symbol2 = Symbol::intern(&i2);
-                let next_token2 = Token::new(token::Ident(symbol2, false), span);
+                let next_token2 = Token::new(token::Ident(symbol2, false), ident2_span);
                 self.bump_with(next_token2); // `.`
                 self.parse_tuple_field_access_expr(lo, base1, symbol2, suffix, None)
             }

src/bootstrap/builder.rs

@@ -193,37 +193,37 @@ impl StepDescription {
             );
         }
 
-        if paths.is_empty() {
-            for (desc, should_run) in v.iter().zip(should_runs) {
+        if paths.is_empty() || builder.config.include_default_paths {
+            for (desc, should_run) in v.iter().zip(&should_runs) {
                 if desc.default && should_run.is_really_default {
                     for pathset in &should_run.paths {
                         desc.maybe_run(builder, pathset);
                     }
                 }
             }
-        } else {
-            for path in paths {
-                // strip CurDir prefix if present
-                let path = match path.strip_prefix(".") {
-                    Ok(p) => p,
-                    Err(_) => path,
-                };
+        }
 
-                let mut attempted_run = false;
-                for (desc, should_run) in v.iter().zip(&should_runs) {
-                    if let Some(suite) = should_run.is_suite_path(path) {
-                        attempted_run = true;
-                        desc.maybe_run(builder, suite);
-                    } else if let Some(pathset) = should_run.pathset_for_path(path) {
-                        attempted_run = true;
-                        desc.maybe_run(builder, pathset);
-                    }
-                }
+        for path in paths {
+            // strip CurDir prefix if present
+            let path = match path.strip_prefix(".") {
+                Ok(p) => p,
+                Err(_) => path,
+            };
 
-                if !attempted_run {
-                    panic!("error: no rules matched {}", path.display());
+            let mut attempted_run = false;
+            for (desc, should_run) in v.iter().zip(&should_runs) {
+                if let Some(suite) = should_run.is_suite_path(path) {
+                    attempted_run = true;
+                    desc.maybe_run(builder, suite);
+                } else if let Some(pathset) = should_run.pathset_for_path(path) {
+                    attempted_run = true;
+                    desc.maybe_run(builder, pathset);
                 }
             }
+
+            if !attempted_run {
+                panic!("error: no rules matched {}", path.display());
+            }
         }
     }
 }
@@ -462,6 +462,7 @@ impl<'a> Builder<'a> {
                 dist::LlvmTools,
                 dist::RustDev,
                 dist::Extended,
+                dist::BuildManifest,
                 dist::HashSign
             ),
             Kind::Install => describe!(

src/bootstrap/config.rs

@@ -61,6 +61,7 @@ pub struct Config {
     pub profiler: bool,
     pub ignore_git: bool,
     pub exclude: Vec<PathBuf>,
+    pub include_default_paths: bool,
     pub rustc_error_format: Option<String>,
     pub json_output: bool,
     pub test_compare_mode: bool,
@@ -532,6 +533,7 @@ impl Config {
 
         let mut config = Config::default_opts();
         config.exclude = flags.exclude;
+        config.include_default_paths = flags.include_default_paths;
         config.rustc_error_format = flags.rustc_error_format;
         config.json_output = flags.json_output;
         config.on_fail = flags.on_fail;

src/bootstrap/dist.rs

@@ -26,24 +26,7 @@ use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
 use time::{self, Timespec};
 
 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
-    if component == "cargo" {
-        format!("{}-{}", component, builder.cargo_package_vers())
-    } else if component == "rls" {
-        format!("{}-{}", component, builder.rls_package_vers())
-    } else if component == "rust-analyzer" {
-        format!("{}-{}", component, builder.rust_analyzer_package_vers())
-    } else if component == "clippy" {
-        format!("{}-{}", component, builder.clippy_package_vers())
-    } else if component == "miri" {
-        format!("{}-{}", component, builder.miri_package_vers())
-    } else if component == "rustfmt" {
-        format!("{}-{}", component, builder.rustfmt_package_vers())
-    } else if component == "llvm-tools" {
-        format!("{}-{}", component, builder.llvm_tools_package_vers())
-    } else {
-        assert!(component.starts_with("rust"));
-        format!("{}-{}", component, builder.rust_package_vers())
-    }
+    format!("{}-{}", component, builder.rust_package_vers())
 }
 
 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
@@ -2370,7 +2353,6 @@ impl Step for HashSign {
         cmd.arg(today.trim());
         cmd.arg(addr);
         cmd.arg(&builder.config.channel);
-        cmd.arg(&builder.src);
         cmd.env("BUILD_MANIFEST_LEGACY", "1");
 
         builder.create_dir(&distdir(builder));
@@ -2601,3 +2583,70 @@ impl Step for RustDev {
         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)))
     }
 }
+
+/// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
+/// release process to avoid cloning the monorepo and building stuff.
+///
+/// Should not be considered stable by end users.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct BuildManifest {
+    pub target: TargetSelection,
+}
+
+impl Step for BuildManifest {
+    type Output = PathBuf;
+    const DEFAULT: bool = false;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/build-manifest")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(BuildManifest { target: run.target });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> PathBuf {
+        let build_manifest = builder.tool_exe(Tool::BuildManifest);
+
+        let name = pkgname(builder, "build-manifest");
+        let tmp = tmpdir(builder);
+
+        // Prepare the image.
+        let image = tmp.join("build-manifest-image");
+        let image_bin = image.join("bin");
+        let _ = fs::remove_dir_all(&image);
+        t!(fs::create_dir_all(&image_bin));
+        builder.install(&build_manifest, &image_bin, 0o755);
+
+        // Prepare the overlay.
+        let overlay = tmp.join("build-manifest-overlay");
+        let _ = fs::remove_dir_all(&overlay);
+        builder.create_dir(&overlay);
+        builder.create(&overlay.join("version"), &builder.rust_version());
+        for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] {
+            builder.install(&builder.src.join(file), &overlay, 0o644);
+        }
+
+        // Create the final tarball.
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+            .arg("--product-name=Rust")
+            .arg("--rel-manifest-dir=rustlib")
+            .arg("--success-message=build-manifest installed.")
+            .arg("--image-dir")
+            .arg(&image)
+            .arg("--work-dir")
+            .arg(&tmpdir(builder))
+            .arg("--output-dir")
+            .arg(&distdir(builder))
+            .arg("--non-installed-overlay")
+            .arg(&overlay)
+            .arg(format!("--package-name={}-{}", name, self.target.triple))
+            .arg("--legacy-manifest-dirs=rustlib,cargo")
+            .arg("--component-name=build-manifest");
+
+        builder.run(&mut cmd);
+        distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple))
+    }
+}

src/bootstrap/flags.rs

@@ -29,6 +29,7 @@ pub struct Flags {
     pub cmd: Subcommand,
     pub incremental: bool,
     pub exclude: Vec<PathBuf>,
+    pub include_default_paths: bool,
     pub rustc_error_format: Option<String>,
     pub json_output: bool,
     pub dry_run: bool,
@@ -133,6 +134,11 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
         opts.optmulti("", "host", "host targets to build", "HOST");
         opts.optmulti("", "target", "target targets to build", "TARGET");
         opts.optmulti("", "exclude", "build paths to exclude", "PATH");
+        opts.optflag(
+            "",
+            "include-default-paths",
+            "include default paths in addition to the provided ones",
+        );
         opts.optopt("", "on-fail", "command to run on failure", "CMD");
         opts.optflag("", "dry-run", "dry run; don't build anything");
         opts.optopt(
@@ -601,6 +607,7 @@ Arguments:
                 .into_iter()
                 .map(|p| p.into())
                 .collect::<Vec<_>>(),
+            include_default_paths: matches.opt_present("include-default-paths"),
             deny_warnings: parse_deny_warnings(&matches),
             llvm_skip_rebuild: matches.opt_str("llvm-skip-rebuild").map(|s| s.to_lowercase()).map(
                 |s| s.parse::<bool>().expect("`llvm-skip-rebuild` should be either true or false"),

src/bootstrap/lib.rs

@@ -1051,40 +1051,6 @@ impl Build {
         self.package_vers(&self.version)
     }
 
-    /// Returns the value of `package_vers` above for Cargo
-    fn cargo_package_vers(&self) -> String {
-        self.package_vers(&self.release_num("cargo"))
-    }
-
-    /// Returns the value of `package_vers` above for rls
-    fn rls_package_vers(&self) -> String {
-        self.package_vers(&self.release_num("rls"))
-    }
-
-    /// Returns the value of `package_vers` above for rust-analyzer
-    fn rust_analyzer_package_vers(&self) -> String {
-        self.package_vers(&self.release_num("rust-analyzer/crates/rust-analyzer"))
-    }
-
-    /// Returns the value of `package_vers` above for clippy
-    fn clippy_package_vers(&self) -> String {
-        self.package_vers(&self.release_num("clippy"))
-    }
-
-    /// Returns the value of `package_vers` above for miri
-    fn miri_package_vers(&self) -> String {
-        self.package_vers(&self.release_num("miri"))
-    }
-
-    /// Returns the value of `package_vers` above for rustfmt
-    fn rustfmt_package_vers(&self) -> String {
-        self.package_vers(&self.release_num("rustfmt"))
-    }
-
-    fn llvm_tools_package_vers(&self) -> String {
-        self.package_vers(&self.version)
-    }
-
     fn llvm_tools_vers(&self) -> String {
         self.rust_version()
     }

src/bootstrap/run.rs

@@ -77,7 +77,6 @@ impl Step for BuildManifest {
         cmd.arg(today.trim());
         cmd.arg(addr);
         cmd.arg(&builder.config.channel);
-        cmd.arg(&builder.src);
 
         builder.create_dir(&distdir(builder));
         builder.run(&mut cmd);

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -98,7 +98,9 @@ ENV RUST_CONFIGURE_ARGS \
       --set llvm.thin-lto=true \
       --set llvm.ninja=false \
       --set rust.jemalloc
-ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS \
+        --include-default-paths \
+        src/tools/build-manifest
 ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang
 
 # This is the only builder which will create source tarballs