Unrolled build for rust-lang#116556

Rollup merge of rust-lang#116556 - bjorn3:sync_cg_clif-2023-10-09, r=bjorn3

Sync rustc_codegen_cranelift

The highlights this time are improved simd and inline asm support, `is_x86_feature_detected!()` returning the actual cpu features when inline asm support is enabled and a couple of bug fixes.

r? ```@ghost```

```@rustbot``` label +A-codegen +A-cranelift +T-compiler +subtree-sync

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -8,15 +8,15 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.98", features = ["unwind", "all-arch"] }
-cranelift-frontend = { version = "0.98" }
-cranelift-module = { version = "0.98" }
-cranelift-native = { version = "0.98" }
-cranelift-jit = { version = "0.98", optional = true }
-cranelift-object = { version = "0.98" }
+cranelift-codegen = { version = "0.100", features = ["unwind", "all-arch"] }
+cranelift-frontend = { version = "0.100" }
+cranelift-module = { version = "0.100" }
+cranelift-native = { version = "0.100" }
+cranelift-jit = { version = "0.100", optional = true }
+cranelift-object = { version = "0.100" }
 target-lexicon = "0.12.0"
-gimli = { version = "0.27.2", default-features = false, features = ["write"]}
-object = { version = "0.30.3", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
+gimli = { version = "0.28", default-features = false, features = ["write"]}
+object = { version = "0.32", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
 indexmap = "2.0.0"
 libloading = { version = "0.7.3", optional = true }

compiler/rustc_codegen_cranelift/Readme.md

@@ -60,18 +60,14 @@ You need to do this steps to successfully compile and use the cranelift backend
 2. Run `python x.py setup` and choose option for compiler (`b`).
 3. Build compiler and necessary tools: `python x.py build --stage=2 compiler library/std src/tools/rustdoc src/tools/rustfmt`
    * (Optional) You can also build cargo by adding `src/tools/cargo` to previous command.
-4. Copy exectutable files from `./build/host/stage2-tools/<your hostname triple>/release`
-to `./build/host/stage2/bin/`. Note that you would need to do this every time you rebuilt `rust` repository.
-5. Copy cargo from another toolchain: `cp $(rustup which cargo) .build/<your hostname triple>/stage2/bin/cargo`
-   * Another option is to build it at step 3 and copy with other executables at step 4.
-6. Link your new `rustc` to toolchain: `rustup toolchain link stage2 ./build/host/stage2/`.
-7. (Windows only) compile the build system: `rustc +stage2 -O build_system/main.rs -o y.exe`.
-8. You need to prefix every `./y.sh` (or `y` if you built `build_system/main.rs` as `y`) command by `rustup run stage2` to make cg_clif use your local changes in rustc.
-
+4. Copy cargo from a nightly toolchain: `cp $(rustup +nightly which cargo) ./build/host/stage2/bin/cargo`. Note that you would need to do this every time you rebuilt `rust` repository.
+5. Link your new `rustc` to toolchain: `rustup toolchain link stage2 ./build/host/stage2/`.
+6. (Windows only) compile the build system: `rustc +stage2 -O build_system/main.rs -o y.exe`.
+7. You need to prefix every `./y.sh` (or `y` if you built `build_system/main.rs` as `y`) command by `rustup run stage2` to make cg_clif use your local changes in rustc.
   * `rustup run stage2 ./y.sh prepare`
   * `rustup run stage2 ./y.sh build`
   * (Optional) run tests: `rustup run stage2 ./y.sh test`
-9. Now you can use your cg_clif build to compile other Rust programs, e.g. you can open any Rust crate and run commands like `$RustCheckoutDir/compiler/rustc_codegen_cranelift/dist/cargo-clif build --release`.
+8. Now you can use your cg_clif build to compile other Rust programs, e.g. you can open any Rust crate and run commands like `$RustCheckoutDir/compiler/rustc_codegen_cranelift/dist/cargo-clif build --release`.
 
 ## Configuration
 

compiler/rustc_codegen_cranelift/build_system/build_backend.rs

@@ -20,6 +20,8 @@ pub(crate) fn build_backend(
 
     let mut rustflags = rustflags_from_env("RUSTFLAGS");
 
+    rustflags.push("-Zallow-features=rustc_private".to_owned());
+
     if is_ci() {
         // Deny warnings on CI
         rustflags.push("-Dwarnings".to_owned());

compiler/rustc_codegen_cranelift/build_system/prepare.rs

@@ -122,10 +122,10 @@ impl GitRepo {
         if download_dir.exists() {
             let actual_hash = format!("{:016x}", hash_dir(&download_dir));
             if actual_hash == self.content_hash {
-                println!("[FRESH] {}", download_dir.display());
+                eprintln!("[FRESH] {}", download_dir.display());
                 return;
             } else {
-                println!(
+                eprintln!(
                     "Mismatched content hash for {download_dir}: {actual_hash} != {content_hash}. Downloading again.",
                     download_dir = download_dir.display(),
                     content_hash = self.content_hash,
@@ -150,7 +150,7 @@ impl GitRepo {
 
         let actual_hash = format!("{:016x}", hash_dir(&download_dir));
         if actual_hash != self.content_hash {
-            println!(
+            eprintln!(
                 "Download of {download_dir} failed with mismatched content hash: {actual_hash} != {content_hash}",
                 download_dir = download_dir.display(),
                 content_hash = self.content_hash,

compiler/rustc_codegen_cranelift/build_system/tests.rs

@@ -9,7 +9,7 @@ use crate::path::{Dirs, RelPath};
 use crate::prepare::{apply_patches, GitRepo};
 use crate::rustc_info::get_default_sysroot;
 use crate::shared_utils::rustflags_from_env;
-use crate::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler, LogGroup};
+use crate::utils::{spawn_and_wait, CargoProject, Compiler, LogGroup};
 use crate::{CodegenBackend, SysrootKind};
 
 static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example");
@@ -101,13 +101,11 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::build_bin("aot.issue-59326", "example/issue-59326.rs"),
 ];
 
-// FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is
-// fixed.
 pub(crate) static RAND_REPO: GitRepo = GitRepo::github(
     "rust-random",
     "rand",
-    "50b9a447410860af8d6db9a208c3576886955874",
-    "446203b96054891e",
+    "f3dd0b885c4597b9617ca79987a0dd899ab29fcb",
+    "3f869e4fcd602b66",
     "rand",
 );
 
@@ -116,8 +114,8 @@ pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir()
 pub(crate) static REGEX_REPO: GitRepo = GitRepo::github(
     "rust-lang",
     "regex",
-    "32fed9429eafba0ae92a64b01796a0c5a75b88c8",
-    "fcc4df7c5b902633",
+    "061ee815ef2c44101dba7b0b124600fcb03c1912",
+    "dc26aefbeeac03ca",
     "regex",
 );
 
@@ -126,8 +124,8 @@ pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir
 pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
     "rust-lang",
     "portable-simd",
-    "7c7dbe0c505ccbc02ff30c1e37381ab1d47bf46f",
-    "5bcc9c544f6fa7bd",
+    "4825b2a64d765317066948867e8714674419359b",
+    "8b188cc41f5af835",
     "portable-simd",
 );
 
@@ -180,40 +178,6 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
             spawn_and_wait(build_cmd);
         }
     }),
-    TestCase::custom("test.regex-shootout-regex-dna", &|runner| {
-        REGEX_REPO.patch(&runner.dirs);
-
-        REGEX.clean(&runner.dirs);
-
-        let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs);
-        build_cmd.arg("--example").arg("shootout-regex-dna");
-        spawn_and_wait(build_cmd);
-
-        if runner.is_native {
-            let mut run_cmd = REGEX.run(&runner.target_compiler, &runner.dirs);
-            run_cmd.arg("--example").arg("shootout-regex-dna");
-
-            let input = fs::read_to_string(
-                REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-input.txt"),
-            )
-            .unwrap();
-            let expected = fs::read_to_string(
-                REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-output.txt"),
-            )
-            .unwrap();
-
-            let output = spawn_and_wait_with_input(run_cmd, input);
-
-            let output_matches = expected.lines().eq(output.lines());
-            if !output_matches {
-                println!("Output files don't match!");
-                println!("Expected Output:\n{}", expected);
-                println!("Actual Output:\n{}", output);
-
-                std::process::exit(1);
-            }
-        }
-    }),
     TestCase::custom("test.regex", &|runner| {
         REGEX_REPO.patch(&runner.dirs);
 
@@ -223,7 +187,22 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
             let mut run_cmd = REGEX.test(&runner.target_compiler, &runner.dirs);
             // regex-capi and regex-debug don't have any tests. Nor do they contain any code
             // that is useful to test with cg_clif. Skip building them to reduce test time.
-            run_cmd.args(["-p", "regex", "-p", "regex-syntax", "--", "-q"]);
+            run_cmd.args([
+                "-p",
+                "regex",
+                "-p",
+                "regex-syntax",
+                "--release",
+                "--all-targets",
+                "--",
+                "-q",
+            ]);
+            spawn_and_wait(run_cmd);
+
+            let mut run_cmd = REGEX.test(&runner.target_compiler, &runner.dirs);
+            // don't run integration tests for regex-autonata. they take like 2min each without
+            // much extra coverage of simd usage.
+            run_cmd.args(["-p", "regex-automata", "--release", "--lib", "--", "-q"]);
             spawn_and_wait(run_cmd);
         } else {
             eprintln!("Cross-Compiling: Not running tests");

compiler/rustc_codegen_cranelift/build_system/utils.rs

@@ -1,8 +1,8 @@
 use std::env;
 use std::fs;
-use std::io::{self, Write};
+use std::io;
 use std::path::{Path, PathBuf};
-use std::process::{self, Command, Stdio};
+use std::process::{self, Command};
 use std::sync::atomic::{AtomicBool, Ordering};
 
 use crate::path::{Dirs, RelPath};
@@ -47,7 +47,7 @@ impl Compiler {
                 self.runner = vec!["wine".to_owned()];
             }
             _ => {
-                println!("Unknown non-native platform");
+                eprintln!("Unknown non-native platform");
             }
         }
     }
@@ -197,7 +197,9 @@ pub(crate) fn try_hard_link(src: impl AsRef<Path>, dst: impl AsRef<Path>) {
 
 #[track_caller]
 pub(crate) fn spawn_and_wait(mut cmd: Command) {
-    if !cmd.spawn().unwrap().wait().unwrap().success() {
+    let status = cmd.spawn().unwrap().wait().unwrap();
+    if !status.success() {
+        eprintln!("{cmd:?} exited with status {:?}", status);
         process::exit(1);
     }
 }
@@ -207,38 +209,17 @@ pub(crate) fn spawn_and_wait(mut cmd: Command) {
 pub(crate) fn retry_spawn_and_wait(tries: u64, mut cmd: Command) {
     for i in 1..tries + 1 {
         if i != 1 {
-            println!("Command failed. Attempt {i}/{tries}:");
+            eprintln!("Command failed. Attempt {i}/{tries}:");
         }
         if cmd.spawn().unwrap().wait().unwrap().success() {
             return;
         }
         std::thread::sleep(std::time::Duration::from_secs(i * 5));
     }
-    println!("The command has failed after {tries} attempts.");
+    eprintln!("The command has failed after {tries} attempts.");
     process::exit(1);
 }
 
-#[track_caller]
-pub(crate) fn spawn_and_wait_with_input(mut cmd: Command, input: String) -> String {
-    let mut child = cmd
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .spawn()
-        .expect("Failed to spawn child process");
-
-    let mut stdin = child.stdin.take().expect("Failed to open stdin");
-    std::thread::spawn(move || {
-        stdin.write_all(input.as_bytes()).expect("Failed to write to stdin");
-    });
-
-    let output = child.wait_with_output().expect("Failed to read stdout");
-    if !output.status.success() {
-        process::exit(1);
-    }
-
-    String::from_utf8(output.stdout).unwrap()
-}
-
 pub(crate) fn remove_dir_if_exists(path: &Path) {
     match fs::remove_dir_all(&path) {
         Ok(()) => {}

compiler/rustc_codegen_cranelift/config.txt

@@ -46,6 +46,5 @@ aot.issue-59326
 testsuite.extended_sysroot
 test.rust-random/rand
 test.libcore
-test.regex-shootout-regex-dna
 test.regex
 test.portable-simd

compiler/rustc_codegen_cranelift/example/mini_core.rs

@@ -683,6 +683,12 @@ pub macro cfg() {
     /* compiler built-in */
 }
 
+#[rustc_builtin_macro]
+#[rustc_macro_transparency = "semitransparent"]
+pub macro asm() {
+    /* compiler built-in */
+}
+
 #[rustc_builtin_macro]
 #[rustc_macro_transparency = "semitransparent"]
 pub macro global_asm() {

compiler/rustc_codegen_cranelift/patches/0001-regex-Ignore-test-which-gets-miscompiled-with-llvm-sysroot.patch

@@ -0,0 +1,25 @@
+From 5d4afb8d807d181038b6a004d17ed055a8d191b2 Mon Sep 17 00:00:00 2001
+From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
+Date: Mon, 2 Oct 2023 13:59:00 +0000
+Subject: [PATCH] Ignore test which gets miscompiled with llvm sysroot
+
+---
+ regex-automata/src/util/pool.rs | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/regex-automata/src/util/pool.rs b/regex-automata/src/util/pool.rs
+index c03d7b0..28b233b 100644
+--- a/regex-automata/src/util/pool.rs
++++ b/regex-automata/src/util/pool.rs
+@@ -1081,6 +1081,8 @@ mod tests {
+     // into the pool. This in turn resulted in this test producing a data race.
+     #[cfg(feature = "std")]
+     #[test]
++    // FIXME(rustc_codegen_cranelift#1395) miscompilation of thread::scope with LLVM sysroot
++    #[ignore]
+     fn thread_owner_sync() {
+         let pool = Pool::new(|| vec!['a']);
+         {
+-- 
+2.34.1
+

compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch

@@ -19,9 +19,9 @@ index 897a5e9..331f66f 100644
  #![feature(const_option_ext)]
  #![feature(const_result)]
 -#![cfg_attr(target_has_atomic = "128", feature(integer_atomics))]
+ #![cfg_attr(test, feature(cfg_match))]
  #![feature(int_roundings)]
  #![feature(slice_group_by)]
- #![feature(split_array)]
 diff --git a/atomic.rs b/atomic.rs
 index b735957..ea728b6 100644
 --- a/atomic.rs