Merge with upstream

.github/workflows/main.yml

@@ -240,7 +240,6 @@ jobs:
           fi
           if grep -q pulley names.log; then
             echo test-nightly=true >> $GITHUB_OUTPUT
-            echo test-miri=true >> $GITHUB_OUTPUT
           fi
         fi
         matrix="$(node ./ci/build-test-matrix.js ./commits.log ./names.log $run_full)"
@@ -876,7 +875,8 @@ jobs:
         # workaround for https://bugs.launchpad.net/ubuntu/+source/llvm-defaults/+bug/1972855
         sudo mkdir -p /usr/lib/local/lib/python3.10/dist-packages/lldb
         sudo ln -s /usr/lib/llvm-15/lib/python3.10/dist-packages/lldb/* /usr/lib/python3/dist-packages/lldb/
-        cargo test --test all -- --ignored --test-threads 1 debug::
+        # Only testing release since it is more likely to expose issues with our low-level symbol handling.
+        cargo test --release --test all -- --ignored --test-threads 1 debug::
       env:
         LLDB: lldb-15 # override default version, 14
 

Cargo.toml

@@ -237,6 +237,7 @@ wasi-common = { path = "crates/wasi-common", version = "=29.0.0", default-featur
 wasmtime-fuzzing = { path = "crates/fuzzing" }
 wasmtime-jit-icache-coherence = { path = "crates/jit-icache-coherence", version = "=29.0.0" }
 wasmtime-wit-bindgen = { path = "crates/wit-bindgen", version = "=29.0.0" }
+wasmtime-math = { path = "crates/math", version = "=29.0.0" }
 test-programs-artifacts = { path = 'crates/test-programs/artifacts' }
 
 pulley-interpreter = { path = 'pulley', version = "=29.0.0" }
@@ -356,6 +357,7 @@ rustc-hash = "2.0.0"
 libtest-mimic = "0.7.0"
 semver = { version = "1.0.17", default-features = false }
 ittapi = "0.4.0"
+libm = "0.2.7"
 
 # =============================================================================
 #

ci/vendor-wit.sh

@@ -37,22 +37,22 @@ make_vendor() {
 cache_dir=$(mktemp -d)
 
 make_vendor "wasi" "
-  cli@v0.2.2
-  clocks@v0.2.2
-  filesystem@v0.2.2
-  io@v0.2.2
-  random@v0.2.2
-  sockets@v0.2.2
+  cli@v0.2.3
+  clocks@v0.2.3
+  filesystem@v0.2.3
+  io@v0.2.3
+  random@v0.2.3
+  sockets@v0.2.3
 "
 
 make_vendor "wasi-http" "
-  cli@v0.2.2
-  clocks@v0.2.2
-  filesystem@v0.2.2
-  io@v0.2.2
-  random@v0.2.2
-  sockets@v0.2.2
-  http@v0.2.2
+  cli@v0.2.3
+  clocks@v0.2.3
+  filesystem@v0.2.3
+  io@v0.2.3
+  random@v0.2.3
+  sockets@v0.2.3
+  http@v0.2.3
 "
 
 make_vendor "wasi-config" "config@f4d699b"

cranelift/bitset/src/scalar.rs

@@ -556,14 +556,17 @@ pub trait ScalarBitSetStorage:
 macro_rules! impl_storage {
     ( $int:ty ) => {
         impl ScalarBitSetStorage for $int {
+            #[inline]
             fn leading_zeros(self) -> u8 {
                 u8::try_from(self.leading_zeros()).unwrap()
             }
 
+            #[inline]
             fn trailing_zeros(self) -> u8 {
                 u8::try_from(self.trailing_zeros()).unwrap()
             }
 
+            #[inline]
             fn count_ones(self) -> u8 {
                 u8::try_from(self.count_ones()).unwrap()
             }

cranelift/codegen/meta/src/pulley.rs

@@ -27,23 +27,52 @@ const OPS: &[Inst<'_>] = pulley_interpreter::for_each_op!(define);
 const EXTENDED_OPS: &[Inst<'_>] = pulley_interpreter::for_each_extended_op!(define);
 
 enum Operand<'a> {
-    Normal { name: &'a str, ty: &'a str },
-    Writable { name: &'a str, ty: &'a str },
-    TrapCode { name: &'a str, ty: &'a str },
-    Binop { reg: &'a str },
+    Normal {
+        name: &'a str,
+        ty: &'a str,
+    },
+    Writable {
+        name: &'a str,
+        ty: &'a str,
+    },
+    TrapCode {
+        name: &'a str,
+        ty: &'a str,
+    },
+    Binop {
+        dst: &'a str,
+        src1: &'a str,
+        src2: &'a str,
+    },
 }
 
 impl Inst<'_> {
     fn operands(&self) -> impl Iterator<Item = Operand<'_>> {
         self.fields
             .iter()
             .map(|(name, ty)| match (*name, *ty) {
-                ("operands", "BinaryOperands < XReg >") => Operand::Binop { reg: "XReg" },
+                ("operands", binop) => {
+                    // Parse "BinaryOperands < A >"` as A/A/A
+                    // Parse "BinaryOperands < A, B >"` as A/B/A
+                    // Parse "BinaryOperands < A, B, C >"` as A/B/C
+                    let mut parts = binop
+                        .strip_prefix("BinaryOperands <")
+                        .unwrap()
+                        .strip_suffix(">")
+                        .unwrap()
+                        .trim()
+                        .split(',')
+                        .map(|x| x.trim());
+                    let dst = parts.next().unwrap();
+                    let src1 = parts.next().unwrap_or(dst);
+                    let src2 = parts.next().unwrap_or(dst);
+                    Operand::Binop { dst, src1, src2 }
+                }
+                ("dst", ty) => Operand::Writable { name, ty },
                 (name, "RegSet < XReg >") => Operand::Normal {
                     name,
-                    ty: "VecXReg",
+                    ty: "XRegSet",
                 },
-                ("dst", ty) => Operand::Writable { name, ty },
                 (name, ty) => Operand::Normal { name, ty },
             })
             .chain(if self.name.contains("Trap") {
@@ -95,10 +124,17 @@ pub fn generate_rust(filename: &str, out_dir: &Path) -> Result<(), Error> {
                     if i > 0 {
                         format_string.push_str(",");
                     }
+
+                    if ty == "XRegSet" {
+                        format_string.push_str(" {");
+                        format_string.push_str(name);
+                        format_string.push_str(":?}");
+                        continue;
+                    }
+
                     format_string.push_str(" {");
                     format_string.push_str(name);
                     format_string.push_str("}");
-
                     if ty.contains("Reg") {
                         if name == "dst" {
                             locals.push_str(&format!("let {name} = reg_name(*{name}.to_reg());\n"));
@@ -112,12 +148,14 @@ pub fn generate_rust(filename: &str, out_dir: &Path) -> Result<(), Error> {
                     pat.push_str(",");
                     format_string.push_str(&format!(" // trap={{{name}:?}}"));
                 }
-                Operand::Binop { reg: _ } => {
+                Operand::Binop { src2, .. } => {
                     pat.push_str("dst, src1, src2,");
                     format_string.push_str(" {dst}, {src1}, {src2}");
                     locals.push_str(&format!("let dst = reg_name(*dst.to_reg());\n"));
                     locals.push_str(&format!("let src1 = reg_name(**src1);\n"));
-                    locals.push_str(&format!("let src2 = reg_name(**src2);\n"));
+                    if src2.contains("Reg") {
+                        locals.push_str(&format!("let src2 = reg_name(**src2);\n"));
+                    }
                 }
             }
         }
@@ -149,6 +187,13 @@ pub fn generate_rust(filename: &str, out_dir: &Path) -> Result<(), Error> {
         let mut defs = Vec::new();
         for op in inst.operands() {
             match op {
+                // `{Push,Pop}Frame{Save,Restore}` doesn't participate in
+                // register allocation.
+                Operand::Normal {
+                    name: _,
+                    ty: "XRegSet",
+                } if *name == "PushFrameSave" || *name == "PopFrameRestore" => {}
+
                 Operand::Normal { name, ty } => {
                     if ty.contains("Reg") {
                         uses.push(name);
@@ -164,11 +209,14 @@ pub fn generate_rust(filename: &str, out_dir: &Path) -> Result<(), Error> {
                     }
                 }
                 Operand::TrapCode { .. } => {}
-                Operand::Binop { reg: _ } => {
-                    pat.push_str("dst, src1, src2,");
+                Operand::Binop { src2, .. } => {
+                    pat.push_str("dst, src1,");
                     uses.push("src1");
-                    uses.push("src2");
                     defs.push("dst");
+                    if src2.contains("Reg") {
+                        pat.push_str("src2,");
+                        uses.push("src2");
+                    }
                 }
             }
         }
@@ -224,7 +272,7 @@ pub fn generate_rust(filename: &str, out_dir: &Path) -> Result<(), Error> {
                     pat.push_str(",");
                     trap.push_str(&format!("sink.add_trap({name});\n"));
                 }
-                Operand::Binop { reg: _ } => {
+                Operand::Binop { .. } => {
                     pat.push_str("dst, src1, src2,");
                     args.push_str(
                         "pulley_interpreter::regs::BinaryOperands::new(dst, src1, src2),",
@@ -268,10 +316,10 @@ pub fn generate_isle(filename: &str, out_dir: &Path) -> Result<(), Error> {
                 Operand::Writable { name, ty } => {
                     isle.push_str(&format!("\n    ({name} Writable{ty})"));
                 }
-                Operand::Binop { reg } => {
-                    isle.push_str(&format!("\n    (dst Writable{reg})"));
-                    isle.push_str(&format!("\n    (src1 {reg})"));
-                    isle.push_str(&format!("\n    (src2 {reg})"));
+                Operand::Binop { dst, src1, src2 } => {
+                    isle.push_str(&format!("\n    (dst Writable{dst})"));
+                    isle.push_str(&format!("\n    (src1 {src1})"));
+                    isle.push_str(&format!("\n    (src2 {src2})"));
                 }
             }
         }
@@ -306,13 +354,13 @@ pub fn generate_isle(filename: &str, out_dir: &Path) -> Result<(), Error> {
                     assert!(result.is_none(), "{} has >1 result", inst.snake_name);
                     result = Some(ty);
                 }
-                Operand::Binop { reg } => {
-                    isle.push_str(&format!("{reg} {reg}"));
+                Operand::Binop { dst, src1, src2 } => {
+                    isle.push_str(&format!("{src1} {src2}"));
                     rule.push_str("src1 src2");
                     ops.push("src1");
                     ops.push("src2");
                     assert!(result.is_none(), "{} has >1 result", inst.snake_name);
-                    result = Some(reg);
+                    result = Some(dst);
                 }
             }
             isle.push_str(" ");

cranelift/codegen/src/ir/immediates.rs

@@ -10,6 +10,7 @@ use core::fmt::{self, Display, Formatter};
 use core::ops::{Add, BitAnd, BitOr, BitXor, Div, Mul, Neg, Not, Sub};
 use core::str::FromStr;
 use core::{i32, u32};
+use cranelift_entity::{Signed, Unsigned};
 #[cfg(feature = "enable-serde")]
 use serde_derive::{Deserialize, Serialize};
 
@@ -92,7 +93,7 @@ impl Imm64 {
     /// Sign extend this immediate as if it were a signed integer of the given
     /// power-of-two width.
     #[must_use]
-    pub(crate) fn sign_extend_from_width(&self, bit_width: u32) -> Self {
+    pub fn sign_extend_from_width(&self, bit_width: u32) -> Self {
         debug_assert!(
             bit_width.is_power_of_two(),
             "{bit_width} is not a power of two"
@@ -107,6 +108,25 @@ impl Imm64 {
         let sign_extended = (self.0 << delta) >> delta;
         Imm64(sign_extended)
     }
+
+    /// Zero extend this immediate as if it were an unsigned integer of the
+    /// given power-of-two width.
+    #[must_use]
+    pub fn zero_extend_from_width(&self, bit_width: u32) -> Self {
+        debug_assert!(
+            bit_width.is_power_of_two(),
+            "{bit_width} is not a power of two"
+        );
+
+        if bit_width >= 64 {
+            return *self;
+        }
+
+        let bit_width = u64::from(bit_width);
+        let delta = 64 - bit_width;
+        let zero_extended = (self.0.unsigned() << delta) >> delta;
+        Imm64(zero_extended.signed())
+    }
 }
 
 impl From<Imm64> for i64 {

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -705,8 +705,12 @@ impl ABIMachineSpec for AArch64MachineDeps {
         // Set this to 3 to keep the max size of the probe to 6 instructions.
         const PROBE_MAX_UNROLL: u32 = 3;
 
-        let probe_count = align_to(frame_size, guard_size) / guard_size;
-        if probe_count <= PROBE_MAX_UNROLL {
+        // Calculate how many probes we need to perform. Round down, as we only
+        // need to probe whole guard_size regions we'd otherwise skip over.
+        let probe_count = frame_size / guard_size;
+        if probe_count == 0 {
+            // No probe necessary
+        } else if probe_count <= PROBE_MAX_UNROLL {
             Self::gen_probestack_unroll(insts, guard_size, probe_count)
         } else {
             Self::gen_probestack_loop(insts, frame_size, guard_size)