Auto merge of rust-lang#116672 - maurer:128-align, r=<try>

LLVM 18 x86 data layout update

With https://reviews.llvm.org/D86310 LLVM now has i128 aligned to 16-bytes on x86 based platforms. This will be in LLVM-18. This patch updates all our spec targets to be 16-byte aligned, and removes the alignment when speaking to older LLVM.

This results in Rust overaligning things relative to LLVM on older LLVMs.

See rust-lang#54341

compiler/rustc_ast/src/ast.rs

@@ -3238,9 +3238,13 @@ mod size_asserts {
     static_assert_size!(Impl, 136);
     static_assert_size!(Item, 136);
     static_assert_size!(ItemKind, 64);
-    static_assert_size!(LitKind, 24);
+    // This can be removed after i128:128 is in the bootstrap compiler's target.
+    #[cfg(not(bootstrap))]
+    static_assert_size!(LitKind, 32);
     static_assert_size!(Local, 72);
-    static_assert_size!(MetaItemLit, 40);
+    // This can be removed after i128:128 is in the bootstrap compiler's target.
+    #[cfg(not(bootstrap))]
+    static_assert_size!(MetaItemLit, 48);
     static_assert_size!(Param, 40);
     static_assert_size!(Pat, 72);
     static_assert_size!(Path, 24);

compiler/rustc_codegen_llvm/src/abi.rs

@@ -6,7 +6,7 @@ use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
 
-use rustc_codegen_ssa::mir::operand::OperandValue;
+use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::MemFlags;
@@ -253,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 bx.lifetime_end(llscratch, scratch_size);
             }
         } else {
-            OperandValue::Immediate(val).store(bx, dst);
+            OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst);
         }
     }
 

compiler/rustc_codegen_llvm/src/builder.rs

@@ -558,10 +558,17 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             OperandValue::Immediate(self.to_immediate(llval, place.layout))
         } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi {
             let b_offset = a.size(self).align_to(b.align(self).abi);
-            let pair_ty = place.layout.llvm_type(self);
 
             let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
-                let llptr = self.struct_gep(pair_ty, place.llval, i as u64);
+                let llptr = if i == 0 {
+                    place.llval
+                } else {
+                    self.inbounds_gep(
+                        self.type_i8(),
+                        place.llval,
+                        &[self.const_usize(b_offset.bytes())],
+                    )
+                };
                 let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
                 let load = self.load(llty, llptr, align);
                 scalar_load_metadata(self, load, scalar, layout, offset);

compiler/rustc_codegen_llvm/src/context.rs

@@ -145,6 +145,14 @@ pub unsafe fn create_module<'ll>(
                 .replace("-Fi64", "");
         }
     }
+    if llvm_version < (18, 0, 0) {
+        if sess.target.arch == "x86" || sess.target.arch == "x86_64" {
+            // LLVM 18 adjusts i128 to be 128-bit aligned on x86 variants.
+            // Earlier LLVMs leave this as default alignment, so remove it.
+            // See https://reviews.llvm.org/D86310
+            target_data_layout = target_data_layout.replace("-i128:128", "");
+        }
+    }
 
     // Ensure the data-layout values hardcoded remain the defaults.
     if sess.target.is_builtin {

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -179,7 +179,10 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 unsafe {
                     llvm::LLVMSetAlignment(load, align);
                 }
-                self.to_immediate(load, self.layout_of(tp_ty))
+                if !result.layout.is_zst() {
+                    self.store(load, result.llval, result.align);
+                }
+                return;
             }
             sym::volatile_store => {
                 let dst = args[0].deref(self.cx());

compiler/rustc_codegen_llvm/src/type_of.rs

@@ -26,16 +26,8 @@ fn uncached_llvm_type<'a, 'tcx>(
             let element = layout.scalar_llvm_type_at(cx, element);
             return cx.type_vector(element, count);
         }
-        Abi::ScalarPair(..) => {
-            return cx.type_struct(
-                &[
-                    layout.scalar_pair_element_llvm_type(cx, 0, false),
-                    layout.scalar_pair_element_llvm_type(cx, 1, false),
-                ],
-                false,
-            );
-        }
-        Abi::Uninhabited | Abi::Aggregate { .. } => {}
+        // Treat ScalarPair like a normal aggregate for the purposes of in-memory representation.
+        Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalarPair(..) => {}
     }
 
     let name = match layout.ty.kind() {
@@ -275,11 +267,25 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
     }
 
     fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type {
-        if let Abi::Scalar(scalar) = self.abi {
-            if scalar.is_bool() {
-                return cx.type_i1();
+        match self.abi {
+            Abi::Scalar(scalar) => {
+                if scalar.is_bool() {
+                    return cx.type_i1();
+                }
             }
-        }
+            Abi::ScalarPair(..) => {
+                // An immediate pair always contains just the two elements, without any padding
+                // filler, as it should never be stored to memory.
+                return cx.type_struct(
+                    &[
+                        self.scalar_pair_element_llvm_type(cx, 0, true),
+                        self.scalar_pair_element_llvm_type(cx, 1, true),
+                    ],
+                    false,
+                );
+            }
+            _ => {}
+        };
         self.llvm_type(cx)
     }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -231,14 +231,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         bx: &mut Bx,
     ) -> V {
         if let OperandValue::Pair(a, b) = self.val {
-            let llty = bx.cx().backend_type(self.layout);
+            let llty = bx.cx().immediate_backend_type(self.layout);
             debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty);
             // Reconstruct the immediate aggregate.
             let mut llpair = bx.cx().const_poison(llty);
-            let imm_a = bx.from_immediate(a);
-            let imm_b = bx.from_immediate(b);
-            llpair = bx.insert_value(llpair, imm_a, 0);
-            llpair = bx.insert_value(llpair, imm_b, 1);
+            llpair = bx.insert_value(llpair, a, 0);
+            llpair = bx.insert_value(llpair, b, 1);
             llpair
         } else {
             self.immediate()
@@ -251,14 +249,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         llval: V,
         layout: TyAndLayout<'tcx>,
     ) -> Self {
-        let val = if let Abi::ScalarPair(a, b) = layout.abi {
+        let val = if let Abi::ScalarPair(..) = layout.abi {
             debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval, layout);
 
             // Deconstruct the immediate aggregate.
             let a_llval = bx.extract_value(llval, 0);
-            let a_llval = bx.to_immediate_scalar(a_llval, a);
             let b_llval = bx.extract_value(llval, 1);
-            let b_llval = bx.to_immediate_scalar(b_llval, b);
             OperandValue::Pair(a_llval, b_llval)
         } else {
             OperandValue::Immediate(llval)
@@ -435,15 +431,14 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
                 let Abi::ScalarPair(a_scalar, b_scalar) = dest.layout.abi else {
                     bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout);
                 };
-                let ty = bx.backend_type(dest.layout);
                 let b_offset = a_scalar.size(bx).align_to(b_scalar.align(bx).abi);
 
-                let llptr = bx.struct_gep(ty, dest.llval, 0);
                 let val = bx.from_immediate(a);
                 let align = dest.align;
-                bx.store_with_flags(val, llptr, align, flags);
+                bx.store_with_flags(val, dest.llval, align, flags);
 
-                let llptr = bx.struct_gep(ty, dest.llval, 1);
+                let llptr =
+                    bx.inbounds_gep(bx.type_i8(), dest.llval, &[bx.const_usize(b_offset.bytes())]);
                 let val = bx.from_immediate(b);
                 let align = dest.align.restrict_for_offset(b_offset);
                 bx.store_with_flags(val, llptr, align, flags);

compiler/rustc_codegen_ssa/src/mir/place.rs

@@ -112,8 +112,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                     if offset == a.size(bx.cx()).align_to(b.align(bx.cx()).abi) =>
                 {
                     // Offset matches second field.
-                    let ty = bx.backend_type(self.layout);
-                    bx.struct_gep(ty, self.llval, 1)
+                    bx.inbounds_gep(bx.type_i8(), self.llval, &[bx.const_usize(offset.bytes())])
                 }
                 Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } if field.is_zst() => {
                     // ZST fields (even some that require alignment) are not included in Scalar,

compiler/rustc_middle/src/mir/mod.rs

@@ -1651,13 +1651,19 @@ mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 136);
+    // This can be removed after i128:128 is in the bootstrap compiler's target.
+    #[cfg(not(bootstrap))]
+    static_assert_size!(BasicBlockData<'_>, 144);
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 72);
     static_assert_size!(Statement<'_>, 32);
     static_assert_size!(StatementKind<'_>, 16);
-    static_assert_size!(Terminator<'_>, 104);
-    static_assert_size!(TerminatorKind<'_>, 88);
+    // This can be removed after i128:128 is in the bootstrap compiler's target.
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Terminator<'_>, 112);
+    // This can be removed after i128:128 is in the bootstrap compiler's target.
+    #[cfg(not(bootstrap))]
+    static_assert_size!(TerminatorKind<'_>, 96);
     static_assert_size!(VarDebugInfo<'_>, 88);
     // tidy-alphabetical-end
 }

compiler/rustc_target/src/spec/targets/i386_apple_ios.rs

@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: ios_sim_llvm_target(arch).into(),
         pointer_width: 32,
         data_layout: "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:128-n8:16:32-S128"
+            i128:128-f64:32:64-f80:128-n8:16:32-S128"
             .into(),
         arch: arch.target_arch(),
         options: TargetOptions { max_atomic_width: Some(64), ..opts("ios", arch) },

compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs

@@ -18,7 +18,7 @@ pub fn target() -> Target {
         llvm_target: macos_llvm_target(Arch::I686).into(),
         pointer_width: 32,
         data_layout: "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:128-n8:16:32-S128"
+            i128:128-f64:32:64-f80:128-n8:16:32-S128"
             .into(),
         arch: arch.target_arch(),
         options: TargetOptions { mcount: "\u{1}mcount".into(), ..base },

compiler/rustc_target/src/spec/targets/i686_linux_android.rs

@@ -17,7 +17,7 @@ pub fn target() -> Target {
         llvm_target: "i686-linux-android".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: TargetOptions { supported_sanitizers: SanitizerSet::ADDRESS, ..base },

compiler/rustc_target/src/spec/targets/i686_pc_windows_gnu.rs

@@ -19,7 +19,7 @@ pub fn target() -> Target {
         llvm_target: "i686-pc-windows-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            i64:64-f80:32-n8:16:32-a:0:32-S32"
+            i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_pc_windows_gnullvm.rs

@@ -18,7 +18,7 @@ pub fn target() -> Target {
         llvm_target: "i686-pc-windows-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            i64:64-f80:32-n8:16:32-a:0:32-S32"
+            i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_pc_windows_msvc.rs

@@ -24,7 +24,7 @@ pub fn target() -> Target {
         llvm_target: "i686-pc-windows-msvc".into(),
         pointer_width: 32,
         data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            i64:64-f80:128-n8:16:32-a:0:32-S32"
+            i64:64-i128:128-f80:128-n8:16:32-a:0:32-S32"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_unknown_freebsd.rs

@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-freebsd".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_unknown_haiku.rs

@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-haiku".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_unknown_hurd_gnu.rs

@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-hurd-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_unknown_linux_gnu.rs

@@ -12,7 +12,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-linux-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_unknown_linux_musl.rs

@@ -25,7 +25,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-linux-musl".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_unknown_netbsd.rs

@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-netbsdelf".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: TargetOptions { mcount: "__mcount".into(), ..base },

compiler/rustc_target/src/spec/targets/i686_unknown_openbsd.rs

@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-openbsd".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_uwp_windows_gnu.rs

@@ -18,7 +18,7 @@ pub fn target() -> Target {
         llvm_target: "i686-pc-windows-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            i64:64-f80:32-n8:16:32-a:0:32-S32"
+            i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_uwp_windows_msvc.rs

@@ -9,7 +9,7 @@ pub fn target() -> Target {
         llvm_target: "i686-pc-windows-msvc".into(),
         pointer_width: 32,
         data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            i64:64-f80:128-n8:16:32-a:0:32-S32"
+            i64:64-i128:128-f80:128-n8:16:32-a:0:32-S32"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/i686_wrs_vxworks.rs

@@ -11,7 +11,7 @@ pub fn target() -> Target {
         llvm_target: "i686-unknown-linux-gnu".into(),
         pointer_width: 32,
         data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
-            f64:32:64-f80:32-n8:16:32-S128"
+            i128:128-f64:32:64-f80:32-n8:16:32-S128"
             .into(),
         arch: "x86".into(),
         options: base,

compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs

@@ -17,8 +17,8 @@ pub fn target() -> Target {
         // correctly, we do too.
         llvm_target: macos_llvm_target(arch).into(),
         pointer_width: 64,
-        data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-            .into(),
+        data_layout:
+            "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
         arch: arch.target_arch(),
         options: TargetOptions { mcount: "\u{1}mcount".into(), ..base },
     }

compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs

@@ -9,8 +9,8 @@ pub fn target() -> Target {
     Target {
         llvm_target: ios_sim_llvm_target(arch).into(),
         pointer_width: 64,
-        data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-            .into(),
+        data_layout:
+            "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
         arch: arch.target_arch(),
         options: TargetOptions { max_atomic_width: Some(128), ..base },
     }

compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs

@@ -12,8 +12,8 @@ pub fn target() -> Target {
     Target {
         llvm_target: llvm_target.into(),
         pointer_width: 64,
-        data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-            .into(),
+        data_layout:
+            "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
         arch: arch.target_arch(),
         options: TargetOptions { max_atomic_width: Some(128), ..base },
     }