Auto merge of #53671 - RalfJung:miri-refactor, r=oli-obk

Miri engine cleanup

* Unify the two maps in memory to store the allocation and its kind together.
* Share the handling of statics between CTFE and miri: The miri engine always
      uses "lazy" `AllocType::Static` when encountering a static.  Acessing that
      static invokes CTFE (no matter the machine).  The machine only has any
      influence when writing to a static, which CTFE outright rejects (but miri
      makes a copy-on-write).
* Add an `AllocId` to by-ref consts so miri can use them as operands without
      making copies.
* Move responsibilities around for the `eval_fn_call` machine hook: The hook
      just has to find the MIR (or entirely take care of everything); pushing the
      new stack frame is taken care of by the miri engine.
* Expose the intrinsics and lang items implemented by CTFE so miri does not
      have to reimplement them.
* Allow Machine to hook into foreign statics (used by miri to get rid of some other hacks).
* Clean up function calling.
* Switch const sanity check to work on operands, not mplaces.
* Move const_eval out of rustc_mir::interpret, to make sure that it does not access private implementation details.

In particular, we can finally make `eval_operand` take `&self`. :-)

Should be merged after #53609, across which I will rebase.

src/librustc/ich/impls_ty.rs

@@ -384,7 +384,8 @@ for ::mir::interpret::ConstValue<'gcx> {
                 a.hash_stable(hcx, hasher);
                 b.hash_stable(hcx, hasher);
             }
-            ByRef(alloc, offset) => {
+            ByRef(id, alloc, offset) => {
+                id.hash_stable(hcx, hasher);
                 alloc.hash_stable(hcx, hasher);
                 offset.hash_stable(hcx, hasher);
             }
@@ -446,7 +447,7 @@ impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::Allocation {
         }
         self.undef_mask.hash_stable(hcx, hasher);
         self.align.hash_stable(hcx, hasher);
-        self.runtime_mutability.hash_stable(hcx, hasher);
+        self.mutability.hash_stable(hcx, hasher);
     }
 }
 
@@ -516,7 +517,6 @@ for ::mir::interpret::EvalErrorKind<'gcx, O> {
             InvalidMemoryAccess |
             InvalidFunctionPointer |
             InvalidBool |
-            InvalidDiscriminant |
             InvalidNullPointerUsage |
             ReadPointerAsBytes |
             ReadBytesAsPointer |
@@ -549,6 +549,7 @@ for ::mir::interpret::EvalErrorKind<'gcx, O> {
             GeneratorResumedAfterReturn |
             GeneratorResumedAfterPanic |
             InfiniteLoop => {}
+            InvalidDiscriminant(val) => val.hash_stable(hcx, hasher),
             Panic { ref msg, ref file, line, col } => {
                 msg.hash_stable(hcx, hasher);
                 file.hash_stable(hcx, hasher);

src/librustc/mir/interpret/error.rs

@@ -190,7 +190,7 @@ pub enum EvalErrorKind<'tcx, O> {
     InvalidMemoryAccess,
     InvalidFunctionPointer,
     InvalidBool,
-    InvalidDiscriminant,
+    InvalidDiscriminant(u128),
     PointerOutOfBounds {
         ptr: Pointer,
         access: bool,
@@ -302,7 +302,7 @@ impl<'tcx, O> EvalErrorKind<'tcx, O> {
                 "tried to use a function pointer after offsetting it",
             InvalidBool =>
                 "invalid boolean value read",
-            InvalidDiscriminant =>
+            InvalidDiscriminant(..) =>
                 "invalid enum discriminant value read",
             PointerOutOfBounds { .. } =>
                 "pointer offset outside bounds of allocation",
@@ -488,6 +488,8 @@ impl<'tcx, O: fmt::Debug> fmt::Debug for EvalErrorKind<'tcx, O> {
                        align {}", size.bytes(), align.abi(), size2.bytes(), align2.abi()),
             Panic { ref msg, line, col, ref file } =>
                 write!(f, "the evaluated program panicked at '{}', {}:{}:{}", msg, file, line, col),
+            InvalidDiscriminant(val) =>
+                write!(f, "encountered invalid enum discriminant {}", val),
             _ => write!(f, "{}", self.description()),
         }
     }

src/librustc/mir/interpret/mod.rs

@@ -393,7 +393,8 @@ impl fmt::Display for AllocId {
 pub enum AllocType<'tcx, M> {
     /// The alloc id is used as a function pointer
     Function(Instance<'tcx>),
-    /// The alloc id points to a static variable
+    /// The alloc id points to a "lazy" static variable that did not get computed (yet).
+    /// This is also used to break the cycle in recursive statics.
     Static(DefId),
     /// The alloc id points to memory
     Memory(M)
@@ -496,13 +497,14 @@ pub struct Allocation {
     pub undef_mask: UndefMask,
     /// The alignment of the allocation to detect unaligned reads.
     pub align: Align,
-    /// Whether the allocation (of a static) should be put into mutable memory when codegenning
-    ///
-    /// Only happens for `static mut` or `static` with interior mutability
-    pub runtime_mutability: Mutability,
+    /// Whether the allocation is mutable.
+    /// Also used by codegen to determine if a static should be put into mutable memory,
+    /// which happens for `static mut` and `static` with interior mutability.
+    pub mutability: Mutability,
 }
 
 impl Allocation {
+    /// Creates a read-only allocation initialized by the given bytes
     pub fn from_bytes(slice: &[u8], align: Align) -> Self {
         let mut undef_mask = UndefMask::new(Size::ZERO);
         undef_mask.grow(Size::from_bytes(slice.len() as u64), true);
@@ -511,7 +513,7 @@ impl Allocation {
             relocations: Relocations::new(),
             undef_mask,
             align,
-            runtime_mutability: Mutability::Immutable,
+            mutability: Mutability::Immutable,
         }
     }
 
@@ -526,7 +528,7 @@ impl Allocation {
             relocations: Relocations::new(),
             undef_mask: UndefMask::new(size),
             align,
-            runtime_mutability: Mutability::Immutable,
+            mutability: Mutability::Mutable,
         }
     }
 }

src/librustc/mir/interpret/value.rs

@@ -14,7 +14,7 @@ use ty::layout::{HasDataLayout, Size};
 use ty::subst::Substs;
 use hir::def_id::DefId;
 
-use super::{EvalResult, Pointer, PointerArithmetic, Allocation};
+use super::{EvalResult, Pointer, PointerArithmetic, Allocation, AllocId, sign_extend};
 
 /// Represents a constant value in Rust. Scalar and ScalarPair are optimizations which
 /// matches the LocalValue optimizations for easy conversions between Value and ConstValue.
@@ -32,8 +32,9 @@ pub enum ConstValue<'tcx> {
     ///
     /// The second field may be undef in case of `Option<usize>::None`
     ScalarPair(Scalar, ScalarMaybeUndef),
-    /// Used only for the remaining cases. An allocation + offset into the allocation
-    ByRef(&'tcx Allocation, Size),
+    /// Used only for the remaining cases. An allocation + offset into the allocation.
+    /// Invariant: The AllocId matches the allocation.
+    ByRef(AllocId, &'tcx Allocation, Size),
 }
 
 impl<'tcx> ConstValue<'tcx> {
@@ -185,6 +186,57 @@ impl<'tcx> Scalar {
             _ => err!(InvalidBool),
         }
     }
+
+    pub fn to_char(self) -> EvalResult<'tcx, char> {
+        let val = self.to_u32()?;
+        match ::std::char::from_u32(val) {
+            Some(c) => Ok(c),
+            None => err!(InvalidChar(val as u128)),
+        }
+    }
+
+    pub fn to_u8(self) -> EvalResult<'static, u8> {
+        let sz = Size::from_bits(8);
+        let b = self.to_bits(sz)?;
+        assert_eq!(b as u8 as u128, b);
+        Ok(b as u8)
+    }
+
+    pub fn to_u32(self) -> EvalResult<'static, u32> {
+        let sz = Size::from_bits(32);
+        let b = self.to_bits(sz)?;
+        assert_eq!(b as u32 as u128, b);
+        Ok(b as u32)
+    }
+
+    pub fn to_usize(self, cx: impl HasDataLayout) -> EvalResult<'static, u64> {
+        let b = self.to_bits(cx.data_layout().pointer_size)?;
+        assert_eq!(b as u64 as u128, b);
+        Ok(b as u64)
+    }
+
+    pub fn to_i8(self) -> EvalResult<'static, i8> {
+        let sz = Size::from_bits(8);
+        let b = self.to_bits(sz)?;
+        let b = sign_extend(b, sz) as i128;
+        assert_eq!(b as i8 as i128, b);
+        Ok(b as i8)
+    }
+
+    pub fn to_i32(self) -> EvalResult<'static, i32> {
+        let sz = Size::from_bits(32);
+        let b = self.to_bits(sz)?;
+        let b = sign_extend(b, sz) as i128;
+        assert_eq!(b as i32 as i128, b);
+        Ok(b as i32)
+    }
+
+    pub fn to_isize(self, cx: impl HasDataLayout) -> EvalResult<'static, i64> {
+        let b = self.to_bits(cx.data_layout().pointer_size)?;
+        let b = sign_extend(b, cx.data_layout().pointer_size) as i128;
+        assert_eq!(b as i64 as i128, b);
+        Ok(b as i64)
+    }
 }
 
 impl From<Pointer> for Scalar {
@@ -228,22 +280,61 @@ impl From<Scalar> for ScalarMaybeUndef {
 }
 
 impl<'tcx> ScalarMaybeUndef {
+    #[inline]
     pub fn not_undef(self) -> EvalResult<'static, Scalar> {
         match self {
             ScalarMaybeUndef::Scalar(scalar) => Ok(scalar),
             ScalarMaybeUndef::Undef => err!(ReadUndefBytes),
         }
     }
 
+    #[inline(always)]
     pub fn to_ptr(self) -> EvalResult<'tcx, Pointer> {
         self.not_undef()?.to_ptr()
     }
 
+    #[inline(always)]
     pub fn to_bits(self, target_size: Size) -> EvalResult<'tcx, u128> {
         self.not_undef()?.to_bits(target_size)
     }
 
+    #[inline(always)]
     pub fn to_bool(self) -> EvalResult<'tcx, bool> {
         self.not_undef()?.to_bool()
     }
+
+    #[inline(always)]
+    pub fn to_char(self) -> EvalResult<'tcx, char> {
+        self.not_undef()?.to_char()
+    }
+
+    #[inline(always)]
+    pub fn to_u8(self) -> EvalResult<'tcx, u8> {
+        self.not_undef()?.to_u8()
+    }
+
+    #[inline(always)]
+    pub fn to_u32(self) -> EvalResult<'tcx, u32> {
+        self.not_undef()?.to_u32()
+    }
+
+    #[inline(always)]
+    pub fn to_usize(self, cx: impl HasDataLayout) -> EvalResult<'tcx, u64> {
+        self.not_undef()?.to_usize(cx)
+    }
+
+    #[inline(always)]
+    pub fn to_i8(self) -> EvalResult<'tcx, i8> {
+        self.not_undef()?.to_i8()
+    }
+
+    #[inline(always)]
+    pub fn to_i32(self) -> EvalResult<'tcx, i32> {
+        self.not_undef()?.to_i32()
+    }
+
+    #[inline(always)]
+    pub fn to_isize(self, cx: impl HasDataLayout) -> EvalResult<'tcx, i64> {
+        self.not_undef()?.to_isize(cx)
+    }
 }

src/librustc/ty/context.rs

@@ -1043,13 +1043,13 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
         }
 
         let interned = self.global_arenas.const_allocs.alloc(alloc);
-        if let Some(prev) = allocs.replace(interned) {
+        if let Some(prev) = allocs.replace(interned) { // insert into interner
             bug!("Tried to overwrite interned Allocation: {:#?}", prev)
         }
         interned
     }
 
-    /// Allocates a byte or string literal for `mir::interpret`
+    /// Allocates a byte or string literal for `mir::interpret`, read-only
     pub fn allocate_bytes(self, bytes: &[u8]) -> interpret::AllocId {
         // create an allocation that just contains these bytes
         let alloc = interpret::Allocation::from_byte_aligned_bytes(bytes);

src/librustc/ty/structural_impls.rs

@@ -498,7 +498,7 @@ impl<'a, 'tcx, O: Lift<'tcx>> Lift<'tcx> for interpret::EvalErrorKind<'a, O> {
             InvalidMemoryAccess => InvalidMemoryAccess,
             InvalidFunctionPointer => InvalidFunctionPointer,
             InvalidBool => InvalidBool,
-            InvalidDiscriminant => InvalidDiscriminant,
+            InvalidDiscriminant(val) => InvalidDiscriminant(val),
             PointerOutOfBounds {
                 ptr,
                 access,
@@ -1139,7 +1139,7 @@ impl<'tcx> TypeFoldable<'tcx> for ConstValue<'tcx> {
         match *self {
             ConstValue::Scalar(v) => ConstValue::Scalar(v),
             ConstValue::ScalarPair(a, b) => ConstValue::ScalarPair(a, b),
-            ConstValue::ByRef(alloc, offset) => ConstValue::ByRef(alloc, offset),
+            ConstValue::ByRef(id, alloc, offset) => ConstValue::ByRef(id, alloc, offset),
             ConstValue::Unevaluated(def_id, substs) => {
                 ConstValue::Unevaluated(def_id, substs.fold_with(folder))
             }
@@ -1150,7 +1150,7 @@ impl<'tcx> TypeFoldable<'tcx> for ConstValue<'tcx> {
         match *self {
             ConstValue::Scalar(_) |
             ConstValue::ScalarPair(_, _) |
-            ConstValue::ByRef(_, _) => false,
+            ConstValue::ByRef(_, _, _) => false,
             ConstValue::Unevaluated(_, substs) => substs.visit_with(visitor),
         }
     }

src/librustc_codegen_llvm/mir/constant.rs

@@ -57,7 +57,7 @@ pub fn scalar_to_llvm(
             let base_addr = match alloc_type {
                 Some(AllocType::Memory(alloc)) => {
                     let init = const_alloc_to_llvm(cx, alloc);
-                    if alloc.runtime_mutability == Mutability::Mutable {
+                    if alloc.mutability == Mutability::Mutable {
                         consts::addr_of_mut(cx, init, alloc.align, None)
                     } else {
                         consts::addr_of(cx, init, alloc.align, None)
@@ -134,7 +134,7 @@ pub fn codegen_static_initializer(
     let static_ = cx.tcx.const_eval(param_env.and(cid))?;
 
     let alloc = match static_.val {
-        ConstValue::ByRef(alloc, n) if n.bytes() == 0 => alloc,
+        ConstValue::ByRef(_, alloc, n) if n.bytes() == 0 => alloc,
         _ => bug!("static const eval returned {:#?}", static_),
     };
     Ok((const_alloc_to_llvm(cx, alloc), alloc))

src/librustc_codegen_llvm/mir/operand.rs

@@ -126,7 +126,7 @@ impl OperandRef<'ll, 'tcx> {
                 };
                 OperandValue::Pair(a_llval, b_llval)
             },
-            ConstValue::ByRef(alloc, offset) => {
+            ConstValue::ByRef(_, alloc, offset) => {
                 return Ok(PlaceRef::from_const_alloc(bx, layout, alloc, offset).load(bx));
             },
         };

src/librustc_codegen_llvm/mir/place.rs

@@ -458,7 +458,7 @@ impl FunctionCx<'a, 'll, 'tcx> {
                 let layout = cx.layout_of(self.monomorphize(&ty));
                 match bx.tcx().const_eval(param_env.and(cid)) {
                     Ok(val) => match val.val {
-                        mir::interpret::ConstValue::ByRef(alloc, offset) => {
+                        mir::interpret::ConstValue::ByRef(_, alloc, offset) => {
                             PlaceRef::from_const_alloc(bx, layout, alloc, offset)
                         }
                         _ => bug!("promoteds should have an allocation: {:?}", val),

src/librustc_lint/builtin.rs

@@ -1612,21 +1612,15 @@ fn validate_const<'a, 'tcx>(
     gid: ::rustc::mir::interpret::GlobalId<'tcx>,
     what: &str,
 ) {
-    let mut ecx = ::rustc_mir::interpret::mk_eval_cx(tcx, gid.instance, param_env).unwrap();
+    let ecx = ::rustc_mir::interpret::mk_eval_cx(tcx, gid.instance, param_env).unwrap();
     let result = (|| {
-        use rustc_target::abi::LayoutOf;
-        use rustc_mir::interpret::OpTy;
-
-        let op = ecx.const_value_to_op(constant.val)?;
-        let layout = ecx.layout_of(constant.ty)?;
-        let place = ecx.allocate_op(OpTy { op, layout })?.into();
-
-        let mut todo = vec![(place, Vec::new())];
+        let op = ecx.const_to_op(constant)?;
+        let mut todo = vec![(op, Vec::new())];
         let mut seen = FxHashSet();
-        seen.insert(place);
-        while let Some((place, mut path)) = todo.pop() {
-            ecx.validate_mplace(
-                place,
+        seen.insert(op);
+        while let Some((op, mut path)) = todo.pop() {
+            ecx.validate_operand(
+                op,
                 &mut path,
                 &mut seen,
                 &mut todo,