Auto merge of #130051 - cjgillot:clone-mir, r=<try>

Post-mono MIR opts

Cargo.lock

@@ -3463,6 +3463,7 @@ dependencies = [
  "rustc_macros",
  "rustc_metadata",
  "rustc_middle",
+ "rustc_mir_transform",
  "rustc_monomorphize",
  "rustc_query_system",
  "rustc_serialize",

compiler/rustc_codegen_ssa/Cargo.toml

@@ -27,6 +27,7 @@ rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
 rustc_middle = { path = "../rustc_middle" }
+rustc_mir_transform = { path = "../rustc_mir_transform" }
 rustc_monomorphize = { path = "../rustc_monomorphize" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_serialize = { path = "../rustc_serialize" }

compiler/rustc_codegen_ssa/src/mir/analyze.rs

@@ -14,16 +14,15 @@ use super::FunctionCx;
 use crate::traits::*;
 
 pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
-    fx: &FunctionCx<'a, 'tcx, Bx>,
+    fx: &FunctionCx<'a, '_, 'tcx, Bx>,
 ) -> BitSet<mir::Local> {
-    let mir = fx.mir;
+    let mir = &fx.mir;
     let dominators = mir.basic_blocks.dominators();
     let locals = mir
         .local_decls
         .iter()
         .map(|decl| {
-            let ty = fx.monomorphize(decl.ty);
-            let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span);
+            let layout = fx.cx.spanned_layout_of(decl.ty, decl.source_info.span);
             if layout.is_zst() {
                 LocalKind::ZST
             } else if fx.cx.is_backend_immediate(layout) || fx.cx.is_backend_scalar_pair(layout) {
@@ -70,7 +69,7 @@ enum LocalKind {
 }
 
 struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
-    fx: &'mir FunctionCx<'a, 'tcx, Bx>,
+    fx: &'mir FunctionCx<'a, 'mir, 'tcx, Bx>,
     dominators: &'mir Dominators<mir::BasicBlock>,
     locals: IndexVec<mir::Local, LocalKind>,
 }
@@ -110,10 +109,9 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx,
             );
             if is_consume {
                 let base_ty = place_base.ty(self.fx.mir, cx.tcx());
-                let base_ty = self.fx.monomorphize(base_ty);
 
                 // ZSTs don't require any actual memory access.
-                let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(elem)).ty;
+                let elem_ty = base_ty.projection_ty(cx.tcx(), elem).ty;
                 let span = self.fx.mir.local_decls[place_ref.local].source_info.span;
                 if cx.spanned_layout_of(elem_ty, span).is_zst() {
                     return;
@@ -237,7 +235,6 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
                 let kind = &mut self.locals[local];
                 if *kind != LocalKind::Memory {
                     let ty = self.fx.mir.local_decls[local].ty;
-                    let ty = self.fx.monomorphize(ty);
                     if self.fx.cx.type_needs_drop(ty) {
                         // Only need the place if we're actually dropping it.
                         *kind = LocalKind::Memory;

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -37,17 +37,17 @@ enum MergingSucc {
 
 /// Used by `FunctionCx::codegen_terminator` for emitting common patterns
 /// e.g., creating a basic block, calling a function, etc.
-struct TerminatorCodegenHelper<'tcx> {
+struct TerminatorCodegenHelper<'mir, 'tcx> {
     bb: mir::BasicBlock,
-    terminator: &'tcx mir::Terminator<'tcx>,
+    terminator: &'mir mir::Terminator<'tcx>,
 }
 
-impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
+impl<'a, 'tcx> TerminatorCodegenHelper<'_, 'tcx> {
     /// Returns the appropriate `Funclet` for the current funclet, if on MSVC,
     /// either already previously cached, or newly created, by `landing_pad_for`.
     fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &'b mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &'b mut FunctionCx<'a, '_, 'tcx, Bx>,
     ) -> Option<&'b Bx::Funclet> {
         let cleanup_kinds = fx.cleanup_kinds.as_ref()?;
         let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb)?;
@@ -73,7 +73,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     /// stuff in it or next to it.
     fn llbb_with_cleanup<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, '_, 'tcx, Bx>,
         target: mir::BasicBlock,
     ) -> Bx::BasicBlock {
         let (needs_landing_pad, is_cleanupret) = self.llbb_characteristics(fx, target);
@@ -97,7 +97,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 
     fn llbb_characteristics<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, '_, 'tcx, Bx>,
         target: mir::BasicBlock,
     ) -> (bool, bool) {
         if let Some(ref cleanup_kinds) = fx.cleanup_kinds {
@@ -122,7 +122,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 
     fn funclet_br<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, '_, 'tcx, Bx>,
         bx: &mut Bx,
         target: mir::BasicBlock,
         mergeable_succ: bool,
@@ -151,7 +151,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     /// return destination `destination` and the unwind action `unwind`.
     fn do_call<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, '_, 'tcx, Bx>,
         bx: &mut Bx,
         fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
         fn_ptr: Bx::Value,
@@ -270,7 +270,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     /// Generates inline assembly with optional `destination` and `unwind`.
     fn do_inlineasm<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'a, '_, 'tcx, Bx>,
         bx: &mut Bx,
         template: &[InlineAsmTemplatePiece],
         operands: &[InlineAsmOperandRef<'tcx, Bx>],
@@ -337,9 +337,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 }
 
 /// Codegen implementations for some terminator variants.
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     /// Generates code for a `Resume` terminator.
-    fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, bx: &mut Bx) {
+    fn codegen_resume_terminator(
+        &mut self,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
+        bx: &mut Bx,
+    ) {
         if let Some(funclet) = helper.funclet(self) {
             bx.cleanup_ret(funclet, None);
         } else {
@@ -356,7 +360,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_switchint_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         discr: &mir::Operand<'tcx>,
         targets: &SwitchTargets,
@@ -496,7 +500,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     #[tracing::instrument(level = "trace", skip(self, helper, bx))]
     fn codegen_drop_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         source_info: &mir::SourceInfo,
         location: mir::Place<'tcx>,
@@ -505,7 +509,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         mergeable_succ: bool,
     ) -> MergingSucc {
         let ty = location.ty(self.mir, bx.tcx()).ty;
-        let ty = self.monomorphize(ty);
         let drop_fn = Instance::resolve_drop_in_place(bx.tcx(), ty);
 
         if let ty::InstanceKind::DropGlue(_, None) = drop_fn.def {
@@ -638,7 +641,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_assert_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         terminator: &mir::Terminator<'tcx>,
         cond: &mir::Operand<'tcx>,
@@ -717,7 +720,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_terminate_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         terminator: &mir::Terminator<'tcx>,
         reason: UnwindTerminateReason,
@@ -747,7 +750,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     /// Returns `Some` if this is indeed a panic intrinsic and codegen is done.
     fn codegen_panic_intrinsic(
         &mut self,
-        helper: &TerminatorCodegenHelper<'tcx>,
+        helper: &TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         intrinsic: ty::IntrinsicDef,
         instance: Option<Instance<'tcx>>,
@@ -815,7 +818,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_call_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         terminator: &mir::Terminator<'tcx>,
         func: &mir::Operand<'tcx>,
@@ -868,10 +871,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let abi = sig.abi();
 
         let extra_args = &args[sig.inputs().skip_binder().len()..];
-        let extra_args = bx.tcx().mk_type_list_from_iter(extra_args.iter().map(|op_arg| {
-            let op_ty = op_arg.node.ty(self.mir, bx.tcx());
-            self.monomorphize(op_ty)
-        }));
+        let extra_args = bx.tcx().mk_type_list_from_iter(
+            extra_args.iter().map(|op_arg| op_arg.node.ty(self.mir, bx.tcx())),
+        );
 
         let fn_abi = match instance {
             Some(instance) => bx.fn_abi_of_instance(instance, extra_args),
@@ -1154,7 +1156,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_asm_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         terminator: &mir::Terminator<'tcx>,
         template: &[ast::InlineAsmTemplatePiece],
@@ -1196,8 +1198,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     InlineAsmOperandRef::Const { string }
                 }
                 mir::InlineAsmOperand::SymFn { ref value } => {
-                    let const_ = self.monomorphize(value.const_);
-                    if let ty::FnDef(def_id, args) = *const_.ty().kind() {
+                    if let ty::FnDef(def_id, args) = *value.const_.ty().kind() {
                         let instance = ty::Instance::resolve_for_fn_ptr(
                             bx.tcx(),
                             ty::ParamEnv::reveal_all(),
@@ -1238,7 +1239,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     }
 }
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     pub fn codegen_block(&mut self, mut bb: mir::BasicBlock) {
         let llbb = match self.try_llbb(bb) {
             Some(llbb) => llbb,
@@ -1293,7 +1294,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         &mut self,
         bx: &mut Bx,
         bb: mir::BasicBlock,
-        terminator: &'tcx mir::Terminator<'tcx>,
+        terminator: &mir::Terminator<'tcx>,
     ) -> MergingSucc {
         debug!("codegen_terminator: {:?}", terminator);
 

compiler/rustc_codegen_ssa/src/mir/constant.rs

@@ -9,21 +9,21 @@ use crate::errors;
 use crate::mir::operand::OperandRef;
 use crate::traits::*;
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     pub fn eval_mir_constant_to_operand(
         &self,
         bx: &mut Bx,
         constant: &mir::ConstOperand<'tcx>,
     ) -> OperandRef<'tcx, Bx::Value> {
         let val = self.eval_mir_constant(constant);
-        let ty = self.monomorphize(constant.ty());
-        OperandRef::from_const(bx, val, ty)
+        OperandRef::from_const(bx, val, constant.ty())
     }
 
     pub fn eval_mir_constant(&self, constant: &mir::ConstOperand<'tcx>) -> mir::ConstValue<'tcx> {
         // `MirUsedCollector` visited all required_consts before codegen began, so if we got here
         // there can be no more constants that fail to evaluate.
-        self.monomorphize(constant.const_)
+        constant
+            .const_
             .eval(self.cx.tcx(), ty::ParamEnv::reveal_all(), constant.span)
             .expect("erroneous constant missed by mono item collection")
     }
@@ -37,7 +37,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         &self,
         constant: &mir::ConstOperand<'tcx>,
     ) -> Result<Result<ty::ValTree<'tcx>, Ty<'tcx>>, ErrorHandled> {
-        let uv = match self.monomorphize(constant.const_) {
+        let uv = match constant.const_ {
             mir::Const::Unevaluated(uv, _) => uv.shrink(),
             mir::Const::Ty(_, c) => match c.kind() {
                 // A constant that came from a const generic but was then used as an argument to old-style
@@ -55,7 +55,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             // const generic, and get rid of this entire function.
             other => span_bug!(constant.span, "{other:#?}"),
         };
-        let uv = self.monomorphize(uv);
         self.cx.tcx().const_eval_resolve_for_typeck(ty::ParamEnv::reveal_all(), uv, constant.span)
     }
 
@@ -65,7 +64,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         bx: &Bx,
         constant: &mir::ConstOperand<'tcx>,
     ) -> (Bx::Value, Ty<'tcx>) {
-        let ty = self.monomorphize(constant.ty());
+        let ty = constant.ty();
         let ty_is_simd = ty.is_simd();
         // FIXME: ideally we'd assert that this is a SIMD type, but simd_shuffle
         // in its current form relies on a regular array being passed as an

compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs

@@ -4,14 +4,10 @@ use rustc_middle::mir::SourceScope;
 use super::FunctionCx;
 use crate::traits::*;
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     pub fn codegen_coverage(&self, bx: &mut Bx, kind: &CoverageKind, scope: SourceScope) {
         // Determine the instance that coverage data was originally generated for.
-        let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) {
-            self.monomorphize(inlined)
-        } else {
-            self.instance
-        };
+        let instance = scope.inlined_instance(&self.mir.source_scopes).unwrap_or(self.instance);
 
         // Handle the coverage info in a backend-specific way.
         bx.add_coverage(instance, kind);

compiler/rustc_codegen_ssa/src/mir/debuginfo.rs

@@ -201,7 +201,7 @@ fn calculate_debuginfo_offset<
     DebugInfoOffset { direct_offset, indirect_offsets, result: place }
 }
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) {
         bx.set_span(source_info.span);
         if let Some(dbg_loc) = self.dbg_loc(source_info) {
@@ -279,9 +279,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             // FIXME(eddyb) is this `+ 1` needed at all?
                             let kind = VariableKind::ArgumentVariable(arg_index + 1);
 
-                            let arg_ty = self.monomorphize(decl.ty);
-
-                            self.cx.create_dbg_var(name, arg_ty, dbg_scope, kind, span)
+                            self.cx.create_dbg_var(name, decl.ty, dbg_scope, kind, span)
                         },
                     )
                 } else {
@@ -457,13 +455,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             };
 
             let var_ty = if let Some(ref fragment) = var.composite {
-                self.monomorphize(fragment.ty)
+                fragment.ty
             } else {
                 match var.value {
                     mir::VarDebugInfoContents::Place(place) => {
                         self.monomorphized_place_ty(place.as_ref())
                     }
-                    mir::VarDebugInfoContents::Const(c) => self.monomorphize(c.ty()),
+                    mir::VarDebugInfoContents::Const(c) => c.ty(),
                 }
             };
 

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -49,7 +49,7 @@ fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     bx.memset(dst, val, size, align, flags);
 }
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> {
     /// In the `Err` case, returns the instance that should be called instead.
     pub fn codegen_intrinsic_call(
         bx: &mut Bx,