Auto merge of #69749 - davidtwco:issue-46477-polymorphization, r=eddyb

Polymorphization

This PR implements an analysis to detect when functions could remain polymorphic during code generation.

Fixes #46477

r? @eddyb
cc @rust-lang/wg-mir-opt @nikomatsakis @pnkfelix

src/librustc_codegen_llvm/callee.rs

@@ -13,7 +13,7 @@ use log::debug;
 use rustc_codegen_ssa::traits::*;
 
 use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt};
-use rustc_middle::ty::{Instance, TypeFoldable};
+use rustc_middle::ty::{self, Instance, TypeFoldable};
 
 /// Codegens a reference to a fn/method item, monomorphizing and
 /// inlining as it goes.
@@ -29,14 +29,18 @@ pub fn get_fn(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value
 
     assert!(!instance.substs.needs_infer());
     assert!(!instance.substs.has_escaping_bound_vars());
-    assert!(!instance.substs.has_param_types_or_consts());
 
     if let Some(&llfn) = cx.instances.borrow().get(&instance) {
         return llfn;
     }
 
     let sym = tcx.symbol_name(instance).name;
-    debug!("get_fn({:?}: {:?}) => {}", instance, instance.monomorphic_ty(cx.tcx()), sym);
+    debug!(
+        "get_fn({:?}: {:?}) => {}",
+        instance,
+        instance.ty(cx.tcx(), ty::ParamEnv::reveal_all()),
+        sym
+    );
 
     let fn_abi = FnAbi::of_instance(cx, instance, &[]);
 

src/librustc_codegen_llvm/consts.rs

@@ -203,7 +203,7 @@ impl CodegenCx<'ll, 'tcx> {
             def_id
         );
 
-        let ty = instance.monomorphic_ty(self.tcx);
+        let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
         let sym = self.tcx.symbol_name(instance).name;
 
         debug!("get_static: sym={} instance={:?}", sym, instance);
@@ -361,7 +361,7 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> {
             };
 
             let instance = Instance::mono(self.tcx, def_id);
-            let ty = instance.monomorphic_ty(self.tcx);
+            let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
             let llty = self.layout_of(ty).llvm_type(self);
             let g = if val_llty == llty {
                 g

src/librustc_codegen_llvm/debuginfo/metadata.rs

@@ -700,6 +700,8 @@ pub fn type_metadata(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>, usage_site_span: Sp
             prepare_tuple_metadata(cx, t, &tys, unique_type_id, usage_site_span, NO_SCOPE_METADATA)
                 .finalize(cx)
         }
+        // Type parameters from polymorphized functions.
+        ty::Param(_) => MetadataCreationResult::new(param_type_metadata(cx, t), false),
         _ => bug!("debuginfo: unexpected type in type_metadata: {:?}", t),
     };
 
@@ -955,6 +957,20 @@ fn pointer_type_metadata(
     }
 }
 
+fn param_type_metadata(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType {
+    debug!("param_type_metadata: {:?}", t);
+    let name = format!("{:?}", t);
+    return unsafe {
+        llvm::LLVMRustDIBuilderCreateBasicType(
+            DIB(cx),
+            name.as_ptr().cast(),
+            name.len(),
+            Size::ZERO.bits(),
+            DW_ATE_unsigned,
+        )
+    };
+}
+
 pub fn compile_unit_metadata(
     tcx: TyCtxt<'_>,
     codegen_unit_name: &str,
@@ -2465,7 +2481,7 @@ pub fn create_global_var_metadata(cx: &CodegenCx<'ll, '_>, def_id: DefId, global
     };
 
     let is_local_to_unit = is_node_local_to_unit(cx, def_id);
-    let variable_type = Instance::mono(cx.tcx, def_id).monomorphic_ty(cx.tcx);
+    let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all());
     let type_metadata = type_metadata(cx, variable_type, span);
     let var_name = tcx.item_name(def_id).as_str();
     let linkage_name = mangled_name_of_instance(cx, Instance::mono(tcx, def_id)).name;

src/librustc_codegen_llvm/debuginfo/mod.rs

@@ -26,7 +26,7 @@ use rustc_index::vec::IndexVec;
 use rustc_middle::mir;
 use rustc_middle::ty::layout::HasTyCtxt;
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
-use rustc_middle::ty::{self, Instance, ParamEnv, Ty};
+use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeFoldable};
 use rustc_session::config::{self, DebugInfo};
 use rustc_span::symbol::Symbol;
 use rustc_span::{self, BytePos, Span};
@@ -470,7 +470,9 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                     match impl_self_ty.kind {
                         ty::Adt(def, ..) if !def.is_box() => {
                             // Again, only create type information if full debuginfo is enabled
-                            if cx.sess().opts.debuginfo == DebugInfo::Full {
+                            if cx.sess().opts.debuginfo == DebugInfo::Full
+                                && !impl_self_ty.needs_subst()
+                            {
                                 Some(type_metadata(cx, impl_self_ty, rustc_span::DUMMY_SP))
                             } else {
                                 Some(namespace::item_namespace(cx, def.did))

src/librustc_codegen_llvm/intrinsic.rs

@@ -160,7 +160,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         caller_instance: ty::Instance<'tcx>,
     ) {
         let tcx = self.tcx;
-        let callee_ty = instance.monomorphic_ty(tcx);
+        let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all());
 
         let (def_id, substs) = match callee_ty.kind {
             ty::FnDef(def_id, substs) => (def_id, substs),

src/librustc_codegen_llvm/mono_item.rs

@@ -10,7 +10,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 pub use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::mir::mono::{Linkage, Visibility};
 use rustc_middle::ty::layout::FnAbiExt;
-use rustc_middle::ty::{Instance, TypeFoldable};
+use rustc_middle::ty::{self, Instance, TypeFoldable};
 use rustc_target::abi::LayoutOf;
 
 impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
@@ -22,7 +22,7 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         symbol_name: &str,
     ) {
         let instance = Instance::mono(self.tcx, def_id);
-        let ty = instance.monomorphic_ty(self.tcx);
+        let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
         let llty = self.layout_of(ty).llvm_type(self);
 
         let g = self.define_global(symbol_name, llty).unwrap_or_else(|| {
@@ -47,7 +47,7 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         visibility: Visibility,
         symbol_name: &str,
     ) {
-        assert!(!instance.substs.needs_infer() && !instance.substs.has_param_types_or_consts());
+        assert!(!instance.substs.needs_infer());
 
         let fn_abi = FnAbi::of_instance(self, instance, &[]);
         let lldecl = self.declare_fn(symbol_name, &fn_abi);

src/librustc_codegen_ssa/debuginfo/type_names.rs

@@ -205,14 +205,17 @@ pub fn push_debuginfo_type_name<'tcx>(
                 tcx.def_key(def_id).disambiguated_data.disambiguator
             ));
         }
+        // Type parameters from polymorphized functions.
+        ty::Param(_) => {
+            output.push_str(&format!("{:?}", t));
+        }
         ty::Error(_)
         | ty::Infer(_)
         | ty::Placeholder(..)
         | ty::Projection(..)
         | ty::Bound(..)
         | ty::Opaque(..)
-        | ty::GeneratorWitness(..)
-        | ty::Param(_) => {
+        | ty::GeneratorWitness(..) => {
             bug!(
                 "debuginfo: Trying to create type name for \
                   unexpected type: {:?}",

src/librustc_codegen_ssa/meth.rs

@@ -94,7 +94,8 @@ pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
                     def_id,
                     substs,
                 )
-                .unwrap(),
+                .unwrap()
+                .polymorphize(cx.tcx()),
             )
         })
     });

src/librustc_codegen_ssa/mir/analyze.rs

@@ -124,8 +124,7 @@ impl<Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, Bx> {
                 let base_ty = self.fx.monomorphize(&base_ty);
 
                 // ZSTs don't require any actual memory access.
-                let elem_ty = base_ty.projection_ty(cx.tcx(), elem).ty;
-                let elem_ty = self.fx.monomorphize(&elem_ty);
+                let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(&elem)).ty;
                 let span = self.fx.mir.local_decls[place_ref.local].source_info.span;
                 if cx.spanned_layout_of(elem_ty, span).is_zst() {
                     return;

src/librustc_codegen_ssa/mir/block.rs

@@ -543,7 +543,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 Some(
                     ty::Instance::resolve(bx.tcx(), ty::ParamEnv::reveal_all(), def_id, substs)
                         .unwrap()
-                        .unwrap(),
+                        .unwrap()
+                        .polymorphize(bx.tcx()),
                 ),
                 None,
             ),

src/librustc_codegen_ssa/mir/rvalue.rs

@@ -190,17 +190,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 if bx.cx().tcx().has_attr(def_id, sym::rustc_args_required_const) {
                                     bug!("reifying a fn ptr that requires const arguments");
                                 }
-                                OperandValue::Immediate(
-                                    bx.get_fn_addr(
-                                        ty::Instance::resolve_for_fn_ptr(
-                                            bx.tcx(),
-                                            ty::ParamEnv::reveal_all(),
-                                            def_id,
-                                            substs,
-                                        )
-                                        .unwrap(),
-                                    ),
+                                let instance = ty::Instance::resolve_for_fn_ptr(
+                                    bx.tcx(),
+                                    ty::ParamEnv::reveal_all(),
+                                    def_id,
+                                    substs,
                                 )
+                                .unwrap()
+                                .polymorphize(bx.cx().tcx());
+                                OperandValue::Immediate(bx.get_fn_addr(instance))
                             }
                             _ => bug!("{} cannot be reified to a fn ptr", operand.layout.ty),
                         }
@@ -213,7 +211,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                     def_id,
                                     substs,
                                     ty::ClosureKind::FnOnce,
-                                );
+                                )
+                                .polymorphize(bx.cx().tcx());
                                 OperandValue::Immediate(bx.cx().get_fn_addr(instance))
                             }
                             _ => bug!("{} cannot be cast to a fn ptr", operand.layout.ty),

src/librustc_data_structures/stable_hasher.rs

@@ -469,6 +469,15 @@ impl<R: vec::Idx, C: vec::Idx, CTX> HashStable<CTX> for bit_set::BitMatrix<R, C>
     }
 }
 
+impl<T, CTX> HashStable<CTX> for bit_set::FiniteBitSet<T>
+where
+    T: HashStable<CTX> + bit_set::FiniteBitSetTy,
+{
+    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
+        self.0.hash_stable(hcx, hasher);
+    }
+}
+
 impl_stable_hash_via_hash!(::std::path::Path);
 impl_stable_hash_via_hash!(::std::path::PathBuf);
 

src/librustc_feature/builtin_attrs.rs

@@ -568,6 +568,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     ),
     rustc_attr!(TEST, rustc_synthetic, AssumedUsed, template!(Word)),
     rustc_attr!(TEST, rustc_symbol_name, AssumedUsed, template!(Word)),
+    rustc_attr!(TEST, rustc_polymorphize_error, AssumedUsed, template!(Word)),
     rustc_attr!(TEST, rustc_def_path, AssumedUsed, template!(Word)),
     rustc_attr!(TEST, rustc_mir, AssumedUsed, template!(List: "arg1, arg2, ...")),
     rustc_attr!(TEST, rustc_dump_program_clauses, AssumedUsed, template!(Word)),

src/librustc_index/bit_set.rs

@@ -4,6 +4,7 @@ use std::fmt;
 use std::iter;
 use std::marker::PhantomData;
 use std::mem;
+use std::ops::{BitAnd, BitAndAssign, BitOrAssign, Not, Range, Shl};
 use std::slice;
 
 #[cfg(test)]
@@ -1001,3 +1002,137 @@ fn word_index_and_mask<T: Idx>(elem: T) -> (usize, Word) {
     let mask = 1 << (elem % WORD_BITS);
     (word_index, mask)
 }
+
+/// Integral type used to represent the bit set.
+pub trait FiniteBitSetTy:
+    BitAnd<Output = Self>
+    + BitAndAssign
+    + BitOrAssign
+    + Clone
+    + Copy
+    + Shl
+    + Not<Output = Self>
+    + PartialEq
+    + Sized
+{
+    /// Size of the domain representable by this type, e.g. 64 for `u64`.
+    const DOMAIN_SIZE: u32;
+
+    /// Value which represents the `FiniteBitSet` having every bit set.
+    const FILLED: Self;
+    /// Value which represents the `FiniteBitSet` having no bits set.
+    const EMPTY: Self;
+
+    /// Value for one as the integral type.
+    const ONE: Self;
+    /// Value for zero as the integral type.
+    const ZERO: Self;
+
+    /// Perform a checked left shift on the integral type.
+    fn checked_shl(self, rhs: u32) -> Option<Self>;
+    /// Perform a checked right shift on the integral type.
+    fn checked_shr(self, rhs: u32) -> Option<Self>;
+}
+
+impl FiniteBitSetTy for u64 {
+    const DOMAIN_SIZE: u32 = 64;
+
+    const FILLED: Self = Self::MAX;
+    const EMPTY: Self = Self::MIN;
+
+    const ONE: Self = 1u64;
+    const ZERO: Self = 0u64;
+
+    fn checked_shl(self, rhs: u32) -> Option<Self> {
+        self.checked_shl(rhs)
+    }
+
+    fn checked_shr(self, rhs: u32) -> Option<Self> {
+        self.checked_shr(rhs)
+    }
+}
+
+impl std::fmt::Debug for FiniteBitSet<u64> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:064b}", self.0)
+    }
+}
+
+impl FiniteBitSetTy for u128 {
+    const DOMAIN_SIZE: u32 = 128;
+
+    const FILLED: Self = Self::MAX;
+    const EMPTY: Self = Self::MIN;
+
+    const ONE: Self = 1u128;
+    const ZERO: Self = 0u128;
+
+    fn checked_shl(self, rhs: u32) -> Option<Self> {
+        self.checked_shl(rhs)
+    }
+
+    fn checked_shr(self, rhs: u32) -> Option<Self> {
+        self.checked_shr(rhs)
+    }
+}
+
+impl std::fmt::Debug for FiniteBitSet<u128> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:0128b}", self.0)
+    }
+}
+
+/// A fixed-sized bitset type represented by an integer type. Indices outwith than the range
+/// representable by `T` are considered set.
+#[derive(Copy, Clone, Eq, PartialEq, RustcDecodable, RustcEncodable)]
+pub struct FiniteBitSet<T: FiniteBitSetTy>(pub T);
+
+impl<T: FiniteBitSetTy> FiniteBitSet<T> {
+    /// Creates a new, empty bitset.
+    pub fn new_empty() -> Self {
+        Self(T::EMPTY)
+    }
+
+    /// Sets the `index`th bit.
+    pub fn set(&mut self, index: u32) {
+        self.0 |= T::ONE.checked_shl(index).unwrap_or(T::ZERO);
+    }
+
+    /// Unsets the `index`th bit.
+    pub fn clear(&mut self, index: u32) {
+        self.0 &= !T::ONE.checked_shl(index).unwrap_or(T::ZERO);
+    }
+
+    /// Sets the `i`th to `j`th bits.
+    pub fn set_range(&mut self, range: Range<u32>) {
+        let bits = T::FILLED
+            .checked_shl(range.end - range.start)
+            .unwrap_or(T::ZERO)
+            .not()
+            .checked_shl(range.start)
+            .unwrap_or(T::ZERO);
+        self.0 |= bits;
+    }
+
+    /// Is the set empty?
+    pub fn is_empty(&self) -> bool {
+        self.0 == T::EMPTY
+    }
+
+    /// Returns the domain size of the bitset.
+    pub fn within_domain(&self, index: u32) -> bool {
+        index < T::DOMAIN_SIZE
+    }
+
+    /// Returns if the `index`th bit is set.
+    pub fn contains(&self, index: u32) -> Option<bool> {
+        self.within_domain(index)
+            .then(|| ((self.0.checked_shr(index).unwrap_or(T::ONE)) & T::ONE) == T::ONE)
+    }
+}
+
+impl<T: FiniteBitSetTy> Default for FiniteBitSet<T> {
+    fn default() -> Self {
+        Self::new_empty()
+    }
+}