diff --git a/Cargo.lock b/Cargo.lock index 3ad21b0abf920..85c364d44b5d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4099,6 +4099,7 @@ dependencies = [ name = "rustc_mir_dataflow" version = "0.0.0" dependencies = [ + "itertools", "polonius-engine", "regex", "rustc_ast", diff --git a/compiler/rustc_mir_dataflow/Cargo.toml b/compiler/rustc_mir_dataflow/Cargo.toml index 7199db677c464..78b1fc3b32faf 100644 --- a/compiler/rustc_mir_dataflow/Cargo.toml +++ b/compiler/rustc_mir_dataflow/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] # tidy-alphabetical-start +itertools = "0.12" polonius-engine = "0.13.0" regex = "1" rustc_ast = { path = "../rustc_ast" } diff --git a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs index 3461808453854..3680b8637d61e 100644 --- a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs +++ b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs @@ -4,11 +4,11 @@ use rustc_hir::lang_items::LangItem; use rustc_index::Idx; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; -use rustc_middle::span_bug; use rustc_middle::traits::Reveal; use rustc_middle::ty::util::IntTypeExt; -use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt}; -use rustc_span::source_map::Spanned; +use rustc_middle::ty::{self, GenericArg, GenericArgsRef, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug, traits}; +use rustc_span::source_map::{dummy_spanned, Spanned}; use rustc_span::DUMMY_SP; use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT}; use tracing::{debug, instrument}; @@ -112,6 +112,9 @@ pub trait DropElaborator<'a, 'tcx>: fmt::Debug { fn body(&self) -> &'a Body<'tcx>; fn tcx(&self) -> TyCtxt<'tcx>; fn param_env(&self) -> ty::ParamEnv<'tcx>; + fn allow_async_drops(&self) -> bool; + + fn terminator_loc(&self, bb: BasicBlock) -> Location; // Drop logic @@ -167,6 +170,7 @@ where path: D::Path, succ: BasicBlock, unwind: Unwind, + dropline: Option, } /// "Elaborates" a drop of `place`/`path` and patches `bb`'s terminator to execute it. @@ -185,11 +189,12 @@ pub fn elaborate_drop<'b, 'tcx, D>( succ: BasicBlock, unwind: Unwind, bb: BasicBlock, + dropline: Option, ) where D: DropElaborator<'b, 'tcx>, 'tcx: 'b, { - DropCtxt { elaborator, source_info, place, path, succ, unwind }.elaborate_drop(bb) + DropCtxt { elaborator, source_info, place, path, succ, unwind, dropline }.elaborate_drop(bb) } impl<'l, 'b, 'tcx, D> DropCtxt<'l, 'b, 'tcx, D> @@ -206,6 +211,207 @@ where self.elaborator.tcx() } + // Generates three blocks: + // * #1:pin_obj_bb: call Pin::new_unchecked(&mut obj) + // * #2:call_drop_bb: fut = call obj.() OR call async_drop_in_place(obj) + // * #3:drop_term_bb: drop (obj, fut, ...) + // We keep async drop unexpanded to poll-loop here, to expand it later, at StateTransform - + // into states expand. + // call_destructor_only - to call only AsyncDrop::drop, not full async_drop_in_place glue + fn build_async_drop( + &mut self, + place: Place<'tcx>, + drop_ty: Ty<'tcx>, + bb: Option, + succ: BasicBlock, + unwind: Unwind, + dropline: Option, + call_destructor_only: bool, + ) -> BasicBlock { + let tcx = self.tcx(); + let span = self.source_info.span; + + let pin_obj_bb = bb.unwrap_or_else(|| { + self.elaborator.patch().new_block(BasicBlockData { + statements: vec![], + terminator: Some(Terminator { + // Temporary terminator, will be replaced by patch + source_info: self.source_info, + kind: TerminatorKind::Return, + }), + is_cleanup: false, + }) + }); + + let (fut_ty, drop_fn_def_id, trait_args) = + if call_destructor_only { + // Resolving obj.() + let trait_ref = ty::TraitRef::new( + tcx, + tcx.require_lang_item(LangItem::AsyncDrop, Some(span)), + [drop_ty], + ); + let (drop_trait, trait_args) = + match tcx.codegen_select_candidate((ty::ParamEnv::reveal_all(), trait_ref)) { + Ok(traits::ImplSource::UserDefined( + traits::ImplSourceUserDefinedData { impl_def_id, args, .. }, + )) => (*impl_def_id, *args), + impl_source => { + span_bug!(span, "invalid `AsyncDrop` impl_source: {:?}", impl_source); + } + }; + let drop_fn_def_id = tcx.associated_item_def_ids(drop_trait)[0]; + let drop_fn = Ty::new_fn_def(tcx, drop_fn_def_id, trait_args); + let sig = drop_fn.fn_sig(tcx); + let sig = tcx.instantiate_bound_regions_with_erased(sig); + (sig.output(), drop_fn_def_id, trait_args) + } else { + // Resolving async_drop_in_place function for drop_ty + let drop_fn_def_id = tcx.require_lang_item(LangItem::AsyncDropInPlace, Some(span)); + let trait_args = tcx.mk_args(&[drop_ty.into()]); + let sig = tcx.fn_sig(drop_fn_def_id).instantiate(tcx, trait_args); + let sig = tcx.instantiate_bound_regions_with_erased(sig); + (sig.output(), drop_fn_def_id, trait_args) + }; + + let fut = Place::from(self.new_temp(fut_ty)); + + // #1:pin_obj_bb >>> obj_ref = &mut obj + let obj_ref_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, drop_ty); + let obj_ref_place = Place::from(self.new_temp(obj_ref_ty)); + + let term_loc = self.elaborator.terminator_loc(pin_obj_bb); + self.elaborator.patch().add_assign( + term_loc, + obj_ref_place, + Rvalue::Ref( + tcx.lifetimes.re_erased, + BorrowKind::Mut { kind: MutBorrowKind::Default }, + place, + ), + ); + + // pin_obj_place preparation + let pin_obj_new_unchecked_fn = Ty::new_fn_def( + tcx, + tcx.require_lang_item(LangItem::PinNewUnchecked, Some(span)), + [GenericArg::from(obj_ref_ty)], + ); + let pin_obj_ty = pin_obj_new_unchecked_fn.fn_sig(tcx).output().no_bound_vars().unwrap(); + let pin_obj_place = Place::from(self.new_temp(pin_obj_ty)); + let pin_obj_new_unchecked_fn = Operand::Constant(Box::new(ConstOperand { + span, + user_ty: None, + const_: Const::zero_sized(pin_obj_new_unchecked_fn), + })); + + // #3:drop_term_bb + let drop_term_bb = self.new_block( + unwind, + TerminatorKind::Drop { + place: place, + target: succ, + unwind: unwind.into_action(), + replace: false, + drop: dropline, + async_fut: Some(fut.local), + }, + ); + + // #2:call_drop_bb + let mut call_statements = Vec::new(); + let drop_arg = if call_destructor_only { + pin_obj_place + } else { + let ty::Adt(adt_def, adt_args) = pin_obj_ty.kind() else { + bug!(); + }; + let obj_ptr_ty = Ty::new_mut_ptr(tcx, drop_ty); + let obj_ptr_place = Place::from(self.new_temp(obj_ptr_ty)); + let unwrap_ty = adt_def.non_enum_variant().fields[FieldIdx::ZERO].ty(tcx, adt_args); + let addr = Rvalue::RawPtr( + Mutability::Mut, + pin_obj_place.project_deeper( + &[ProjectionElem::Field(FieldIdx::ZERO, unwrap_ty), ProjectionElem::Deref], + tcx, + ), + ); + call_statements.push(self.assign(obj_ptr_place, addr)); + obj_ptr_place + }; + call_statements.push(Statement { + source_info: self.source_info, + kind: StatementKind::StorageLive(fut.local), + }); + + let call_drop_bb = self.new_block_with_statements( + unwind, + call_statements, + TerminatorKind::Call { + func: Operand::function_handle(tcx, drop_fn_def_id, trait_args, span), + args: [Spanned { node: Operand::Move(drop_arg), span: DUMMY_SP }].into(), + destination: fut, + target: Some(drop_term_bb), + unwind: unwind.into_action(), + call_source: CallSource::Misc, + fn_span: self.source_info.span, + }, + ); + + // StorageDead(fut) in self.succ block (at the begin) + self.elaborator.patch().add_statement( + Location { block: self.succ, statement_index: 0 }, + StatementKind::StorageDead(fut.local), + ); + + // #1:pin_obj_bb >>> call Pin::new_unchecked(&mut obj) + self.elaborator.patch().patch_terminator( + pin_obj_bb, + TerminatorKind::Call { + func: pin_obj_new_unchecked_fn, + args: [dummy_spanned(Operand::Move(obj_ref_place))].into(), + destination: pin_obj_place, + target: Some(call_drop_bb), + unwind: unwind.into_action(), + call_source: CallSource::Misc, + fn_span: span, + }, + ); + pin_obj_bb + } + + fn build_drop(&mut self, bb: BasicBlock) { + let drop_ty = self.place_ty(self.place); + if self.tcx().features().async_drop + && self.elaborator.body().coroutine.is_some() + && self.elaborator.allow_async_drops() + && !self.elaborator.body()[bb].is_cleanup + && drop_ty.needs_async_drop(self.tcx(), self.elaborator.param_env()) + { + self.build_async_drop( + self.place, + drop_ty, + Some(bb), + self.succ, + self.unwind, + self.dropline, + false, + ); + } else { + self.elaborator.patch().patch_terminator( + bb, + TerminatorKind::Drop { + place: self.place, + target: self.succ, + unwind: self.unwind.into_action(), + replace: false, + drop: None, + async_fut: None, + }, + ); + } + } + /// This elaborates a single drop instruction, located at `bb`, and /// patches over it. /// @@ -233,17 +439,7 @@ where .patch_terminator(bb, TerminatorKind::Goto { target: self.succ }); } DropStyle::Static => { - self.elaborator.patch().patch_terminator( - bb, - TerminatorKind::Drop { - place: self.place, - target: self.succ, - unwind: self.unwind.into_action(), - replace: false, - drop: None, - async_fut: None, - }, - ); + self.build_drop(bb); } DropStyle::Conditional => { let drop_bb = self.complete_drop(self.succ, self.unwind); @@ -293,6 +489,7 @@ where path: Option, succ: BasicBlock, unwind: Unwind, + dropline: Option, ) -> BasicBlock { if let Some(path) = path { debug!("drop_subpath: for std field {:?}", place); @@ -304,6 +501,7 @@ where place, succ, unwind, + dropline, } .elaborated_drop_block() } else { @@ -315,6 +513,7 @@ where place, succ, unwind, + dropline, // Using `self.path` here to condition the drop on // our own drop flag. path: self.path, @@ -329,25 +528,36 @@ where /// /// `unwind_ladder` is such a list of steps in reverse order, /// which is called if the matching step of the drop glue panics. + /// + /// `dropline_ladder` is a similar list of steps in reverse order, + /// which is called if the matching step of the drop glue will contain async drop + /// (expanded later to Yield) and the containing coroutine will be dropped at this point. fn drop_halfladder( &mut self, unwind_ladder: &[Unwind], + dropline_ladder: &[Option], mut succ: BasicBlock, fields: &[(Place<'tcx>, Option)], ) -> Vec { iter::once(succ) - .chain(fields.iter().rev().zip(unwind_ladder).map(|(&(place, path), &unwind_succ)| { - succ = self.drop_subpath(place, path, succ, unwind_succ); - succ - })) + .chain(itertools::izip!(fields.iter().rev(), unwind_ladder, dropline_ladder).map( + |(&(place, path), &unwind_succ, &dropline_to)| { + succ = self.drop_subpath(place, path, succ, unwind_succ, dropline_to); + succ + }, + )) .collect() } - fn drop_ladder_bottom(&mut self) -> (BasicBlock, Unwind) { + fn drop_ladder_bottom(&mut self) -> (BasicBlock, Unwind, Option) { // Clear the "master" drop flag at the end. This is needed // because the "master" drop protects the ADT's discriminant, // which is invalidated after the ADT is dropped. - (self.drop_flag_reset_block(DropFlagMode::Shallow, self.succ, self.unwind), self.unwind) + ( + self.drop_flag_reset_block(DropFlagMode::Shallow, self.succ, self.unwind), + self.unwind, + self.dropline, + ) } /// Creates a full drop ladder, consisting of 2 connected half-drop-ladders @@ -365,6 +575,22 @@ where /// .c2: /// ELAB(drop location.2 [target=`self.unwind`]) /// + /// For possible-async drops in coroutines we also need dropline ladder + /// .d0 (mainline): + /// ELAB(drop location.0 [target=.d1, unwind=.c1, drop=.e1]) + /// .d1 (mainline): + /// ELAB(drop location.1 [target=.d2, unwind=.c2, drop=.e2]) + /// .d2 (mainline): + /// ELAB(drop location.2 [target=`self.succ`, unwind=`self.unwind`, drop=`self.drop`]) + /// .c1 (unwind): + /// ELAB(drop location.1 [target=.c2]) + /// .c2 (unwind): + /// ELAB(drop location.2 [target=`self.unwind`]) + /// .e1 (dropline): + /// ELAB(drop location.1 [target=.e2, unwind=.c2]) + /// .e2 (dropline): + /// ELAB(drop location.2 [target=`self.drop`, unwind=`self.unwind`]) + /// /// NOTE: this does not clear the master drop flag, so you need /// to point succ/unwind on a `drop_ladder_bottom`. fn drop_ladder( @@ -372,8 +598,13 @@ where fields: Vec<(Place<'tcx>, Option)>, succ: BasicBlock, unwind: Unwind, - ) -> (BasicBlock, Unwind) { + dropline: Option, + ) -> (BasicBlock, Unwind, Option) { debug!("drop_ladder({:?}, {:?})", self, fields); + assert!( + if unwind.is_cleanup() { dropline.is_none() } else { true }, + "Dropline is set for cleanup drop ladder" + ); let mut fields = fields; fields.retain(|&(place, _)| { @@ -382,17 +613,28 @@ where debug!("drop_ladder - fields needing drop: {:?}", fields); + let dropline_ladder: Vec> = vec![None; fields.len() + 1]; let unwind_ladder = vec![Unwind::InCleanup; fields.len() + 1]; - let unwind_ladder: Vec<_> = if let Unwind::To(target) = unwind { - let halfladder = self.drop_halfladder(&unwind_ladder, target, &fields); + let unwind_ladder: Vec<_> = if let Unwind::To(succ) = unwind { + let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields); halfladder.into_iter().map(Unwind::To).collect() } else { unwind_ladder }; + let dropline_ladder: Vec<_> = if let Some(succ) = dropline { + let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields); + halfladder.into_iter().map(Some).collect() + } else { + dropline_ladder + }; - let normal_ladder = self.drop_halfladder(&unwind_ladder, succ, &fields); + let normal_ladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields); - (*normal_ladder.last().unwrap(), *unwind_ladder.last().unwrap()) + ( + *normal_ladder.last().unwrap(), + *unwind_ladder.last().unwrap(), + *dropline_ladder.last().unwrap(), + ) } fn open_drop_for_tuple(&mut self, tys: &[Ty<'tcx>]) -> BasicBlock { @@ -409,8 +651,8 @@ where }) .collect(); - let (succ, unwind) = self.drop_ladder_bottom(); - self.drop_ladder(fields, succ, unwind).0 + let (succ, unwind, dropline) = self.drop_ladder_bottom(); + self.drop_ladder(fields, succ, unwind, dropline).0 } /// Drops the T contained in a `Box` if it has not been moved out of @@ -421,6 +663,7 @@ where args: GenericArgsRef<'tcx>, succ: BasicBlock, unwind: Unwind, + dropline: Option, ) -> BasicBlock { // drop glue is sent straight to codegen // box cannot be directly dereferenced @@ -436,7 +679,7 @@ where let interior_path = self.elaborator.deref_subpath(self.path); - self.drop_subpath(interior, interior_path, succ, unwind) + self.drop_subpath(interior, interior_path, succ, unwind, dropline) } #[instrument(level = "debug", ret)] @@ -458,19 +701,22 @@ where let skip_contents = adt.is_union() || adt.is_manually_drop(); let contents_drop = if skip_contents { - (self.succ, self.unwind) + (self.succ, self.unwind, self.dropline) } else { self.open_drop_for_adt_contents(adt, args) }; if adt.is_box() { // we need to drop the inside of the box before running the destructor - let succ = self.destructor_call_block(contents_drop); + let succ = self.destructor_call_block_sync((contents_drop.0, contents_drop.1)); let unwind = contents_drop .1 - .map(|unwind| self.destructor_call_block((unwind, Unwind::InCleanup))); + .map(|unwind| self.destructor_call_block_sync((unwind, Unwind::InCleanup))); + let dropline = contents_drop + .2 + .map(|dropline| self.destructor_call_block_sync((dropline, contents_drop.1))); - self.open_drop_for_box_contents(adt, args, succ, unwind) + self.open_drop_for_box_contents(adt, args, succ, unwind, dropline) } else if adt.has_dtor(self.tcx()) { self.destructor_call_block(contents_drop) } else { @@ -482,14 +728,14 @@ where &mut self, adt: ty::AdtDef<'tcx>, args: GenericArgsRef<'tcx>, - ) -> (BasicBlock, Unwind) { - let (succ, unwind) = self.drop_ladder_bottom(); + ) -> (BasicBlock, Unwind, Option) { + let (succ, unwind, dropline) = self.drop_ladder_bottom(); if !adt.is_enum() { let fields = self.move_paths_for_fields(self.place, self.path, adt.variant(FIRST_VARIANT), args); - self.drop_ladder(fields, succ, unwind) + self.drop_ladder(fields, succ, unwind, dropline) } else { - self.open_drop_for_multivariant(adt, args, succ, unwind) + self.open_drop_for_multivariant(adt, args, succ, unwind, dropline) } } @@ -499,11 +745,14 @@ where args: GenericArgsRef<'tcx>, succ: BasicBlock, unwind: Unwind, - ) -> (BasicBlock, Unwind) { + dropline: Option, + ) -> (BasicBlock, Unwind, Option) { let mut values = Vec::with_capacity(adt.variants().len()); let mut normal_blocks = Vec::with_capacity(adt.variants().len()); let mut unwind_blocks = if unwind.is_cleanup() { None } else { Some(Vec::with_capacity(adt.variants().len())) }; + let mut dropline_blocks = + if dropline.is_none() { None } else { Some(Vec::with_capacity(adt.variants().len())) }; let mut have_otherwise_with_drop_glue = false; let mut have_otherwise = false; @@ -541,11 +790,16 @@ where let unwind_blocks = unwind_blocks.as_mut().unwrap(); let unwind_ladder = vec![Unwind::InCleanup; fields.len() + 1]; - let halfladder = self.drop_halfladder(&unwind_ladder, unwind, &fields); + let dropline_ladder: Vec> = vec![None; fields.len() + 1]; + let halfladder = + self.drop_halfladder(&unwind_ladder, &dropline_ladder, unwind, &fields); unwind_blocks.push(halfladder.last().cloned().unwrap()); } - let (normal, _) = self.drop_ladder(fields, succ, unwind); + let (normal, _, drop_bb) = self.drop_ladder(fields, succ, unwind, dropline); normal_blocks.push(normal); + if dropline.is_some() { + dropline_blocks.as_mut().unwrap().push(drop_bb.unwrap()); + } } else { have_otherwise = true; @@ -585,6 +839,9 @@ where Unwind::InCleanup, ) }), + dropline.map(|dropline| { + self.adt_switch_block(adt, dropline_blocks.unwrap(), &values, dropline, unwind) + }), ) } @@ -624,8 +881,8 @@ where self.drop_flag_test_block(switch_block, succ, unwind) } - fn destructor_call_block(&mut self, (succ, unwind): (BasicBlock, Unwind)) -> BasicBlock { - debug!("destructor_call_block({:?}, {:?})", self, succ); + fn destructor_call_block_sync(&mut self, (succ, unwind): (BasicBlock, Unwind)) -> BasicBlock { + debug!("destructor_call_block_sync({:?}, {:?})", self, succ); let tcx = self.tcx(); let drop_trait = tcx.require_lang_item(LangItem::Drop, None); let drop_fn = tcx.associated_item_def_ids(drop_trait)[0]; @@ -673,6 +930,30 @@ where self.drop_flag_test_block(destructor_block, succ, unwind) } + fn destructor_call_block( + &mut self, + (succ, unwind, dropline): (BasicBlock, Unwind, Option), + ) -> BasicBlock { + debug!("destructor_call_block({:?}, {:?})", self, succ); + let ty = self.place_ty(self.place); + if self.tcx().features().async_drop + && self.elaborator.body().coroutine.is_some() + && self.elaborator.allow_async_drops() + && !unwind.is_cleanup() + && ty.is_async_drop(self.tcx(), self.elaborator.param_env()) + { + let destructor_block = + self.build_async_drop(self.place, ty, None, succ, unwind, dropline, true); + + let block_start = Location { block: destructor_block, statement_index: 0 }; + self.elaborator.clear_drop_flag(block_start, self.path, DropFlagMode::Shallow); + + self.drop_flag_test_block(destructor_block, succ, unwind) + } else { + self.destructor_call_block_sync((succ, unwind)) + } + } + /// Create a loop that drops an array: /// /// ```text @@ -691,6 +972,7 @@ where len: Local, ety: Ty<'tcx>, unwind: Unwind, + dropline: Option, ) -> BasicBlock { let copy = |place: Place<'tcx>| Operand::Copy(place); let move_ = |place: Place<'tcx>| Operand::Move(place); @@ -734,18 +1016,35 @@ where }; let loop_block = self.elaborator.patch().new_block(loop_block); - self.elaborator.patch().patch_terminator( - drop_block, - TerminatorKind::Drop { - place: tcx.mk_place_deref(ptr), - target: loop_block, - unwind: unwind.into_action(), - replace: false, - drop: None, - async_fut: None, - }, - ); - + let place = tcx.mk_place_deref(ptr); + if self.tcx().features().async_drop + && self.elaborator.body().coroutine.is_some() + && self.elaborator.allow_async_drops() + && !unwind.is_cleanup() + && ety.needs_async_drop(self.tcx(), self.elaborator.param_env()) + { + self.build_async_drop( + place, + ety, + Some(drop_block), + loop_block, + unwind, + dropline, + false, + ); + } else { + self.elaborator.patch().patch_terminator( + drop_block, + TerminatorKind::Drop { + place: place, + target: loop_block, + unwind: unwind.into_action(), + replace: false, + drop: None, + async_fut: None, + }, + ); + } loop_block } @@ -807,26 +1106,30 @@ where (tcx.mk_place_elem(self.place, project), path) }) .collect::>(); - let (succ, unwind) = self.drop_ladder_bottom(); - return self.drop_ladder(fields, succ, unwind).0; + let (succ, unwind, dropline) = self.drop_ladder_bottom(); + return self.drop_ladder(fields, succ, unwind, dropline).0; } } - self.drop_loop_pair(ety) + self.drop_loop_trio(ety) } - /// Creates a pair of drop-loops of `place`, which drops its contents, even - /// in the case of 1 panic. - fn drop_loop_pair(&mut self, ety: Ty<'tcx>) -> BasicBlock { - debug!("drop_loop_pair({:?})", ety); + /// Creates a trio of drop-loops of `place`, which drops its contents, even + /// in the case of 1 panic or in the case of coroutine drop + fn drop_loop_trio(&mut self, ety: Ty<'tcx>) -> BasicBlock { + debug!("drop_loop_trio({:?})", ety); let tcx = self.tcx(); let len = self.new_temp(tcx.types.usize); let cur = self.new_temp(tcx.types.usize); - let unwind = - self.unwind.map(|unwind| self.drop_loop(unwind, cur, len, ety, Unwind::InCleanup)); + let unwind = self + .unwind + .map(|unwind| self.drop_loop(unwind, cur, len, ety, Unwind::InCleanup, None)); - let loop_block = self.drop_loop(self.succ, cur, len, ety, unwind); + let dropline = + self.dropline.map(|dropline| self.drop_loop(dropline, cur, len, ety, unwind, None)); + + let loop_block = self.drop_loop(self.succ, cur, len, ety, unwind, dropline); let zero = self.constant_usize(0); let block = BasicBlockData { @@ -876,7 +1179,7 @@ where let size = size.try_eval_target_usize(self.tcx(), self.elaborator.param_env()); self.open_drop_for_array(*ety, size) } - ty::Slice(ety) => self.drop_loop_pair(*ety), + ty::Slice(ety) => self.drop_loop_trio(*ety), _ => span_bug!(self.source_info.span, "open drop from non-ADT `{:?}`", ty), } @@ -913,23 +1216,53 @@ where fn elaborated_drop_block(&mut self) -> BasicBlock { debug!("elaborated_drop_block({:?})", self); - let blk = self.drop_block(self.succ, self.unwind); + let blk = self.drop_block_simple(self.succ, self.unwind); self.elaborate_drop(blk); blk } - fn drop_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock { + fn drop_block_simple(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock { let block = TerminatorKind::Drop { place: self.place, target, unwind: unwind.into_action(), replace: false, - drop: None, + drop: self.dropline, async_fut: None, }; self.new_block(unwind, block) } + fn drop_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock { + let drop_ty = self.place_ty(self.place); + if self.tcx().features().async_drop + && self.elaborator.body().coroutine.is_some() + && self.elaborator.allow_async_drops() + && !unwind.is_cleanup() + && drop_ty.needs_async_drop(self.tcx(), self.elaborator.param_env()) + { + self.build_async_drop( + self.place, + drop_ty, + None, + self.succ, + unwind, + self.dropline, + false, + ) + } else { + let block = TerminatorKind::Drop { + place: self.place, + target, + unwind: unwind.into_action(), + replace: false, + drop: None, + async_fut: None, + }; + self.new_block(unwind, block) + } + } + fn goto_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock { let block = TerminatorKind::Goto { target }; self.new_block(unwind, block) @@ -971,6 +1304,19 @@ where }) } + fn new_block_with_statements( + &mut self, + unwind: Unwind, + statements: Vec>, + k: TerminatorKind<'tcx>, + ) -> BasicBlock { + self.elaborator.patch().new_block(BasicBlockData { + statements, + terminator: Some(Terminator { source_info: self.source_info, kind: k }), + is_cleanup: unwind.is_cleanup(), + }) + } + fn new_temp(&mut self, ty: Ty<'tcx>) -> Local { self.elaborator.patch().new_temp(ty, self.source_info.span) } diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 7c625823a7187..56ed338a2904d 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -1174,7 +1174,13 @@ fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let def_id = body.source.def_id(); let param_env = tcx.param_env(def_id); - let mut elaborator = DropShimElaborator { body, patch: MirPatch::new(body), tcx, param_env }; + let mut elaborator = DropShimElaborator { + body, + patch: MirPatch::new(body), + tcx, + param_env, + produce_async_drops: false, + }; for (block, block_data) in body.basic_blocks.iter_enumerated() { let (target, unwind, source_info) = match block_data.terminator() { @@ -1213,6 +1219,7 @@ fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { *target, unwind, block, + None, ); } elaborator.patch.apply(body); diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index 15acdd7ae7694..b034f2a38387f 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -165,6 +165,14 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, '_, 'tcx> { self.ctxt.param_env() } + fn allow_async_drops(&self) -> bool { + true + } + + fn terminator_loc(&self, bb: BasicBlock) -> Location { + self.ctxt.patch.terminator_loc(self.ctxt.body, bb) + } + #[instrument(level = "debug", skip(self), ret)] fn drop_style(&self, path: Self::Path, mode: DropFlagMode) -> DropStyle { let ((maybe_live, maybe_dead), multipart) = match mode { @@ -333,7 +341,7 @@ impl<'b, 'mir, 'tcx> ElaborateDropsCtxt<'b, 'mir, 'tcx> { // This function should mirror what `collect_drop_flags` does. for (bb, data) in self.body.basic_blocks.iter_enumerated() { let terminator = data.terminator(); - let TerminatorKind::Drop { place, target, unwind, replace, drop: _, async_fut: _ } = + let TerminatorKind::Drop { place, target, unwind, replace, drop, async_fut: _ } = terminator.kind else { continue; @@ -379,6 +387,7 @@ impl<'b, 'mir, 'tcx> ElaborateDropsCtxt<'b, 'mir, 'tcx> { target, unwind, bb, + drop, ) } LookupResult::Parent(None) => {} diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 31648a5665809..e19726c6811d0 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -281,8 +281,13 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option>) if ty.is_some() { let patch = { let param_env = tcx.param_env_reveal_all_normalized(def_id); - let mut elaborator = - DropShimElaborator { body: &body, patch: MirPatch::new(&body), tcx, param_env }; + let mut elaborator = DropShimElaborator { + body: &body, + patch: MirPatch::new(&body), + tcx, + param_env, + produce_async_drops: false, + }; let dropee = tcx.mk_place_deref(dropee_ptr); let resume_block = elaborator.patch.resume_block(); elaborate_drops::elaborate_drop( @@ -293,6 +298,7 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option>) return_block, elaborate_drops::Unwind::To(resume_block), START_BLOCK, + None, ); elaborator.patch }; @@ -341,6 +347,7 @@ pub struct DropShimElaborator<'a, 'tcx> { pub patch: MirPatch<'tcx>, pub tcx: TyCtxt<'tcx>, pub param_env: ty::ParamEnv<'tcx>, + pub produce_async_drops: bool, } impl fmt::Debug for DropShimElaborator<'_, '_> { @@ -365,6 +372,13 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> { self.param_env } + fn terminator_loc(&self, bb: BasicBlock) -> Location { + self.patch.terminator_loc(self.body, bb) + } + fn allow_async_drops(&self) -> bool { + self.produce_async_drops + } + fn drop_style(&self, _path: Self::Path, mode: DropFlagMode) -> DropStyle { match mode { DropFlagMode::Shallow => {