Skip to content

Commit 025a613

Browse files
azhoginazhogin
committed
Async drop codegen with async drop glue through templated coroutine
1 parent 536235f commit 025a613

File tree

98 files changed

+3385
-1928
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

98 files changed

+3385
-1928
lines changed

‎compiler/rustc_borrowck/src/lib.rs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -696,7 +696,14 @@ impl<'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R>
696696
TerminatorKind::SwitchInt { discr, targets: _ } => {
697697
self.consume_operand(loc, (discr, span), flow_state);
698698
}
699-
TerminatorKind::Drop { place, target: _, unwind: _, replace } => {
699+
TerminatorKind::Drop {
700+
place,
701+
target: _,
702+
unwind: _,
703+
replace,
704+
drop: _,
705+
async_fut: _,
706+
} => {
700707
debug!(
701708
"visit_terminator_drop \
702709
loc: {:?} term: {:?} place: {:?} span: {:?}",

‎compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,14 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
100100
TerminatorKind::SwitchInt { discr, targets: _ } => {
101101
self.consume_operand(location, discr);
102102
}
103-
TerminatorKind::Drop { place: drop_place, target: _, unwind: _, replace } => {
103+
TerminatorKind::Drop {
104+
place: drop_place,
105+
target: _,
106+
unwind: _,
107+
replace,
108+
drop: _,
109+
async_fut: _,
110+
} => {
104111
let write_kind =
105112
if *replace { WriteKind::Replace } else { WriteKind::StorageDeadOrDrop };
106113
self.access_place(

‎compiler/rustc_borrowck/src/type_check/mod.rs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1690,8 +1690,14 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
16901690
}
16911691
}
16921692
TerminatorKind::Unreachable => {}
1693-
TerminatorKind::Drop { target, unwind, .. }
1694-
| TerminatorKind::Assert { target, unwind, .. } => {
1693+
TerminatorKind::Drop { target, unwind, drop, .. } => {
1694+
self.assert_iscleanup(body, block_data, target, is_cleanup);
1695+
self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
1696+
if let Some(drop) = drop {
1697+
self.assert_iscleanup(body, block_data, drop, is_cleanup);
1698+
}
1699+
}
1700+
TerminatorKind::Assert { target, unwind, .. } => {
16951701
self.assert_iscleanup(body, block_data, target, is_cleanup);
16961702
self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
16971703
}

‎compiler/rustc_codegen_cranelift/src/abi/mod.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -412,7 +412,9 @@ pub(crate) fn codegen_terminator_call<'tcx>(
412412
Err(instance) => Some(instance),
413413
}
414414
}
415-
InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None) => {
415+
// We don't need AsyncDropGlueCtorShim here because it is not `noop func`,
416+
// it is `func returning noop future`
417+
InstanceKind::DropGlue(_, None) => {
416418
// empty drop glue - a nop.
417419
let dest = target.expect("Non terminating drop_in_place_real???");
418420
let ret_block = fx.get_block(dest);
@@ -598,9 +600,8 @@ pub(crate) fn codegen_drop<'tcx>(
598600
let ty = drop_place.layout().ty;
599601
let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx);
600602

601-
if let ty::InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None) =
602-
drop_instance.def
603-
{
603+
// AsyncDropGlueCtorShim can't be here
604+
if let ty::InstanceKind::DropGlue(_, None) = drop_instance.def {
604605
// we don't actually need to drop anything
605606
} else {
606607
match ty.kind() {

‎compiler/rustc_codegen_cranelift/src/base.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -546,7 +546,11 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
546546
| TerminatorKind::CoroutineDrop => {
547547
bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
548548
}
549-
TerminatorKind::Drop { place, target, unwind: _, replace: _ } => {
549+
TerminatorKind::Drop { place, target, unwind: _, replace: _, drop, async_fut } => {
550+
assert!(
551+
async_fut.is_none() && drop.is_none(),
552+
"Async Drop must be expanded or reset to sync before codegen"
553+
);
550554
let drop_place = codegen_place(fx, *place);
551555
crate::abi::codegen_drop(fx, source_info, drop_place, *target);
552556
}

‎compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -683,8 +683,7 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>(
683683
_ => unreachable!(),
684684
};
685685

686-
let coroutine_layout =
687-
cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args.kind_ty()).unwrap();
686+
let coroutine_layout = cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args.args).unwrap();
688687

689688
let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id);
690689
let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx);

‎compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -158,10 +158,8 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>(
158158
DIFlags::FlagZero,
159159
),
160160
|cx, coroutine_type_di_node| {
161-
let coroutine_layout = cx
162-
.tcx
163-
.coroutine_layout(coroutine_def_id, coroutine_args.as_coroutine().kind_ty())
164-
.unwrap();
161+
let coroutine_layout =
162+
cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args).unwrap();
165163

166164
let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } =
167165
coroutine_type_and_layout.variants

‎compiler/rustc_codegen_ssa/src/back/symbol_export.rs

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,16 @@ fn exported_symbols_provider_local(
383383
},
384384
));
385385
}
386+
MonoItem::Fn(Instance { def: InstanceKind::AsyncDropGlue(_, ty), args: _ }) => {
387+
symbols.push((
388+
ExportedSymbol::AsyncDropGlue(ty),
389+
SymbolExportInfo {
390+
level: SymbolExportLevel::Rust,
391+
kind: SymbolExportKind::Text,
392+
used: false,
393+
},
394+
));
395+
}
386396
_ => {
387397
// Any other symbols don't qualify for sharing
388398
}
@@ -406,6 +416,8 @@ fn upstream_monomorphizations_provider(
406416

407417
let drop_in_place_fn_def_id = tcx.lang_items().drop_in_place_fn();
408418
let async_drop_in_place_fn_def_id = tcx.lang_items().async_drop_in_place_fn();
419+
let async_drop_in_place_poll_fn_def_id = tcx.lang_items().async_drop_in_place_poll_fn();
420+
let future_drop_poll_fn_def_id = tcx.lang_items().future_drop_poll_fn();
409421

410422
for &cnum in cnums.iter() {
411423
for (exported_symbol, _) in tcx.exported_symbols(cnum).iter() {
@@ -424,8 +436,20 @@ fn upstream_monomorphizations_provider(
424436
if let Some(async_drop_in_place_fn_def_id) = async_drop_in_place_fn_def_id {
425437
(async_drop_in_place_fn_def_id, tcx.mk_args(&[ty.into()]))
426438
} else {
427-
// `drop_in_place` in place does not exist, don't try
428-
// to use it.
439+
continue;
440+
}
441+
}
442+
ExportedSymbol::AsyncDropGlue(ty) => {
443+
if let Some(poll_fn_def_id) = async_drop_in_place_poll_fn_def_id {
444+
(poll_fn_def_id, tcx.mk_args(&[ty.into()]))
445+
} else {
446+
continue;
447+
}
448+
}
449+
ExportedSymbol::FutureDropPoll(ty) => {
450+
if let Some(poll_fn_def_id) = future_drop_poll_fn_def_id {
451+
(poll_fn_def_id, tcx.mk_args(&[ty.into()]))
452+
} else {
429453
continue;
430454
}
431455
}
@@ -577,6 +601,20 @@ pub fn symbol_name_for_instance_in_crate<'tcx>(
577601
instantiating_crate,
578602
)
579603
}
604+
ExportedSymbol::AsyncDropGlue(ty) => {
605+
rustc_symbol_mangling::symbol_name_for_instance_in_crate(
606+
tcx,
607+
Instance::resolve_async_drop_in_place_poll(tcx, ty),
608+
instantiating_crate,
609+
)
610+
}
611+
ExportedSymbol::FutureDropPoll(ty) => {
612+
rustc_symbol_mangling::symbol_name_for_instance_in_crate(
613+
tcx,
614+
Instance::resolve_future_drop_poll(tcx, ty),
615+
instantiating_crate,
616+
)
617+
}
580618
ExportedSymbol::NoDefId(symbol_name) => symbol_name.to_string(),
581619
}
582620
}
@@ -628,6 +666,8 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
628666
// AsyncDropGlueCtorShim always use the Rust calling convention and thus follow the
629667
// target's default symbol decoration scheme.
630668
ExportedSymbol::AsyncDropGlueCtorShim(..) => None,
669+
ExportedSymbol::AsyncDropGlue(..) => None,
670+
ExportedSymbol::FutureDropPoll(..) => None,
631671
// NoDefId always follow the target's default symbol decoration scheme.
632672
ExportedSymbol::NoDefId(..) => None,
633673
// ThreadLocalShim always follow the target's default symbol decoration scheme.

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -854,10 +854,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
854854

855855
let def = instance.map(|i| i.def);
856856

857-
if let Some(
858-
ty::InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None),
859-
) = def
860-
{
857+
// We don't need AsyncDropGlueCtorShim here because it is not `noop func`,
858+
// it is `func returning noop future`
859+
if let Some(ty::InstanceKind::DropGlue(_, None)) = def {
861860
// Empty drop glue; a no-op.
862861
let target = target.unwrap();
863862
return helper.funclet_br(self, bx, target, mergeable_succ);
@@ -1363,16 +1362,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
13631362
MergingSucc::False
13641363
}
13651364

1366-
mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => self
1367-
.codegen_drop_terminator(
1365+
mir::TerminatorKind::Drop { place, target, unwind, replace: _, drop, async_fut } => {
1366+
assert!(
1367+
async_fut.is_none() && drop.is_none(),
1368+
"Async Drop must be expanded or reset to sync before codegen"
1369+
);
1370+
self.codegen_drop_terminator(
13681371
helper,
13691372
bx,
13701373
&terminator.source_info,
13711374
place,
13721375
target,
13731376
unwind,
13741377
mergeable_succ(),
1375-
),
1378+
)
1379+
}
13761380

13771381
mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, unwind } => self
13781382
.codegen_assert_terminator(

‎compiler/rustc_const_eval/src/const_eval/machine.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -572,6 +572,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
572572
RemainderByZero(op) => RemainderByZero(eval_to_int(op)?),
573573
ResumedAfterReturn(coroutine_kind) => ResumedAfterReturn(*coroutine_kind),
574574
ResumedAfterPanic(coroutine_kind) => ResumedAfterPanic(*coroutine_kind),
575+
ResumedAfterDrop(coroutine_kind) => ResumedAfterDrop(*coroutine_kind),
575576
MisalignedPointerDereference { ref required, ref found } => {
576577
MisalignedPointerDereference {
577578
required: eval_to_int(required)?,

0 commit comments

Comments
 (0)