Skip to content

Commit f86169a

Browse files
committed
mir_transform: implement forced inlining
Adds `#[rustc_force_inline]` which is similar to always inlining but reports an error if the inlining was not possible, and which always attempts to inline annotated items, regardless of optimisation levels. It can only be applied to free functions to guarantee that the MIR inliner will be able to resolve calls.
1 parent 336209e commit f86169a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+2091
-670
lines changed

compiler/rustc_attr_data_structures/src/attributes.rs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,22 @@ pub enum InlineAttr {
1111
Hint,
1212
Always,
1313
Never,
14+
/// `#[rustc_force_inline]` forces inlining to happen in the MIR inliner - it reports an error
15+
/// if the inlining cannot happen. It is limited to only free functions so that the calls
16+
/// can always be resolved.
17+
Force {
18+
attr_span: Span,
19+
reason: Option<Symbol>,
20+
},
21+
}
22+
23+
impl InlineAttr {
24+
pub fn always(&self) -> bool {
25+
match self {
26+
InlineAttr::Always | InlineAttr::Force { .. } => true,
27+
InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
28+
}
29+
}
1430
}
1531

1632
#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

compiler/rustc_codegen_gcc/src/attributes.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ fn inline_attr<'gcc, 'tcx>(
2020
) -> Option<FnAttribute<'gcc>> {
2121
match inline {
2222
InlineAttr::Hint => Some(FnAttribute::Inline),
23-
InlineAttr::Always => Some(FnAttribute::AlwaysInline),
23+
InlineAttr::Always | InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline),
2424
InlineAttr::Never => {
2525
if cx.sess().target.arch != "amdgpu" {
2626
Some(FnAttribute::NoInline)

compiler/rustc_codegen_llvm/src/attributes.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
3737
}
3838
match inline {
3939
InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
40-
InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)),
40+
InlineAttr::Always | InlineAttr::Force { .. } => {
41+
Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
42+
}
4143
InlineAttr::Never => {
4244
if cx.sess().target.arch != "amdgpu" {
4345
Some(AttributeKind::NoInline.create_attr(cx.llcx))

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

Lines changed: 39 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use rustc_session::parse::feature_err;
1818
use rustc_session::{Session, lint};
1919
use rustc_span::{Ident, Span, sym};
2020
use rustc_target::spec::{SanitizerSet, abi};
21+
use tracing::debug;
2122

2223
use crate::errors;
2324
use crate::target_features::{check_target_feature_trait_unsafe, from_target_feature_attr};
@@ -522,26 +523,36 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
522523
mixed_export_name_no_mangle_lint_state.lint_if_mixed(tcx);
523524

524525
codegen_fn_attrs.inline = attrs.iter().fold(InlineAttr::None, |ia, attr| {
525-
if !attr.has_name(sym::inline) {
526-
return ia;
527-
}
528-
if attr.is_word() {
529-
InlineAttr::Hint
530-
} else if let Some(ref items) = attr.meta_item_list() {
531-
inline_span = Some(attr.span);
532-
if items.len() != 1 {
533-
struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument").emit();
534-
InlineAttr::None
535-
} else if list_contains_name(items, sym::always) {
536-
InlineAttr::Always
537-
} else if list_contains_name(items, sym::never) {
538-
InlineAttr::Never
539-
} else {
540-
struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
541-
.with_help("valid inline arguments are `always` and `never`")
542-
.emit();
526+
if attr.has_name(sym::inline) {
527+
if attr.is_word() {
528+
InlineAttr::Hint
529+
} else if let Some(ref items) = attr.meta_item_list() {
530+
inline_span = Some(attr.span);
531+
if items.len() != 1 {
532+
struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument").emit();
533+
InlineAttr::None
534+
} else if list_contains_name(items, sym::always) {
535+
InlineAttr::Always
536+
} else if list_contains_name(items, sym::never) {
537+
InlineAttr::Never
538+
} else {
539+
struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
540+
.with_help("valid inline arguments are `always` and `never`")
541+
.emit();
543542

544-
InlineAttr::None
543+
InlineAttr::None
544+
}
545+
} else {
546+
ia
547+
}
548+
} else if attr.has_name(sym::rustc_force_inline) && tcx.features().rustc_attrs() {
549+
if attr.is_word() {
550+
InlineAttr::Force { attr_span: attr.span, reason: None }
551+
} else if let Some(val) = attr.value_str() {
552+
InlineAttr::Force { attr_span: attr.span, reason: Some(val) }
553+
} else {
554+
debug!("`rustc_force_inline` not checked by attribute validation");
555+
ia
545556
}
546557
} else {
547558
ia
@@ -596,7 +607,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
596607
// is probably a poor usage of `#[inline(always)]` and easily avoided by not using the attribute.
597608
if tcx.features().target_feature_11()
598609
&& tcx.is_closure_like(did.to_def_id())
599-
&& codegen_fn_attrs.inline != InlineAttr::Always
610+
&& !codegen_fn_attrs.inline.always()
600611
{
601612
let owner_id = tcx.parent(did.to_def_id());
602613
if tcx.def_kind(owner_id).has_codegen_attrs() {
@@ -606,11 +617,15 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
606617
}
607618
}
608619

609-
// If a function uses #[target_feature] it can't be inlined into general
620+
// If a function uses `#[target_feature]` it can't be inlined into general
610621
// purpose functions as they wouldn't have the right target features
611-
// enabled. For that reason we also forbid #[inline(always)] as it can't be
622+
// enabled. For that reason we also forbid `#[inline(always)]` as it can't be
612623
// respected.
613-
if !codegen_fn_attrs.target_features.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always
624+
//
625+
// `#[rustc_force_inline]` doesn't need to be prohibited here, that
626+
// is implemented entirely in rustc can attempt to inline and error if it cannot.
627+
if !codegen_fn_attrs.target_features.is_empty()
628+
&& matches!(codegen_fn_attrs.inline, InlineAttr::Always)
614629
{
615630
if let Some(span) = inline_span {
616631
tcx.dcx().span_err(
@@ -621,7 +636,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
621636
}
622637
}
623638

624-
if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always {
639+
if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() {
625640
if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) {
626641
let hir_id = tcx.local_def_id_to_hir_id(did);
627642
tcx.node_span_lint(

compiler/rustc_feature/src/builtin_attrs.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1019,6 +1019,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
10191019
rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
10201020
"#[rustc_no_mir_inline] prevents the MIR inliner from inlining a function while not affecting codegen"
10211021
),
1022+
rustc_attr!(
1023+
rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes,
1024+
"#![rustc_force_inline] forces a free function to be inlined"
1025+
),
10221026

10231027
// ==========================================================================
10241028
// Internal attributes, Testing:

compiler/rustc_hir_typeck/src/coercion.rs

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
use std::ops::Deref;
3939

4040
use rustc_abi::ExternAbi;
41+
use rustc_attr_parsing::InlineAttr;
4142
use rustc_errors::codes::*;
4243
use rustc_errors::{Applicability, Diag, struct_span_code_err};
4344
use rustc_hir as hir;
@@ -926,8 +927,13 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
926927
return Err(TypeError::IntrinsicCast);
927928
}
928929

929-
// Safe `#[target_feature]` functions are not assignable to safe fn pointers (RFC 2396).
930+
let fn_attrs = self.tcx.codegen_fn_attrs(def_id);
931+
if matches!(fn_attrs.inline, InlineAttr::Force { .. }) {
932+
return Err(TypeError::ForceInlineCast);
933+
}
930934

935+
// Safe `#[target_feature]` functions are not assignable to safe fn pointers
936+
// (RFC 2396).
931937
if b_hdr.safety.is_safe()
932938
&& !self.tcx.codegen_fn_attrs(def_id).target_features.is_empty()
933939
{
@@ -1197,6 +1203,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
11971203
return Ok(prev_ty);
11981204
}
11991205

1206+
let is_force_inline = |ty: Ty<'tcx>| {
1207+
if let ty::FnDef(did, _) = ty.kind() {
1208+
matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. })
1209+
} else {
1210+
false
1211+
}
1212+
};
1213+
if is_force_inline(prev_ty) || is_force_inline(new_ty) {
1214+
return Err(TypeError::ForceInlineCast);
1215+
}
1216+
12001217
// Special-case that coercion alone cannot handle:
12011218
// Function items or non-capturing closures of differing IDs or GenericArgs.
12021219
let (a_sig, b_sig) = {

compiler/rustc_middle/src/mir/mono.rs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,9 +132,10 @@ impl<'tcx> MonoItem<'tcx> {
132132
// creating one copy of this `#[inline]` function which may
133133
// conflict with upstream crates as it could be an exported
134134
// symbol.
135-
match tcx.codegen_fn_attrs(instance.def_id()).inline {
136-
InlineAttr::Always => InstantiationMode::LocalCopy,
137-
_ => InstantiationMode::GloballyShared { may_conflict: true },
135+
if tcx.codegen_fn_attrs(instance.def_id()).inline.always() {
136+
InstantiationMode::LocalCopy
137+
} else {
138+
InstantiationMode::GloballyShared { may_conflict: true }
138139
}
139140
}
140141
MonoItem::Static(..) | MonoItem::GlobalAsm(..) => {

compiler/rustc_middle/src/ty/error.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,9 @@ impl<'tcx> TypeError<'tcx> {
109109
TypeError::ConstMismatch(ref values) => {
110110
format!("expected `{}`, found `{}`", values.expected, values.found).into()
111111
}
112+
TypeError::ForceInlineCast => {
113+
"cannot coerce functions which must be inlined to function pointers".into()
114+
}
112115
TypeError::IntrinsicCast => "cannot coerce intrinsics to function pointers".into(),
113116
TypeError::TargetFeatureCast(_) => {
114117
"cannot coerce functions with `#[target_feature]` to safe function pointers".into()

compiler/rustc_mir_transform/messages.ftl

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,17 @@ mir_transform_ffi_unwind_call = call to {$foreign ->
1919
mir_transform_fn_item_ref = taking a reference to a function item does not give a function pointer
2020
.suggestion = cast `{$ident}` to obtain a function pointer
2121
22+
mir_transform_force_inline =
23+
`{$callee}` could not be inlined into `{$caller}` but is required to be inlined
24+
.call = ...`{$callee}` called here
25+
.attr = inlining due to this annotation
26+
.caller = within `{$caller}`...
27+
.callee = `{$callee}` defined here
28+
.note = could not be inlined due to: {$reason}
29+
30+
mir_transform_force_inline_justification =
31+
`{$callee}` is required to be inlined to: {$sym}
32+
2233
mir_transform_must_not_suspend = {$pre}`{$def_path}`{$post} held across a suspend point, but should not be
2334
.label = the value is held across this suspend point
2435
.note = {$reason}

compiler/rustc_mir_transform/src/cross_crate_inline.rs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
4646
// #[inline(never)] to force code generation.
4747
match codegen_fn_attrs.inline {
4848
InlineAttr::Never => return false,
49-
InlineAttr::Hint | InlineAttr::Always => return true,
49+
InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
5050
_ => {}
5151
}
5252

@@ -69,8 +69,9 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
6969
// Don't do any inference if codegen optimizations are disabled and also MIR inlining is not
7070
// enabled. This ensures that we do inference even if someone only passes -Zinline-mir,
7171
// which is less confusing than having to also enable -Copt-level=1.
72-
if matches!(tcx.sess.opts.optimize, OptLevel::No) && !pm::should_run_pass(tcx, &inline::Inline)
73-
{
72+
let inliner_will_run = pm::should_run_pass(tcx, &inline::Inline)
73+
|| inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id());
74+
if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run {
7475
return false;
7576
}
7677

0 commit comments

Comments
 (0)