Skip to content

Commit b3a447a

Browse files
committed
mir_transform: implement forced inlining
Adds `#[rustc_force_inline]` which is similar to always inlining but reports an error if the inlining was not possible, and which always attempts to inline annotated items, regardless of optimisation levels. It can only be applied to free functions to guarantee that the MIR inliner will be able to resolve calls.
1 parent 4606a4d commit b3a447a

File tree

45 files changed

+2078
-673
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+2078
-673
lines changed

compiler/rustc_attr/src/builtin.rs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,22 @@ pub enum InlineAttr {
4747
Hint,
4848
Always,
4949
Never,
50+
/// `#[rustc_force_inline]` forces inlining to happen in the MIR inliner - it reports an error
51+
/// if the inlining cannot happen. It is limited to only free functions so that the calls
52+
/// can always be resolved.
53+
Force {
54+
attr_span: Span,
55+
reason: Option<Symbol>,
56+
},
57+
}
58+
59+
impl InlineAttr {
60+
pub fn always(&self) -> bool {
61+
match self {
62+
InlineAttr::Always | InlineAttr::Force { .. } => true,
63+
InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
64+
}
65+
}
5066
}
5167

5268
#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

compiler/rustc_codegen_gcc/src/attributes.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ fn inline_attr<'gcc, 'tcx>(
2020
) -> Option<FnAttribute<'gcc>> {
2121
match inline {
2222
InlineAttr::Hint => Some(FnAttribute::Inline),
23-
InlineAttr::Always => Some(FnAttribute::AlwaysInline),
23+
InlineAttr::Always | InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline),
2424
InlineAttr::Never => {
2525
if cx.sess().target.arch != "amdgpu" {
2626
Some(FnAttribute::NoInline)

compiler/rustc_codegen_llvm/src/attributes.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
3737
}
3838
match inline {
3939
InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
40-
InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)),
40+
InlineAttr::Always | InlineAttr::Force { .. } => {
41+
Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
42+
}
4143
InlineAttr::Never => {
4244
if cx.sess().target.arch != "amdgpu" {
4345
Some(AttributeKind::NoInline.create_attr(cx.llcx))

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

Lines changed: 49 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use rustc_session::{Session, lint};
1818
use rustc_span::symbol::Ident;
1919
use rustc_span::{Span, sym};
2020
use rustc_target::spec::{SanitizerSet, abi};
21+
use tracing::debug;
2122

2223
use crate::errors;
2324
use crate::target_features::{check_target_feature_trait_unsafe, from_target_feature_attr};
@@ -522,31 +523,50 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
522523
mixed_export_name_no_mangle_lint_state.lint_if_mixed(tcx);
523524

524525
codegen_fn_attrs.inline = attrs.iter().fold(InlineAttr::None, |ia, attr| {
525-
if !attr.has_name(sym::inline) {
526-
return ia;
527-
}
528-
match attr.meta_kind() {
529-
Some(MetaItemKind::Word) => InlineAttr::Hint,
530-
Some(MetaItemKind::List(ref items)) => {
531-
inline_span = Some(attr.span);
532-
if items.len() != 1 {
533-
struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument")
534-
.emit();
535-
InlineAttr::None
536-
} else if list_contains_name(items, sym::always) {
537-
InlineAttr::Always
538-
} else if list_contains_name(items, sym::never) {
539-
InlineAttr::Never
540-
} else {
541-
struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
526+
if attr.has_name(sym::inline) {
527+
match attr.meta_kind() {
528+
Some(MetaItemKind::Word) => InlineAttr::Hint,
529+
Some(MetaItemKind::List(ref items)) => {
530+
inline_span = Some(attr.span);
531+
if items.len() != 1 {
532+
struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument")
533+
.emit();
534+
InlineAttr::None
535+
} else if list_contains_name(items, sym::always) {
536+
InlineAttr::Always
537+
} else if list_contains_name(items, sym::never) {
538+
InlineAttr::Never
539+
} else {
540+
struct_span_code_err!(
541+
tcx.dcx(),
542+
items[0].span(),
543+
E0535,
544+
"invalid argument"
545+
)
542546
.with_help("valid inline arguments are `always` and `never`")
543547
.emit();
544548

545-
InlineAttr::None
549+
InlineAttr::None
550+
}
546551
}
552+
Some(MetaItemKind::NameValue(_)) => ia,
553+
None => ia,
547554
}
548-
Some(MetaItemKind::NameValue(_)) => ia,
549-
None => ia,
555+
} else if attr.has_name(sym::rustc_force_inline) && tcx.features().rustc_attrs() {
556+
match attr.meta_kind() {
557+
Some(MetaItemKind::NameValue(lit)) => {
558+
InlineAttr::Force { attr_span: attr.span, reason: Some(lit.symbol) }
559+
}
560+
Some(MetaItemKind::Word) => {
561+
InlineAttr::Force { attr_span: attr.span, reason: None }
562+
}
563+
_ => {
564+
debug!("`rustc_force_inline` not checked by attribute validation");
565+
ia
566+
}
567+
}
568+
} else {
569+
ia
550570
}
551571
});
552572

@@ -601,7 +621,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
601621
// is probably a poor usage of `#[inline(always)]` and easily avoided by not using the attribute.
602622
if tcx.features().target_feature_11()
603623
&& tcx.is_closure_like(did.to_def_id())
604-
&& codegen_fn_attrs.inline != InlineAttr::Always
624+
&& !codegen_fn_attrs.inline.always()
605625
{
606626
let owner_id = tcx.parent(did.to_def_id());
607627
if tcx.def_kind(owner_id).has_codegen_attrs() {
@@ -611,11 +631,15 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
611631
}
612632
}
613633

614-
// If a function uses #[target_feature] it can't be inlined into general
634+
// If a function uses `#[target_feature]` it can't be inlined into general
615635
// purpose functions as they wouldn't have the right target features
616-
// enabled. For that reason we also forbid #[inline(always)] as it can't be
636+
// enabled. For that reason we also forbid `#[inline(always)]` as it can't be
617637
// respected.
618-
if !codegen_fn_attrs.target_features.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always
638+
//
639+
// `#[rustc_force_inline]` doesn't need to be prohibited here, that
640+
// is implemented entirely in rustc can attempt to inline and error if it cannot.
641+
if !codegen_fn_attrs.target_features.is_empty()
642+
&& matches!(codegen_fn_attrs.inline, InlineAttr::Always)
619643
{
620644
if let Some(span) = inline_span {
621645
tcx.dcx().span_err(
@@ -626,7 +650,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
626650
}
627651
}
628652

629-
if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always {
653+
if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() {
630654
if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) {
631655
let hir_id = tcx.local_def_id_to_hir_id(did);
632656
tcx.node_span_lint(

compiler/rustc_feature/src/builtin_attrs.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1014,6 +1014,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
10141014
rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
10151015
"#[rustc_no_mir_inline] prevents the MIR inliner from inlining a function while not affecting codegen"
10161016
),
1017+
rustc_attr!(
1018+
rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes,
1019+
"#![rustc_force_inline] forces a free function to be inlined"
1020+
),
10171021

10181022
// ==========================================================================
10191023
// Internal attributes, Testing:

compiler/rustc_hir_typeck/src/coercion.rs

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
use std::ops::Deref;
3939

4040
use rustc_abi::ExternAbi;
41+
use rustc_attr::InlineAttr;
4142
use rustc_errors::codes::*;
4243
use rustc_errors::{Applicability, Diag, struct_span_code_err};
4344
use rustc_hir as hir;
@@ -923,11 +924,14 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
923924
return Err(TypeError::IntrinsicCast);
924925
}
925926

926-
// Safe `#[target_feature]` functions are not assignable to safe fn pointers (RFC 2396).
927+
let fn_attrs = self.tcx.codegen_fn_attrs(def_id);
928+
if matches!(fn_attrs.inline, rustc_attr::InlineAttr::Force { .. }) {
929+
return Err(TypeError::ForceInlineCast);
930+
}
927931

928-
if b_hdr.safety == hir::Safety::Safe
929-
&& !self.tcx.codegen_fn_attrs(def_id).target_features.is_empty()
930-
{
932+
// Safe `#[target_feature]` functions are not assignable to safe fn pointers
933+
// (RFC 2396).
934+
if b_hdr.safety == hir::Safety::Safe && !fn_attrs.target_features.is_empty() {
931935
return Err(TypeError::TargetFeatureCast(def_id));
932936
}
933937
}
@@ -1194,6 +1198,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
11941198
return Ok(prev_ty);
11951199
}
11961200

1201+
let is_force_inline = |ty: Ty<'tcx>| {
1202+
if let ty::FnDef(did, _) = ty.kind() {
1203+
matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. })
1204+
} else {
1205+
false
1206+
}
1207+
};
1208+
if is_force_inline(prev_ty) || is_force_inline(new_ty) {
1209+
return Err(TypeError::ForceInlineCast);
1210+
}
1211+
11971212
// Special-case that coercion alone cannot handle:
11981213
// Function items or non-capturing closures of differing IDs or GenericArgs.
11991214
let (a_sig, b_sig) = {

compiler/rustc_lint_defs/src/builtin.rs

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ declare_lint_pass! {
4646
EXPORTED_PRIVATE_DEPENDENCIES,
4747
FFI_UNWIND_CALLS,
4848
FORBIDDEN_LINT_GROUPS,
49+
FORCED_INLINE,
4950
FUNCTION_ITEM_REFERENCES,
5051
FUZZY_PROVENANCE_CASTS,
5152
HIDDEN_GLOB_REEXPORTS,
@@ -5178,3 +5179,31 @@ declare_lint! {
51785179
reference: "issue #116558 <https://github.com/rust-lang/rust/issues/116558>",
51795180
};
51805181
}
5182+
5183+
declare_lint! {
5184+
/// The `forced_inline` lint is emitted when a function annotated with
5185+
/// `#[rustc_force_inline]` was not able to be inlined.
5186+
///
5187+
/// ### Example
5188+
///
5189+
/// ```rust,ignore (needs rustc_attrs)
5190+
/// #[rustc_no_mir_inline]
5191+
/// #[rustc_force_inline]
5192+
/// fn foo() { }
5193+
///
5194+
/// fn bar() { foo() }
5195+
/// ```
5196+
///
5197+
/// ### Explanation
5198+
///
5199+
/// Functions can be marked as `#[rustc_force_inline]` in the standard
5200+
/// library if they are required to be inlined in order to uphold
5201+
/// security properties or some other similar guarantee.
5202+
///
5203+
/// In some circumstances, these functions cannot be inlined and a
5204+
/// reason will be provided, this can either be rectified or the
5205+
/// lint can be silenced if the risk is acceptable.
5206+
pub FORCED_INLINE,
5207+
Deny,
5208+
"`#[rustc_force_inline]`-annotated function could not be inlined"
5209+
}

compiler/rustc_middle/src/mir/mono.rs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
use std::fmt;
22
use std::hash::Hash;
33

4-
use rustc_attr::InlineAttr;
54
use rustc_data_structures::base_n::{BaseNString, CASE_INSENSITIVE, ToBaseN};
65
use rustc_data_structures::fingerprint::Fingerprint;
76
use rustc_data_structures::fx::FxIndexMap;
@@ -114,7 +113,8 @@ impl<'tcx> MonoItem<'tcx> {
114113
return InstantiationMode::GloballyShared { may_conflict: false };
115114
}
116115

117-
if let InlineAttr::Never = tcx.codegen_fn_attrs(instance.def_id()).inline
116+
if let rustc_attr::InlineAttr::Never =
117+
tcx.codegen_fn_attrs(instance.def_id()).inline
118118
&& self.is_generic_fn()
119119
{
120120
// Upgrade inline(never) to a globally shared instance.
@@ -133,9 +133,10 @@ impl<'tcx> MonoItem<'tcx> {
133133
// creating one copy of this `#[inline]` function which may
134134
// conflict with upstream crates as it could be an exported
135135
// symbol.
136-
match tcx.codegen_fn_attrs(instance.def_id()).inline {
137-
InlineAttr::Always => InstantiationMode::LocalCopy,
138-
_ => InstantiationMode::GloballyShared { may_conflict: true },
136+
if tcx.codegen_fn_attrs(instance.def_id()).inline.always() {
137+
InstantiationMode::LocalCopy
138+
} else {
139+
InstantiationMode::GloballyShared { may_conflict: true }
139140
}
140141
}
141142
MonoItem::Static(..) | MonoItem::GlobalAsm(..) => {

compiler/rustc_middle/src/ty/error.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,9 @@ impl<'tcx> TypeError<'tcx> {
109109
TypeError::ConstMismatch(ref values) => {
110110
format!("expected `{}`, found `{}`", values.expected, values.found).into()
111111
}
112+
TypeError::ForceInlineCast => {
113+
"cannot coerce functions which must be inlined to function pointers".into()
114+
}
112115
TypeError::IntrinsicCast => "cannot coerce intrinsics to function pointers".into(),
113116
TypeError::TargetFeatureCast(_) => {
114117
"cannot coerce functions with `#[target_feature]` to safe function pointers".into()

compiler/rustc_mir_transform/messages.ftl

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,17 @@ mir_transform_ffi_unwind_call = call to {$foreign ->
1919
mir_transform_fn_item_ref = taking a reference to a function item does not give a function pointer
2020
.suggestion = cast `{$ident}` to obtain a function pointer
2121
22+
mir_transform_force_inline =
23+
`{$callee}` could not be inlined into `{$caller}` but is required to be inlined
24+
.call = ...`{$callee}` called here
25+
.attr = inlining due to this annotation
26+
.caller = within `{$caller}`...
27+
.callee = `{$callee}` defined here
28+
.note = could not be inlined due to: {$reason}
29+
30+
mir_transform_force_inline_justification =
31+
`{$callee}` is required to be inlined to: {$sym}
32+
2233
mir_transform_must_not_suspend = {$pre}`{$def_path}`{$post} held across a suspend point, but should not be
2334
.label = the value is held across this suspend point
2435
.note = {$reason}

0 commit comments

Comments
 (0)