Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 9eafd4a

Browse files
committed
Auto merge of rust-lang#134082 - davidtwco:forced-inlining, r=saethlin
mir_transform: implement `#[rustc_force_inline]` Adds `#[rustc_force_inline]` which is similar to always inlining but reports an error if the inlining was not possible. - `#[rustc_force_inline]` can only be applied to free functions to guarantee that the MIR inliner will be able to resolve calls. - `rustc_mir_transform::inline::Inline` is refactored into two passes (`Inline` and `ForceInline`), sharing the vast majority of the implementation. - `rustc_mir_transform::inline::ForceInline` can't be disabled so annotated items are always inlined. - `rustc_mir_transform::inline::ForceInline` runs regardless of optimisation level. - `#[rustc_force_inline]` won't inline unless target features match, as with normal inlining. - MIR validation will ICE if a `#[rustc_force_inline]` isn't inlined, to guarantee that it will never be codegened independently. As a further guarantee, monomorphisation collection will always decide that `#[rustc_force_inline]` functions cannot be codegened locally. - Like other intrinsics, `#[rustc_force_inline]` annotated functions cannot be cast to function pointers. - As with other rustc attrs, this cannot be used by users, just within the compiler and standard library. - This is only implemented within rustc, so should avoid any limitations of LLVM's inlining. It is intended that this attribute be used with intrinsics that must be inlined for security reasons. For example, pointer authentication intrinsics would allow Rust users to make use of pointer authentication instructions, but if these intrinsic functions were in the binary then they could be used as gadgets with ROP attacks, defeating the point of introducing them. We don't have any intrinsics like this today, but I expect to upstream some once a force inlining mechanism such as this is available. cc rust-lang#131687 rust-lang/rfcs#3711 - this approach should resolve the concerns from these previous attempts r? `@saethlin`
2 parents 336209e + 6bef877 commit 9eafd4a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+2327
-661
lines changed

Cargo.lock

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4158,6 +4158,7 @@ dependencies = [
41584158
"rustc_apfloat",
41594159
"rustc_arena",
41604160
"rustc_ast",
4161+
"rustc_attr_parsing",
41614162
"rustc_data_structures",
41624163
"rustc_errors",
41634164
"rustc_fluent_macro",

compiler/rustc_attr_data_structures/src/attributes.rs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,22 @@ pub enum InlineAttr {
1111
Hint,
1212
Always,
1313
Never,
14+
/// `#[rustc_force_inline]` forces inlining to happen in the MIR inliner - it reports an error
15+
/// if the inlining cannot happen. It is limited to only free functions so that the calls
16+
/// can always be resolved.
17+
Force {
18+
attr_span: Span,
19+
reason: Option<Symbol>,
20+
},
21+
}
22+
23+
impl InlineAttr {
24+
pub fn always(&self) -> bool {
25+
match self {
26+
InlineAttr::Always | InlineAttr::Force { .. } => true,
27+
InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
28+
}
29+
}
1430
}
1531

1632
#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

compiler/rustc_codegen_gcc/src/attributes.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ fn inline_attr<'gcc, 'tcx>(
2020
) -> Option<FnAttribute<'gcc>> {
2121
match inline {
2222
InlineAttr::Hint => Some(FnAttribute::Inline),
23-
InlineAttr::Always => Some(FnAttribute::AlwaysInline),
23+
InlineAttr::Always | InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline),
2424
InlineAttr::Never => {
2525
if cx.sess().target.arch != "amdgpu" {
2626
Some(FnAttribute::NoInline)

compiler/rustc_codegen_llvm/src/attributes.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
3737
}
3838
match inline {
3939
InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
40-
InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)),
40+
InlineAttr::Always | InlineAttr::Force { .. } => {
41+
Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
42+
}
4143
InlineAttr::Never => {
4244
if cx.sess().target.arch != "amdgpu" {
4345
Some(AttributeKind::NoInline.create_attr(cx.llcx))

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use rustc_session::parse::feature_err;
1818
use rustc_session::{Session, lint};
1919
use rustc_span::{Ident, Span, sym};
2020
use rustc_target::spec::{SanitizerSet, abi};
21+
use tracing::debug;
2122

2223
use crate::errors;
2324
use crate::target_features::{check_target_feature_trait_unsafe, from_target_feature_attr};
@@ -525,6 +526,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
525526
if !attr.has_name(sym::inline) {
526527
return ia;
527528
}
529+
528530
if attr.is_word() {
529531
InlineAttr::Hint
530532
} else if let Some(ref items) = attr.meta_item_list() {
@@ -547,6 +549,20 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
547549
ia
548550
}
549551
});
552+
codegen_fn_attrs.inline = attrs.iter().fold(codegen_fn_attrs.inline, |ia, attr| {
553+
if !attr.has_name(sym::rustc_force_inline) || !tcx.features().rustc_attrs() {
554+
return ia;
555+
}
556+
557+
if attr.is_word() {
558+
InlineAttr::Force { attr_span: attr.span, reason: None }
559+
} else if let Some(val) = attr.value_str() {
560+
InlineAttr::Force { attr_span: attr.span, reason: Some(val) }
561+
} else {
562+
debug!("`rustc_force_inline` not checked by attribute validation");
563+
ia
564+
}
565+
});
550566

551567
// naked function MUST NOT be inlined! This attribute is required for the rust compiler itself,
552568
// but not for the code generation backend because at that point the naked function will just be
@@ -596,7 +612,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
596612
// is probably a poor usage of `#[inline(always)]` and easily avoided by not using the attribute.
597613
if tcx.features().target_feature_11()
598614
&& tcx.is_closure_like(did.to_def_id())
599-
&& codegen_fn_attrs.inline != InlineAttr::Always
615+
&& !codegen_fn_attrs.inline.always()
600616
{
601617
let owner_id = tcx.parent(did.to_def_id());
602618
if tcx.def_kind(owner_id).has_codegen_attrs() {
@@ -606,22 +622,28 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
606622
}
607623
}
608624

609-
// If a function uses #[target_feature] it can't be inlined into general
625+
// If a function uses `#[target_feature]` it can't be inlined into general
610626
// purpose functions as they wouldn't have the right target features
611-
// enabled. For that reason we also forbid #[inline(always)] as it can't be
627+
// enabled. For that reason we also forbid `#[inline(always)]` as it can't be
612628
// respected.
613-
if !codegen_fn_attrs.target_features.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always
629+
//
630+
// `#[rustc_force_inline]` doesn't need to be prohibited here, only
631+
// `#[inline(always)]`, as forced inlining is implemented entirely within
632+
// rustc (and so the MIR inliner can do any necessary checks for compatible target
633+
// features).
634+
//
635+
// This sidesteps the LLVM blockers in enabling `target_features` +
636+
// `inline(always)` to be used together (see rust-lang/rust#116573 and
637+
// llvm/llvm-project#70563).
638+
if !codegen_fn_attrs.target_features.is_empty()
639+
&& matches!(codegen_fn_attrs.inline, InlineAttr::Always)
614640
{
615641
if let Some(span) = inline_span {
616-
tcx.dcx().span_err(
617-
span,
618-
"cannot use `#[inline(always)]` with \
619-
`#[target_feature]`",
620-
);
642+
tcx.dcx().span_err(span, "cannot use `#[inline(always)]` with `#[target_feature]`");
621643
}
622644
}
623645

624-
if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always {
646+
if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() {
625647
if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) {
626648
let hir_id = tcx.local_def_id_to_hir_id(did);
627649
tcx.node_span_lint(

compiler/rustc_feature/src/builtin_attrs.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1019,6 +1019,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
10191019
rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
10201020
"#[rustc_no_mir_inline] prevents the MIR inliner from inlining a function while not affecting codegen"
10211021
),
1022+
rustc_attr!(
1023+
rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes,
1024+
"#![rustc_force_inline] forces a free function to be inlined"
1025+
),
10221026

10231027
// ==========================================================================
10241028
// Internal attributes, Testing:

compiler/rustc_hir_typeck/src/coercion.rs

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
use std::ops::Deref;
3939

4040
use rustc_abi::ExternAbi;
41+
use rustc_attr_parsing::InlineAttr;
4142
use rustc_errors::codes::*;
4243
use rustc_errors::{Applicability, Diag, struct_span_code_err};
4344
use rustc_hir as hir;
@@ -926,8 +927,13 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
926927
return Err(TypeError::IntrinsicCast);
927928
}
928929

929-
// Safe `#[target_feature]` functions are not assignable to safe fn pointers (RFC 2396).
930+
let fn_attrs = self.tcx.codegen_fn_attrs(def_id);
931+
if matches!(fn_attrs.inline, InlineAttr::Force { .. }) {
932+
return Err(TypeError::ForceInlineCast);
933+
}
930934

935+
// Safe `#[target_feature]` functions are not assignable to safe fn pointers
936+
// (RFC 2396).
931937
if b_hdr.safety.is_safe()
932938
&& !self.tcx.codegen_fn_attrs(def_id).target_features.is_empty()
933939
{
@@ -1197,6 +1203,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
11971203
return Ok(prev_ty);
11981204
}
11991205

1206+
let is_force_inline = |ty: Ty<'tcx>| {
1207+
if let ty::FnDef(did, _) = ty.kind() {
1208+
matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. })
1209+
} else {
1210+
false
1211+
}
1212+
};
1213+
if is_force_inline(prev_ty) || is_force_inline(new_ty) {
1214+
return Err(TypeError::ForceInlineCast);
1215+
}
1216+
12001217
// Special-case that coercion alone cannot handle:
12011218
// Function items or non-capturing closures of differing IDs or GenericArgs.
12021219
let (a_sig, b_sig) = {

compiler/rustc_middle/src/mir/mono.rs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,9 +132,10 @@ impl<'tcx> MonoItem<'tcx> {
132132
// creating one copy of this `#[inline]` function which may
133133
// conflict with upstream crates as it could be an exported
134134
// symbol.
135-
match tcx.codegen_fn_attrs(instance.def_id()).inline {
136-
InlineAttr::Always => InstantiationMode::LocalCopy,
137-
_ => InstantiationMode::GloballyShared { may_conflict: true },
135+
if tcx.codegen_fn_attrs(instance.def_id()).inline.always() {
136+
InstantiationMode::LocalCopy
137+
} else {
138+
InstantiationMode::GloballyShared { may_conflict: true }
138139
}
139140
}
140141
MonoItem::Static(..) | MonoItem::GlobalAsm(..) => {

compiler/rustc_middle/src/ty/error.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,9 @@ impl<'tcx> TypeError<'tcx> {
109109
TypeError::ConstMismatch(ref values) => {
110110
format!("expected `{}`, found `{}`", values.expected, values.found).into()
111111
}
112+
TypeError::ForceInlineCast => {
113+
"cannot coerce functions which must be inlined to function pointers".into()
114+
}
112115
TypeError::IntrinsicCast => "cannot coerce intrinsics to function pointers".into(),
113116
TypeError::TargetFeatureCast(_) => {
114117
"cannot coerce functions with `#[target_feature]` to safe function pointers".into()

compiler/rustc_mir_build/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ rustc_abi = { path = "../rustc_abi" }
1212
rustc_apfloat = "0.2.0"
1313
rustc_arena = { path = "../rustc_arena" }
1414
rustc_ast = { path = "../rustc_ast" }
15+
rustc_attr_parsing = { path = "../rustc_attr_parsing" }
1516
rustc_data_structures = { path = "../rustc_data_structures" }
1617
rustc_errors = { path = "../rustc_errors" }
1718
rustc_fluent_macro = { path = "../rustc_fluent_macro" }

0 commit comments

Comments
 (0)