Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 31d3b5c

Browse files
committed
Auto merge of rust-lang#130679 - saethlin:inline-usually, r=<try>
Add inline(usually) r? `@ghost` I'm looking into what kind of things could recover the perf improvement detected in rust-lang#121417 (comment)
2 parents 0af7f0f + e4ff49d commit 31d3b5c

File tree

47 files changed

+586
-347
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+586
-347
lines changed

compiler/rustc_attr/src/builtin.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ pub enum InlineAttr {
4646
Hint,
4747
Always,
4848
Never,
49+
Usually,
4950
}
5051

5152
#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

compiler/rustc_codegen_gcc/src/attributes.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ fn inline_attr<'gcc, 'tcx>(
3030
None
3131
}
3232
}
33+
InlineAttr::Usually => Some(FnAttribute::Inline),
3334
InlineAttr::None => None,
3435
}
3536
}

compiler/rustc_codegen_llvm/src/attributes.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
4747
}
4848
}
4949
InlineAttr::None => None,
50+
InlineAttr::Usually => {
51+
Some(llvm::CreateAttrStringValue(cx.llcx, "function-inline-cost", "0"))
52+
}
5053
}
5154
}
5255

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -525,9 +525,11 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
525525
.emit();
526526
InlineAttr::None
527527
} else if list_contains_name(items, sym::always) {
528-
InlineAttr::Always
528+
InlineAttr::Usually
529529
} else if list_contains_name(items, sym::never) {
530530
InlineAttr::Never
531+
} else if list_contains_name(items, sym::usually) {
532+
InlineAttr::Usually
531533
} else {
532534
struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
533535
.with_help("valid inline arguments are `always` and `never`")

compiler/rustc_mir_transform/src/cross_crate_inline.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
4646
// #[inline(never)] to force code generation.
4747
match codegen_fn_attrs.inline {
4848
InlineAttr::Never => return false,
49-
InlineAttr::Hint | InlineAttr::Always => return true,
49+
InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually => return true,
5050
_ => {}
5151
}
5252

compiler/rustc_mir_transform/src/inline.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
106106
changed: false,
107107
caller_is_inline_forwarder: matches!(
108108
codegen_fn_attrs.inline,
109-
InlineAttr::Hint | InlineAttr::Always
109+
InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually
110110
) && body_is_forwarder(body),
111111
};
112112
let blocks = START_BLOCK..body.basic_blocks.next_index();

compiler/rustc_span/src/symbol.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2101,6 +2101,7 @@ symbols! {
21012101
usize_legacy_fn_max_value,
21022102
usize_legacy_fn_min_value,
21032103
usize_legacy_mod,
2104+
usually,
21042105
va_arg,
21052106
va_copy,
21062107
va_end,

library/core/src/alloc/layout.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ impl Layout {
8484
true
8585
}
8686

87-
#[inline(always)]
87+
#[cfg_attr(bootstrap, inline(always))]
88+
#[cfg_attr(not(bootstrap), inline(usually))]
8889
const fn max_size_for_align(align: Alignment) -> usize {
8990
// (power-of-two implies align != 0.)
9091

library/core/src/alloc/mod.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -360,7 +360,8 @@ pub unsafe trait Allocator {
360360
/// Creates a "by reference" adapter for this instance of `Allocator`.
361361
///
362362
/// The returned adapter also implements `Allocator` and will simply borrow this.
363-
#[inline(always)]
363+
#[cfg_attr(bootstrap, inline(always))]
364+
#[cfg_attr(not(bootstrap), inline(usually))]
364365
fn by_ref(&self) -> &Self
365366
where
366367
Self: Sized,

library/core/src/cell.rs

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -813,12 +813,14 @@ fn panic_already_mutably_borrowed(err: BorrowError) -> ! {
813813
type BorrowFlag = isize;
814814
const UNUSED: BorrowFlag = 0;
815815

816-
#[inline(always)]
816+
#[cfg_attr(bootstrap, inline(always))]
817+
#[cfg_attr(not(bootstrap), inline(usually))]
817818
fn is_writing(x: BorrowFlag) -> bool {
818819
x < UNUSED
819820
}
820821

821-
#[inline(always)]
822+
#[cfg_attr(bootstrap, inline(always))]
823+
#[cfg_attr(not(bootstrap), inline(usually))]
822824
fn is_reading(x: BorrowFlag) -> bool {
823825
x > UNUSED
824826
}
@@ -2079,7 +2081,8 @@ impl<T> UnsafeCell<T> {
20792081
/// ```
20802082
#[stable(feature = "rust1", since = "1.0.0")]
20812083
#[rustc_const_stable(feature = "const_unsafe_cell_new", since = "1.32.0")]
2082-
#[inline(always)]
2084+
#[cfg_attr(bootstrap, inline(always))]
2085+
#[cfg_attr(not(bootstrap), inline(usually))]
20832086
pub const fn new(value: T) -> UnsafeCell<T> {
20842087
UnsafeCell { value }
20852088
}
@@ -2095,7 +2098,8 @@ impl<T> UnsafeCell<T> {
20952098
///
20962099
/// let five = uc.into_inner();
20972100
/// ```
2098-
#[inline(always)]
2101+
#[cfg_attr(bootstrap, inline(always))]
2102+
#[cfg_attr(not(bootstrap), inline(usually))]
20992103
#[stable(feature = "rust1", since = "1.0.0")]
21002104
// When this is const stabilized, please remove `primitive_into_inner` below.
21012105
#[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")]
@@ -2119,7 +2123,8 @@ impl<T: ?Sized> UnsafeCell<T> {
21192123
/// *uc.get_mut() -= 1;
21202124
/// assert_eq!(*uc.get_mut(), 41);
21212125
/// ```
2122-
#[inline(always)]
2126+
#[cfg_attr(bootstrap, inline(always))]
2127+
#[cfg_attr(not(bootstrap), inline(usually))]
21232128
#[unstable(feature = "unsafe_cell_from_mut", issue = "111645")]
21242129
pub const fn from_mut(value: &mut T) -> &mut UnsafeCell<T> {
21252130
// SAFETY: `UnsafeCell<T>` has the same memory layout as `T` due to #[repr(transparent)].
@@ -2142,7 +2147,8 @@ impl<T: ?Sized> UnsafeCell<T> {
21422147
///
21432148
/// let five = uc.get();
21442149
/// ```
2145-
#[inline(always)]
2150+
#[cfg_attr(bootstrap, inline(always))]
2151+
#[cfg_attr(not(bootstrap), inline(usually))]
21462152
#[stable(feature = "rust1", since = "1.0.0")]
21472153
#[rustc_const_stable(feature = "const_unsafecell_get", since = "1.32.0")]
21482154
#[rustc_never_returns_null_ptr]
@@ -2168,7 +2174,8 @@ impl<T: ?Sized> UnsafeCell<T> {
21682174
///
21692175
/// assert_eq!(*c.get_mut(), 6);
21702176
/// ```
2171-
#[inline(always)]
2177+
#[cfg_attr(bootstrap, inline(always))]
2178+
#[cfg_attr(not(bootstrap), inline(usually))]
21722179
#[stable(feature = "unsafe_cell_get_mut", since = "1.50.0")]
21732180
#[rustc_const_unstable(feature = "const_unsafecell_get_mut", issue = "88836")]
21742181
pub const fn get_mut(&mut self) -> &mut T {
@@ -2203,7 +2210,8 @@ impl<T: ?Sized> UnsafeCell<T> {
22032210
///
22042211
/// assert_eq!(uc.into_inner(), 5);
22052212
/// ```
2206-
#[inline(always)]
2213+
#[cfg_attr(bootstrap, inline(always))]
2214+
#[cfg_attr(not(bootstrap), inline(usually))]
22072215
#[stable(feature = "unsafe_cell_raw_get", since = "1.56.0")]
22082216
#[rustc_const_stable(feature = "unsafe_cell_raw_get", since = "1.56.0")]
22092217
#[rustc_diagnostic_item = "unsafe_cell_raw_get"]

0 commit comments

Comments
 (0)