Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 31d3b5c

Browse files
borsbors
committed
Auto merge of rust-lang#130679 - saethlin:inline-usually, r=<try>
Add inline(usually) r? `@ghost` I'm looking into what kind of things could recover the perf improvement detected in rust-lang#121417 (comment)
2 parents 0af7f0f + e4ff49d commit 31d3b5c

File tree

47 files changed

+586
-347
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+586
-347
lines changed

‎compiler/rustc_attr/src/builtin.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ pub enum InlineAttr {
4646
Hint,
4747
Always,
4848
Never,
49+
Usually,
4950
}
5051

5152
#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

‎compiler/rustc_codegen_gcc/src/attributes.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ fn inline_attr<'gcc, 'tcx>(
3030
None
3131
}
3232
}
33+
InlineAttr::Usually => Some(FnAttribute::Inline),
3334
InlineAttr::None => None,
3435
}
3536
}

‎compiler/rustc_codegen_llvm/src/attributes.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
4747
}
4848
}
4949
InlineAttr::None => None,
50+
InlineAttr::Usually => {
51+
Some(llvm::CreateAttrStringValue(cx.llcx, "function-inline-cost", "0"))
52+
}
5053
}
5154
}
5255

‎compiler/rustc_codegen_ssa/src/codegen_attrs.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -525,9 +525,11 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
525525
.emit();
526526
InlineAttr::None
527527
} else if list_contains_name(items, sym::always) {
528-
InlineAttr::Always
528+
InlineAttr::Usually
529529
} else if list_contains_name(items, sym::never) {
530530
InlineAttr::Never
531+
} else if list_contains_name(items, sym::usually) {
532+
InlineAttr::Usually
531533
} else {
532534
struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
533535
.with_help("valid inline arguments are `always` and `never`")

‎compiler/rustc_mir_transform/src/cross_crate_inline.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
4646
// #[inline(never)] to force code generation.
4747
match codegen_fn_attrs.inline {
4848
InlineAttr::Never => return false,
49-
InlineAttr::Hint | InlineAttr::Always => return true,
49+
InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually => return true,
5050
_ => {}
5151
}
5252

‎compiler/rustc_mir_transform/src/inline.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
106106
changed: false,
107107
caller_is_inline_forwarder: matches!(
108108
codegen_fn_attrs.inline,
109-
InlineAttr::Hint | InlineAttr::Always
109+
InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually
110110
) && body_is_forwarder(body),
111111
};
112112
let blocks = START_BLOCK..body.basic_blocks.next_index();

‎compiler/rustc_span/src/symbol.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2101,6 +2101,7 @@ symbols! {
21012101
usize_legacy_fn_max_value,
21022102
usize_legacy_fn_min_value,
21032103
usize_legacy_mod,
2104+
usually,
21042105
va_arg,
21052106
va_copy,
21062107
va_end,

‎library/core/src/alloc/layout.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ impl Layout {
8484
true
8585
}
8686

87-
#[inline(always)]
87+
#[cfg_attr(bootstrap, inline(always))]
88+
#[cfg_attr(not(bootstrap), inline(usually))]
8889
const fn max_size_for_align(align: Alignment) -> usize {
8990
// (power-of-two implies align != 0.)
9091

‎library/core/src/alloc/mod.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -360,7 +360,8 @@ pub unsafe trait Allocator {
360360
/// Creates a "by reference" adapter for this instance of `Allocator`.
361361
///
362362
/// The returned adapter also implements `Allocator` and will simply borrow this.
363-
#[inline(always)]
363+
#[cfg_attr(bootstrap, inline(always))]
364+
#[cfg_attr(not(bootstrap), inline(usually))]
364365
fn by_ref(&self) -> &Self
365366
where
366367
Self: Sized,

‎library/core/src/cell.rs

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -813,12 +813,14 @@ fn panic_already_mutably_borrowed(err: BorrowError) -> ! {
813813
type BorrowFlag = isize;
814814
const UNUSED: BorrowFlag = 0;
815815

816-
#[inline(always)]
816+
#[cfg_attr(bootstrap, inline(always))]
817+
#[cfg_attr(not(bootstrap), inline(usually))]
817818
fn is_writing(x: BorrowFlag) -> bool {
818819
x < UNUSED
819820
}
820821

821-
#[inline(always)]
822+
#[cfg_attr(bootstrap, inline(always))]
823+
#[cfg_attr(not(bootstrap), inline(usually))]
822824
fn is_reading(x: BorrowFlag) -> bool {
823825
x > UNUSED
824826
}
@@ -2079,7 +2081,8 @@ impl<T> UnsafeCell<T> {
20792081
/// ```
20802082
#[stable(feature = "rust1", since = "1.0.0")]
20812083
#[rustc_const_stable(feature = "const_unsafe_cell_new", since = "1.32.0")]
2082-
#[inline(always)]
2084+
#[cfg_attr(bootstrap, inline(always))]
2085+
#[cfg_attr(not(bootstrap), inline(usually))]
20832086
pub const fn new(value: T) -> UnsafeCell<T> {
20842087
UnsafeCell { value }
20852088
}
@@ -2095,7 +2098,8 @@ impl<T> UnsafeCell<T> {
20952098
///
20962099
/// let five = uc.into_inner();
20972100
/// ```
2098-
#[inline(always)]
2101+
#[cfg_attr(bootstrap, inline(always))]
2102+
#[cfg_attr(not(bootstrap), inline(usually))]
20992103
#[stable(feature = "rust1", since = "1.0.0")]
21002104
// When this is const stabilized, please remove `primitive_into_inner` below.
21012105
#[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")]
@@ -2119,7 +2123,8 @@ impl<T: ?Sized> UnsafeCell<T> {
21192123
/// *uc.get_mut() -= 1;
21202124
/// assert_eq!(*uc.get_mut(), 41);
21212125
/// ```
2122-
#[inline(always)]
2126+
#[cfg_attr(bootstrap, inline(always))]
2127+
#[cfg_attr(not(bootstrap), inline(usually))]
21232128
#[unstable(feature = "unsafe_cell_from_mut", issue = "111645")]
21242129
pub const fn from_mut(value: &mut T) -> &mut UnsafeCell<T> {
21252130
// SAFETY: `UnsafeCell<T>` has the same memory layout as `T` due to #[repr(transparent)].
@@ -2142,7 +2147,8 @@ impl<T: ?Sized> UnsafeCell<T> {
21422147
///
21432148
/// let five = uc.get();
21442149
/// ```
2145-
#[inline(always)]
2150+
#[cfg_attr(bootstrap, inline(always))]
2151+
#[cfg_attr(not(bootstrap), inline(usually))]
21462152
#[stable(feature = "rust1", since = "1.0.0")]
21472153
#[rustc_const_stable(feature = "const_unsafecell_get", since = "1.32.0")]
21482154
#[rustc_never_returns_null_ptr]
@@ -2168,7 +2174,8 @@ impl<T: ?Sized> UnsafeCell<T> {
21682174
///
21692175
/// assert_eq!(*c.get_mut(), 6);
21702176
/// ```
2171-
#[inline(always)]
2177+
#[cfg_attr(bootstrap, inline(always))]
2178+
#[cfg_attr(not(bootstrap), inline(usually))]
21722179
#[stable(feature = "unsafe_cell_get_mut", since = "1.50.0")]
21732180
#[rustc_const_unstable(feature = "const_unsafecell_get_mut", issue = "88836")]
21742181
pub const fn get_mut(&mut self) -> &mut T {
@@ -2203,7 +2210,8 @@ impl<T: ?Sized> UnsafeCell<T> {
22032210
///
22042211
/// assert_eq!(uc.into_inner(), 5);
22052212
/// ```
2206-
#[inline(always)]
2213+
#[cfg_attr(bootstrap, inline(always))]
2214+
#[cfg_attr(not(bootstrap), inline(usually))]
22072215
#[stable(feature = "unsafe_cell_raw_get", since = "1.56.0")]
22082216
#[rustc_const_stable(feature = "unsafe_cell_raw_get", since = "1.56.0")]
22092217
#[rustc_diagnostic_item = "unsafe_cell_raw_get"]

‎library/core/src/clone.rs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ mod impls {
321321
$(
322322
#[stable(feature = "rust1", since = "1.0.0")]
323323
impl Clone for $t {
324-
#[inline(always)]
324+
#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
325325
fn clone(&self) -> Self {
326326
*self
327327
}
@@ -347,15 +347,17 @@ mod impls {
347347

348348
#[stable(feature = "rust1", since = "1.0.0")]
349349
impl<T: ?Sized> Clone for *const T {
350-
#[inline(always)]
350+
#[cfg_attr(bootstrap, inline(always))]
351+
#[cfg_attr(not(bootstrap), inline(usually))]
351352
fn clone(&self) -> Self {
352353
*self
353354
}
354355
}
355356

356357
#[stable(feature = "rust1", since = "1.0.0")]
357358
impl<T: ?Sized> Clone for *mut T {
358-
#[inline(always)]
359+
#[cfg_attr(bootstrap, inline(always))]
360+
#[cfg_attr(not(bootstrap), inline(usually))]
359361
fn clone(&self) -> Self {
360362
*self
361363
}
@@ -364,7 +366,8 @@ mod impls {
364366
/// Shared references can be cloned, but mutable references *cannot*!
365367
#[stable(feature = "rust1", since = "1.0.0")]
366368
impl<T: ?Sized> Clone for &T {
367-
#[inline(always)]
369+
#[cfg_attr(bootstrap, inline(always))]
370+
#[cfg_attr(not(bootstrap), inline(usually))]
368371
#[rustc_diagnostic_item = "noop_method_clone"]
369372
fn clone(&self) -> Self {
370373
*self

‎library/core/src/cmp.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1520,13 +1520,13 @@ mod impls {
15201520
(true, true) => Some(Equal),
15211521
}
15221522
}
1523-
#[inline(always)]
1523+
#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
15241524
fn lt(&self, other: &$t) -> bool { (*self) < (*other) }
1525-
#[inline(always)]
1525+
#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
15261526
fn le(&self, other: &$t) -> bool { (*self) <= (*other) }
1527-
#[inline(always)]
1527+
#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
15281528
fn ge(&self, other: &$t) -> bool { (*self) >= (*other) }
1529-
#[inline(always)]
1529+
#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
15301530
fn gt(&self, other: &$t) -> bool { (*self) > (*other) }
15311531
}
15321532
)*)
@@ -1558,13 +1558,13 @@ mod impls {
15581558
fn partial_cmp(&self, other: &$t) -> Option<Ordering> {
15591559
Some(crate::intrinsics::three_way_compare(*self, *other))
15601560
}
1561-
#[inline(always)]
1561+
#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
15621562
fn lt(&self, other: &$t) -> bool { (*self) < (*other) }
1563-
#[inline(always)]
1563+
#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
15641564
fn le(&self, other: &$t) -> bool { (*self) <= (*other) }
1565-
#[inline(always)]
1565+
#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
15661566
fn ge(&self, other: &$t) -> bool { (*self) >= (*other) }
1567-
#[inline(always)]
1567+
#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
15681568
fn gt(&self, other: &$t) -> bool { (*self) > (*other) }
15691569
}
15701570

‎library/core/src/convert/mod.rs

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,8 @@ pub use num::FloatToInt;
9999
/// ```
100100
#[stable(feature = "convert_id", since = "1.33.0")]
101101
#[rustc_const_stable(feature = "const_identity", since = "1.33.0")]
102-
#[inline(always)]
102+
#[cfg_attr(bootstrap, inline(always))]
103+
#[cfg_attr(not(bootstrap), inline(usually))]
103104
#[rustc_diagnostic_item = "convert_identity"]
104105
pub const fn identity<T>(x: T) -> T {
105106
x
@@ -764,7 +765,8 @@ where
764765
#[stable(feature = "rust1", since = "1.0.0")]
765766
impl<T> From<T> for T {
766767
/// Returns the argument unchanged.
767-
#[inline(always)]
768+
#[cfg_attr(bootstrap, inline(always))]
769+
#[cfg_attr(not(bootstrap), inline(usually))]
768770
fn from(t: T) -> T {
769771
t
770772
}
@@ -820,31 +822,35 @@ where
820822

821823
#[stable(feature = "rust1", since = "1.0.0")]
822824
impl<T> AsRef<[T]> for [T] {
823-
#[inline(always)]
825+
#[cfg_attr(bootstrap, inline(always))]
826+
#[cfg_attr(not(bootstrap), inline(usually))]
824827
fn as_ref(&self) -> &[T] {
825828
self
826829
}
827830
}
828831

829832
#[stable(feature = "rust1", since = "1.0.0")]
830833
impl<T> AsMut<[T]> for [T] {
831-
#[inline(always)]
834+
#[cfg_attr(bootstrap, inline(always))]
835+
#[cfg_attr(not(bootstrap), inline(usually))]
832836
fn as_mut(&mut self) -> &mut [T] {
833837
self
834838
}
835839
}
836840

837841
#[stable(feature = "rust1", since = "1.0.0")]
838842
impl AsRef<str> for str {
839-
#[inline(always)]
843+
#[cfg_attr(bootstrap, inline(always))]
844+
#[cfg_attr(not(bootstrap), inline(usually))]
840845
fn as_ref(&self) -> &str {
841846
self
842847
}
843848
}
844849

845850
#[stable(feature = "as_mut_str_for_str", since = "1.51.0")]
846851
impl AsMut<str> for str {
847-
#[inline(always)]
852+
#[cfg_attr(bootstrap, inline(always))]
853+
#[cfg_attr(not(bootstrap), inline(usually))]
848854
fn as_mut(&mut self) -> &mut str {
849855
self
850856
}

‎library/core/src/convert/num.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ macro_rules! impl_from {
7373
// Rustdocs on the impl block show a "[+] show undocumented items" toggle.
7474
// Rustdocs on functions do not.
7575
#[doc = $doc]
76-
#[inline(always)]
76+
#[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
7777
fn from(small: $Small) -> Self {
7878
small as Self
7979
}

‎library/core/src/default.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,8 @@ macro_rules! default_impl {
151151
($t:ty, $v:expr, $doc:tt) => {
152152
#[stable(feature = "rust1", since = "1.0.0")]
153153
impl Default for $t {
154-
#[inline(always)]
154+
#[cfg_attr(bootstrap, inline(always))]
155+
#[cfg_attr(not(bootstrap), inline(usually))]
155156
#[doc = $doc]
156157
fn default() -> $t {
157158
$v

0 commit comments

Comments
 (0)