Skip to content

Commit e897ba5

Browse files
yvtyvt
committed
perf: add #[inline(always)] to functions that are supposed to be inlined but aren't
Reduces the static and dynamic instruction count. Looking at the compiled code (compiled with `-C lto -C codegen-units=1 -C opt-level=z`), I was surprised to see that the calls to these functions aren't inlined despite the fact that each of them has exactly one call site. For instance, I saw the following "physical" call sequence: Reset → main → ? → ? → State::port_boot → Kernel::boot → PortThreading::dispatch_first_task → State::dispatch_first_task → State::idle_task After this change, the call sequence has been shortened to this: Reset → main → ? → State::port_boot → State::idle_task
1 parent 615d5b9 commit e897ba5

File tree

7 files changed

+36
-1
lines changed

7 files changed

+36
-1
lines changed

‎src/r3_kernel/src/lib.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -713,6 +713,7 @@ pub trait PortToKernel {
713713
}
714714

715715
impl<Traits: KernelTraits> PortToKernel for Traits {
716+
#[inline(always)]
716717
unsafe fn boot() -> ! {
717718
let mut lock = unsafe { klock::assume_cpu_lock::<Traits>() };
718719

@@ -749,7 +750,7 @@ impl<Traits: KernelTraits> PortToKernel for Traits {
749750
}
750751
}
751752

752-
#[inline]
753+
#[inline(always)]
753754
unsafe fn choose_running_task() {
754755
// Safety: The precondition of this method includes CPU Lock being
755756
// active
@@ -761,6 +762,7 @@ impl<Traits: KernelTraits> PortToKernel for Traits {
761762
forget(lock);
762763
}
763764

765+
#[inline(always)]
764766
unsafe fn timer_tick() {
765767
timeout::handle_tick::<Traits>();
766768
}

‎src/r3_port_arm/src/threading/cfg.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ macro_rules! use_port {
9696
unsafe impl PortInstance for $Traits {}
9797

9898
impl EntryPoint for $Traits {
99+
#[inline]
99100
unsafe fn start() -> ! {
100101
unsafe { <Self as PortInstance>::port_state().port_boot::<Self>() }
101102
}
@@ -119,14 +120,17 @@ macro_rules! use_port {
119120
// must be double-word aligned."
120121
const STACK_ALIGN: usize = 8;
121122

123+
#[inline(always)]
122124
unsafe fn dispatch_first_task() -> ! {
123125
<Self as PortInstance>::port_state().dispatch_first_task::<Self>()
124126
}
125127

128+
#[inline(always)]
126129
unsafe fn yield_cpu() {
127130
<Self as PortInstance>::port_state().yield_cpu::<Self>()
128131
}
129132

133+
#[inline(always)]
130134
unsafe fn exit_and_dispatch(task: &'static TaskCb<Self>) -> ! {
131135
<Self as PortInstance>::port_state().exit_and_dispatch::<Self>(task)
132136
}
@@ -141,14 +145,17 @@ macro_rules! use_port {
141145
<Self as PortInstance>::port_state().leave_cpu_lock::<Self>()
142146
}
143147

148+
#[inline(always)]
144149
unsafe fn initialize_task_state(task: &'static TaskCb<Self>) {
145150
<Self as PortInstance>::port_state().initialize_task_state::<Self>(task)
146151
}
147152

153+
#[inline(always)]
148154
fn is_cpu_lock_active() -> bool {
149155
<Self as PortInstance>::port_state().is_cpu_lock_active::<Self>()
150156
}
151157

158+
#[inline(always)]
152159
fn is_task_context() -> bool {
153160
<Self as PortInstance>::port_state().is_task_context::<Self>()
154161
}

‎src/r3_port_arm/src/threading/imp.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ pub unsafe trait PortInstance:
1919
{
2020
sym_static!(static PORT_STATE: SymStatic<State> = zeroed!());
2121

22+
#[inline(always)]
2223
fn port_state() -> &'static State {
2324
sym_static(Self::PORT_STATE).as_ref()
2425
}
@@ -58,6 +59,7 @@ impl Init for TaskState {
5859
}
5960

6061
impl State {
62+
#[inline(always)]
6163
pub unsafe fn port_boot<Traits: PortInstance>(&self) -> ! {
6264
unsafe { self.enter_cpu_lock::<Traits>() };
6365

@@ -73,6 +75,7 @@ impl State {
7375
unsafe { <Traits as PortToKernel>::boot() };
7476
}
7577

78+
#[inline(always)]
7679
pub unsafe fn dispatch_first_task<Traits: PortInstance>(&'static self) -> ! {
7780
debug_assert!(self.is_cpu_lock_active::<Traits>());
7881

@@ -338,6 +341,7 @@ impl State {
338341
}
339342
}
340343

344+
#[inline(always)]
341345
pub unsafe fn exit_and_dispatch<Traits: PortInstance>(
342346
&'static self,
343347
_task: &'static TaskCb<Traits>,

‎src/r3_port_arm_m/src/threading/cfg.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ macro_rules! use_port {
119119
cfg::{ThreadingOptions, EntryPoint},
120120
};
121121

122+
#[inline(always)]
122123
pub(super) fn port_state() -> &'static State {
123124
<$Traits as PortInstance>::port_state()
124125
}
@@ -145,14 +146,17 @@ macro_rules! use_port {
145146
// must be double-word aligned."
146147
const STACK_ALIGN: usize = 8;
147148

149+
#[inline(always)]
148150
unsafe fn dispatch_first_task() -> ! {
149151
port_state().dispatch_first_task::<Self>()
150152
}
151153

154+
#[inline(always)]
152155
unsafe fn yield_cpu() {
153156
port_state().yield_cpu::<Self>()
154157
}
155158

159+
#[inline(always)]
156160
unsafe fn exit_and_dispatch(task: &'static TaskCb<Self>) -> ! {
157161
port_state().exit_and_dispatch::<Self>(task);
158162
}
@@ -167,14 +171,17 @@ macro_rules! use_port {
167171
port_state().leave_cpu_lock::<Self>()
168172
}
169173

174+
#[inline(always)]
170175
unsafe fn initialize_task_state(task: &'static TaskCb<Self>) {
171176
port_state().initialize_task_state::<Self>(task)
172177
}
173178

179+
#[inline(always)]
174180
fn is_cpu_lock_active() -> bool {
175181
port_state().is_cpu_lock_active::<Self>()
176182
}
177183

184+
#[inline(always)]
178185
fn is_task_context() -> bool {
179186
port_state().is_task_context::<Self>()
180187
}
@@ -215,6 +222,7 @@ macro_rules! use_port {
215222
}
216223

217224
unsafe impl EntryPoint for $Traits {
225+
#[inline]
218226
unsafe fn start() -> ! {
219227
unsafe { port_state().port_boot::<$Traits>() }
220228
}

‎src/r3_port_arm_m/src/threading/imp.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ pub unsafe trait PortInstance:
3333
{
3434
sym_static!(static PORT_STATE: SymStatic<State> = zeroed!());
3535

36+
#[inline(always)]
3637
fn port_state() -> &'static State {
3738
sym_static(Self::PORT_STATE).as_ref()
3839
}
@@ -77,6 +78,7 @@ impl Init for TaskState {
7778
}
7879

7980
impl State {
81+
#[inline(always)]
8082
pub unsafe fn port_boot<Traits: PortInstance>(&self) -> ! {
8183
unsafe { self.enter_cpu_lock::<Traits>() };
8284

@@ -102,6 +104,7 @@ impl State {
102104
}
103105
}
104106

107+
#[inline(always)]
105108
pub unsafe fn dispatch_first_task<Traits: PortInstance>(&'static self) -> ! {
106109
// Pend PendSV
107110
cortex_m::peripheral::SCB::set_pendsv();
@@ -183,6 +186,7 @@ impl State {
183186
compiler_fence(Ordering::Acquire);
184187
}
185188

189+
#[inline(always)]
186190
pub unsafe fn exit_and_dispatch<Traits: PortInstance>(
187191
&'static self,
188192
_task: &'static TaskCb<Traits>,

‎src/r3_port_riscv/src/threading/cfg.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ macro_rules! use_port {
100100
}
101101

102102
impl EntryPoint for $Traits {
103+
#[inline(always)]
103104
unsafe fn start() -> ! {
104105
unsafe { PORT_STATE.port_boot::<Self>() };
105106
}
@@ -136,14 +137,17 @@ macro_rules! use_port {
136137
// aligned to a word boundary.
137138
const STACK_ALIGN: usize = 16;
138139

140+
#[inline(always)]
139141
unsafe fn dispatch_first_task() -> ! {
140142
PORT_STATE.dispatch_first_task::<Self>()
141143
}
142144

145+
#[inline(always)]
143146
unsafe fn yield_cpu() {
144147
PORT_STATE.yield_cpu::<Self>()
145148
}
146149

150+
#[inline(always)]
147151
unsafe fn exit_and_dispatch(task: &'static TaskCb<Self>) -> ! {
148152
PORT_STATE.exit_and_dispatch::<Self>(task);
149153
}
@@ -163,14 +167,17 @@ macro_rules! use_port {
163167
PORT_STATE.leave_cpu_lock::<Self>()
164168
}
165169

170+
#[inline(always)]
166171
unsafe fn initialize_task_state(task: &'static TaskCb<Self>) {
167172
PORT_STATE.initialize_task_state::<Self>(task)
168173
}
169174

175+
#[inline(always)]
170176
fn is_cpu_lock_active() -> bool {
171177
PORT_STATE.is_cpu_lock_active::<Self>()
172178
}
173179

180+
#[inline(always)]
174181
fn is_task_context() -> bool {
175182
PORT_STATE.is_task_context::<Self>()
176183
}

‎src/r3_port_riscv/src/threading/imp.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,7 @@ impl Init for TaskState {
221221
}
222222

223223
impl State {
224+
#[inline(always)]
224225
pub unsafe fn port_boot<Traits: PortInstance>(&self) -> ! {
225226
unsafe { self.enter_cpu_lock::<Traits>() };
226227

@@ -254,6 +255,7 @@ impl State {
254255
unsafe { <Traits as PortToKernel>::boot() };
255256
}
256257

258+
#[inline(always)]
257259
pub unsafe fn dispatch_first_task<Traits: PortInstance>(&'static self) -> ! {
258260
debug_assert!(self.is_cpu_lock_active::<Traits>());
259261

@@ -820,6 +822,7 @@ impl State {
820822
}
821823
}
822824

825+
#[inline(always)]
823826
pub unsafe fn exit_and_dispatch<Traits: PortInstance>(
824827
&'static self,
825828
_task: &'static TaskCb<Traits>,

0 commit comments

Comments
 (0)