Skip to content

Commit 3405765

Browse files
committed
futex: Avoid redundant task lookup
No need to do the full VPID based task lookup and validation of the top waiter when the user space futex was acquired on it's behalf during the requeue_pi operation. The task is known already and it cannot go away before requeue_pi_wake_futex() has been invoked. Split out the actual attach code from attach_pi_state_owner() and use that instead of the full blown variant. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lore.kernel.org/r/20210902094414.676104881@linutronix.de
1 parent 249955e commit 3405765

File tree

1 file changed

+37
-30
lines changed

1 file changed

+37
-30
lines changed

kernel/futex.c

Lines changed: 37 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1263,6 +1263,36 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval,
12631263
return -ESRCH;
12641264
}
12651265

1266+
static void __attach_to_pi_owner(struct task_struct *p, union futex_key *key,
1267+
struct futex_pi_state **ps)
1268+
{
1269+
/*
1270+
* No existing pi state. First waiter. [2]
1271+
*
1272+
* This creates pi_state, we have hb->lock held, this means nothing can
1273+
* observe this state, wait_lock is irrelevant.
1274+
*/
1275+
struct futex_pi_state *pi_state = alloc_pi_state();
1276+
1277+
/*
1278+
* Initialize the pi_mutex in locked state and make @p
1279+
* the owner of it:
1280+
*/
1281+
rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);
1282+
1283+
/* Store the key for possible exit cleanups: */
1284+
pi_state->key = *key;
1285+
1286+
WARN_ON(!list_empty(&pi_state->list));
1287+
list_add(&pi_state->list, &p->pi_state_list);
1288+
/*
1289+
* Assignment without holding pi_state->pi_mutex.wait_lock is safe
1290+
* because there is no concurrency as the object is not published yet.
1291+
*/
1292+
pi_state->owner = p;
1293+
1294+
*ps = pi_state;
1295+
}
12661296
/*
12671297
* Lookup the task for the TID provided from user space and attach to
12681298
* it after doing proper sanity checks.
@@ -1272,7 +1302,6 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,
12721302
struct task_struct **exiting)
12731303
{
12741304
pid_t pid = uval & FUTEX_TID_MASK;
1275-
struct futex_pi_state *pi_state;
12761305
struct task_struct *p;
12771306

12781307
/*
@@ -1324,36 +1353,11 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,
13241353
return ret;
13251354
}
13261355

1327-
/*
1328-
* No existing pi state. First waiter. [2]
1329-
*
1330-
* This creates pi_state, we have hb->lock held, this means nothing can
1331-
* observe this state, wait_lock is irrelevant.
1332-
*/
1333-
pi_state = alloc_pi_state();
1334-
1335-
/*
1336-
* Initialize the pi_mutex in locked state and make @p
1337-
* the owner of it:
1338-
*/
1339-
rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);
1340-
1341-
/* Store the key for possible exit cleanups: */
1342-
pi_state->key = *key;
1343-
1344-
WARN_ON(!list_empty(&pi_state->list));
1345-
list_add(&pi_state->list, &p->pi_state_list);
1346-
/*
1347-
* Assignment without holding pi_state->pi_mutex.wait_lock is safe
1348-
* because there is no concurrency as the object is not published yet.
1349-
*/
1350-
pi_state->owner = p;
1356+
__attach_to_pi_owner(p, key, ps);
13511357
raw_spin_unlock_irq(&p->pi_lock);
13521358

13531359
put_task_struct(p);
13541360

1355-
*ps = pi_state;
1356-
13571361
return 0;
13581362
}
13591363

@@ -1464,11 +1468,14 @@ static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
14641468
* @task is guaranteed to be alive and it cannot be exiting
14651469
* because it is either sleeping or waiting in
14661470
* futex_requeue_pi_wakeup_sync().
1471+
*
1472+
* No need to do the full attach_to_pi_owner() exercise
1473+
* because @task is known and valid.
14671474
*/
14681475
if (set_waiters) {
1469-
ret = attach_to_pi_owner(uaddr, newval, key, ps,
1470-
exiting);
1471-
WARN_ON(ret);
1476+
raw_spin_lock_irq(&task->pi_lock);
1477+
__attach_to_pi_owner(task, key, ps);
1478+
raw_spin_unlock_irq(&task->pi_lock);
14721479
}
14731480
return 1;
14741481
}

0 commit comments

Comments
 (0)