@@ -5518,30 +5518,6 @@ static bool exclusive_event_installable(struct perf_event *event,
5518
5518
5519
5519
static void perf_free_addr_filters (struct perf_event * event );
5520
5520
5521
- static void perf_pending_task_sync (struct perf_event * event )
5522
- {
5523
- struct callback_head * head = & event -> pending_task ;
5524
-
5525
- if (!event -> pending_work )
5526
- return ;
5527
- /*
5528
- * If the task is queued to the current task's queue, we
5529
- * obviously can't wait for it to complete. Simply cancel it.
5530
- */
5531
- if (task_work_cancel (current , head )) {
5532
- event -> pending_work = 0 ;
5533
- local_dec (& event -> ctx -> nr_no_switch_fast );
5534
- return ;
5535
- }
5536
-
5537
- /*
5538
- * All accesses related to the event are within the same RCU section in
5539
- * perf_pending_task(). The RCU grace period before the event is freed
5540
- * will make sure all those accesses are complete by then.
5541
- */
5542
- rcuwait_wait_event (& event -> pending_work_wait , !event -> pending_work , TASK_UNINTERRUPTIBLE );
5543
- }
5544
-
5545
5521
/* vs perf_event_alloc() error */
5546
5522
static void __free_event (struct perf_event * event )
5547
5523
{
@@ -5599,7 +5575,6 @@ static void _free_event(struct perf_event *event)
5599
5575
{
5600
5576
irq_work_sync (& event -> pending_irq );
5601
5577
irq_work_sync (& event -> pending_disable_irq );
5602
- perf_pending_task_sync (event );
5603
5578
5604
5579
unaccount_event (event );
5605
5580
@@ -5692,10 +5667,17 @@ static void perf_remove_from_owner(struct perf_event *event)
5692
5667
5693
5668
static void put_event (struct perf_event * event )
5694
5669
{
5670
+ struct perf_event * parent ;
5671
+
5695
5672
if (!atomic_long_dec_and_test (& event -> refcount ))
5696
5673
return ;
5697
5674
5675
+ parent = event -> parent ;
5698
5676
_free_event (event );
5677
+
5678
+ /* Matches the refcount bump in inherit_event() */
5679
+ if (parent )
5680
+ put_event (parent );
5699
5681
}
5700
5682
5701
5683
/*
@@ -5779,11 +5761,6 @@ int perf_event_release_kernel(struct perf_event *event)
5779
5761
if (tmp == child ) {
5780
5762
perf_remove_from_context (child , DETACH_GROUP );
5781
5763
list_move (& child -> child_list , & free_list );
5782
- /*
5783
- * This matches the refcount bump in inherit_event();
5784
- * this can't be the last reference.
5785
- */
5786
- put_event (event );
5787
5764
} else {
5788
5765
var = & ctx -> refcount ;
5789
5766
}
@@ -5809,7 +5786,8 @@ int perf_event_release_kernel(struct perf_event *event)
5809
5786
void * var = & child -> ctx -> refcount ;
5810
5787
5811
5788
list_del (& child -> child_list );
5812
- free_event (child );
5789
+ /* Last reference unless ->pending_task work is pending */
5790
+ put_event (child );
5813
5791
5814
5792
/*
5815
5793
* Wake any perf_event_free_task() waiting for this event to be
@@ -5820,7 +5798,11 @@ int perf_event_release_kernel(struct perf_event *event)
5820
5798
}
5821
5799
5822
5800
no_ctx :
5823
- put_event (event ); /* Must be the 'last' reference */
5801
+ /*
5802
+ * Last reference unless ->pending_task work is pending on this event
5803
+ * or any of its children.
5804
+ */
5805
+ put_event (event );
5824
5806
return 0 ;
5825
5807
}
5826
5808
EXPORT_SYMBOL_GPL (perf_event_release_kernel );
@@ -7235,12 +7217,6 @@ static void perf_pending_task(struct callback_head *head)
7235
7217
struct perf_event * event = container_of (head , struct perf_event , pending_task );
7236
7218
int rctx ;
7237
7219
7238
- /*
7239
- * All accesses to the event must belong to the same implicit RCU read-side
7240
- * critical section as the ->pending_work reset. See comment in
7241
- * perf_pending_task_sync().
7242
- */
7243
- rcu_read_lock ();
7244
7220
/*
7245
7221
* If we 'fail' here, that's OK, it means recursion is already disabled
7246
7222
* and we won't recurse 'further'.
@@ -7251,9 +7227,8 @@ static void perf_pending_task(struct callback_head *head)
7251
7227
event -> pending_work = 0 ;
7252
7228
perf_sigtrap (event );
7253
7229
local_dec (& event -> ctx -> nr_no_switch_fast );
7254
- rcuwait_wake_up (& event -> pending_work_wait );
7255
7230
}
7256
- rcu_read_unlock ( );
7231
+ put_event ( event );
7257
7232
7258
7233
if (rctx >= 0 )
7259
7234
perf_swevent_put_recursion_context (rctx );
@@ -10248,6 +10223,7 @@ static int __perf_event_overflow(struct perf_event *event,
10248
10223
!task_work_add (current , & event -> pending_task , notify_mode )) {
10249
10224
event -> pending_work = pending_id ;
10250
10225
local_inc (& event -> ctx -> nr_no_switch_fast );
10226
+ WARN_ON_ONCE (!atomic_long_inc_not_zero (& event -> refcount ));
10251
10227
10252
10228
event -> pending_addr = 0 ;
10253
10229
if (valid_sample && (data -> sample_flags & PERF_SAMPLE_ADDR ))
@@ -12610,7 +12586,6 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
12610
12586
init_irq_work (& event -> pending_irq , perf_pending_irq );
12611
12587
event -> pending_disable_irq = IRQ_WORK_INIT_HARD (perf_pending_disable );
12612
12588
init_task_work (& event -> pending_task , perf_pending_task );
12613
- rcuwait_init (& event -> pending_work_wait );
12614
12589
12615
12590
mutex_init (& event -> mmap_mutex );
12616
12591
raw_spin_lock_init (& event -> addr_filters .lock );
@@ -13747,8 +13722,7 @@ perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx)
13747
13722
* Kick perf_poll() for is_event_hup();
13748
13723
*/
13749
13724
perf_event_wakeup (parent_event );
13750
- free_event (event );
13751
- put_event (parent_event );
13725
+ put_event (event );
13752
13726
return ;
13753
13727
}
13754
13728
@@ -13872,13 +13846,11 @@ static void perf_free_event(struct perf_event *event,
13872
13846
list_del_init (& event -> child_list );
13873
13847
mutex_unlock (& parent -> child_mutex );
13874
13848
13875
- put_event (parent );
13876
-
13877
13849
raw_spin_lock_irq (& ctx -> lock );
13878
13850
perf_group_detach (event );
13879
13851
list_del_event (event , ctx );
13880
13852
raw_spin_unlock_irq (& ctx -> lock );
13881
- free_event (event );
13853
+ put_event (event );
13882
13854
}
13883
13855
13884
13856
/*
@@ -14016,6 +13988,9 @@ inherit_event(struct perf_event *parent_event,
14016
13988
if (IS_ERR (child_event ))
14017
13989
return child_event ;
14018
13990
13991
+ get_ctx (child_ctx );
13992
+ child_event -> ctx = child_ctx ;
13993
+
14019
13994
pmu_ctx = find_get_pmu_context (child_event -> pmu , child_ctx , child_event );
14020
13995
if (IS_ERR (pmu_ctx )) {
14021
13996
free_event (child_event );
@@ -14037,8 +14012,6 @@ inherit_event(struct perf_event *parent_event,
14037
14012
return NULL ;
14038
14013
}
14039
14014
14040
- get_ctx (child_ctx );
14041
-
14042
14015
/*
14043
14016
* Make the child state follow the state of the parent event,
14044
14017
* not its attr.disabled bit. We hold the parent's mutex,
@@ -14059,7 +14032,6 @@ inherit_event(struct perf_event *parent_event,
14059
14032
local64_set (& hwc -> period_left , sample_period );
14060
14033
}
14061
14034
14062
- child_event -> ctx = child_ctx ;
14063
14035
child_event -> overflow_handler = parent_event -> overflow_handler ;
14064
14036
child_event -> overflow_handler_context
14065
14037
= parent_event -> overflow_handler_context ;
0 commit comments