Skip to content

Commit fdf8e6d

Browse files
author
Paolo Abeni
committed
Daniel Borkmann says: ==================== pull-request: bpf 2024-01-25 The following pull-request contains BPF updates for your *net* tree. We've added 12 non-merge commits during the last 2 day(s) which contain a total of 13 files changed, 190 insertions(+), 91 deletions(-). The main changes are: 1) Fix bpf_xdp_adjust_tail() in context of XSK zero-copy drivers which support XDP multi-buffer. The former triggered a NULL pointer dereference upon shrinking, from Maciej Fijalkowski & Tirthendu Sarkar. 2) Fix a bug in riscv64 BPF JIT which emitted a wrong prologue and epilogue for struct_ops programs, from Pu Lehui. * tag 'for-netdev' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: i40e: update xdp_rxq_info::frag_size for ZC enabled Rx queue i40e: set xdp_rxq_info::frag_size xdp: reflect tail increase for MEM_TYPE_XSK_BUFF_POOL ice: update xdp_rxq_info::frag_size for ZC enabled Rx queue intel: xsk: initialize skb_frag_t::bv_offset in ZC drivers ice: remove redundant xdp_rxq_info registration i40e: handle multi-buffer packets that are shrunk by xdp prog ice: work on pre-XDP prog frag count xsk: fix usage of multi-buffer BPF helpers for ZC XDP xsk: make xsk_buff_pool responsible for clearing xdp_buff::flags xsk: recycle buffer in case Rx queue was full riscv, bpf: Fix unpredictable kernel crash about RV64 struct_ops ==================== Link: https://lore.kernel.org/r/20240125084416.10876-1-daniel@iogearbox.net Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2 parents 5e34480 + 9d71bc8 commit fdf8e6d

File tree

13 files changed

+190
-91
lines changed

13 files changed

+190
-91
lines changed

‎arch/riscv/net/bpf_jit_comp64.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -795,6 +795,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
795795
struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
796796
struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
797797
struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
798+
bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
798799
void *orig_call = func_addr;
799800
bool save_ret;
800801
u32 insn;
@@ -878,7 +879,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
878879

879880
stack_size = round_up(stack_size, 16);
880881

881-
if (func_addr) {
882+
if (!is_struct_ops) {
882883
/* For the trampoline called from function entry,
883884
* the frame of traced function and the frame of
884885
* trampoline need to be considered.
@@ -998,7 +999,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
998999

9991000
emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx);
10001001

1001-
if (func_addr) {
1002+
if (!is_struct_ops) {
10021003
/* trampoline called from function entry */
10031004
emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx);
10041005
emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx);

‎drivers/net/ethernet/intel/i40e/i40e_main.c

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3588,40 +3588,55 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
35883588
struct i40e_hmc_obj_rxq rx_ctx;
35893589
int err = 0;
35903590
bool ok;
3591-
int ret;
35923591

35933592
bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
35943593

35953594
/* clear the context structure first */
35963595
memset(&rx_ctx, 0, sizeof(rx_ctx));
35973596

3598-
if (ring->vsi->type == I40E_VSI_MAIN)
3599-
xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
3597+
ring->rx_buf_len = vsi->rx_buf_len;
3598+
3599+
/* XDP RX-queue info only needed for RX rings exposed to XDP */
3600+
if (ring->vsi->type != I40E_VSI_MAIN)
3601+
goto skip;
3602+
3603+
if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) {
3604+
err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
3605+
ring->queue_index,
3606+
ring->q_vector->napi.napi_id,
3607+
ring->rx_buf_len);
3608+
if (err)
3609+
return err;
3610+
}
36003611

36013612
ring->xsk_pool = i40e_xsk_pool(ring);
36023613
if (ring->xsk_pool) {
3603-
ring->rx_buf_len =
3604-
xsk_pool_get_rx_frame_size(ring->xsk_pool);
3605-
ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3614+
xdp_rxq_info_unreg(&ring->xdp_rxq);
3615+
ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool);
3616+
err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
3617+
ring->queue_index,
3618+
ring->q_vector->napi.napi_id,
3619+
ring->rx_buf_len);
3620+
if (err)
3621+
return err;
3622+
err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
36063623
MEM_TYPE_XSK_BUFF_POOL,
36073624
NULL);
3608-
if (ret)
3609-
return ret;
3625+
if (err)
3626+
return err;
36103627
dev_info(&vsi->back->pdev->dev,
36113628
"Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
36123629
ring->queue_index);
36133630

36143631
} else {
3615-
ring->rx_buf_len = vsi->rx_buf_len;
3616-
if (ring->vsi->type == I40E_VSI_MAIN) {
3617-
ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3618-
MEM_TYPE_PAGE_SHARED,
3619-
NULL);
3620-
if (ret)
3621-
return ret;
3622-
}
3632+
err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3633+
MEM_TYPE_PAGE_SHARED,
3634+
NULL);
3635+
if (err)
3636+
return err;
36233637
}
36243638

3639+
skip:
36253640
xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq);
36263641

36273642
rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,

‎drivers/net/ethernet/intel/i40e/i40e_txrx.c

Lines changed: 23 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1548,7 +1548,6 @@ void i40e_free_rx_resources(struct i40e_ring *rx_ring)
15481548
int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
15491549
{
15501550
struct device *dev = rx_ring->dev;
1551-
int err;
15521551

15531552
u64_stats_init(&rx_ring->syncp);
15541553

@@ -1569,14 +1568,6 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
15691568
rx_ring->next_to_process = 0;
15701569
rx_ring->next_to_use = 0;
15711570

1572-
/* XDP RX-queue info only needed for RX rings exposed to XDP */
1573-
if (rx_ring->vsi->type == I40E_VSI_MAIN) {
1574-
err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
1575-
rx_ring->queue_index, rx_ring->q_vector->napi.napi_id);
1576-
if (err < 0)
1577-
return err;
1578-
}
1579-
15801571
rx_ring->xdp_prog = rx_ring->vsi->xdp_prog;
15811572

15821573
rx_ring->rx_bi =
@@ -2087,7 +2078,8 @@ static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
20872078
static void i40e_process_rx_buffs(struct i40e_ring *rx_ring, int xdp_res,
20882079
struct xdp_buff *xdp)
20892080
{
2090-
u32 next = rx_ring->next_to_clean;
2081+
u32 nr_frags = xdp_get_shared_info_from_buff(xdp)->nr_frags;
2082+
u32 next = rx_ring->next_to_clean, i = 0;
20912083
struct i40e_rx_buffer *rx_buffer;
20922084

20932085
xdp->flags = 0;
@@ -2100,10 +2092,10 @@ static void i40e_process_rx_buffs(struct i40e_ring *rx_ring, int xdp_res,
21002092
if (!rx_buffer->page)
21012093
continue;
21022094

2103-
if (xdp_res == I40E_XDP_CONSUMED)
2104-
rx_buffer->pagecnt_bias++;
2105-
else
2095+
if (xdp_res != I40E_XDP_CONSUMED)
21062096
i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);
2097+
else if (i++ <= nr_frags)
2098+
rx_buffer->pagecnt_bias++;
21072099

21082100
/* EOP buffer will be put in i40e_clean_rx_irq() */
21092101
if (next == rx_ring->next_to_process)
@@ -2117,20 +2109,20 @@ static void i40e_process_rx_buffs(struct i40e_ring *rx_ring, int xdp_res,
21172109
* i40e_construct_skb - Allocate skb and populate it
21182110
* @rx_ring: rx descriptor ring to transact packets on
21192111
* @xdp: xdp_buff pointing to the data
2120-
* @nr_frags: number of buffers for the packet
21212112
*
21222113
* This function allocates an skb. It then populates it with the page
21232114
* data from the current receive descriptor, taking care to set up the
21242115
* skb correctly.
21252116
*/
21262117
static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
2127-
struct xdp_buff *xdp,
2128-
u32 nr_frags)
2118+
struct xdp_buff *xdp)
21292119
{
21302120
unsigned int size = xdp->data_end - xdp->data;
21312121
struct i40e_rx_buffer *rx_buffer;
2122+
struct skb_shared_info *sinfo;
21322123
unsigned int headlen;
21332124
struct sk_buff *skb;
2125+
u32 nr_frags = 0;
21342126

21352127
/* prefetch first cache line of first page */
21362128
net_prefetch(xdp->data);
@@ -2168,6 +2160,10 @@ static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
21682160
memcpy(__skb_put(skb, headlen), xdp->data,
21692161
ALIGN(headlen, sizeof(long)));
21702162

2163+
if (unlikely(xdp_buff_has_frags(xdp))) {
2164+
sinfo = xdp_get_shared_info_from_buff(xdp);
2165+
nr_frags = sinfo->nr_frags;
2166+
}
21712167
rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
21722168
/* update all of the pointers */
21732169
size -= headlen;
@@ -2187,9 +2183,8 @@ static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
21872183
}
21882184

21892185
if (unlikely(xdp_buff_has_frags(xdp))) {
2190-
struct skb_shared_info *sinfo, *skinfo = skb_shinfo(skb);
2186+
struct skb_shared_info *skinfo = skb_shinfo(skb);
21912187

2192-
sinfo = xdp_get_shared_info_from_buff(xdp);
21932188
memcpy(&skinfo->frags[skinfo->nr_frags], &sinfo->frags[0],
21942189
sizeof(skb_frag_t) * nr_frags);
21952190

@@ -2212,17 +2207,17 @@ static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
22122207
* i40e_build_skb - Build skb around an existing buffer
22132208
* @rx_ring: Rx descriptor ring to transact packets on
22142209
* @xdp: xdp_buff pointing to the data
2215-
* @nr_frags: number of buffers for the packet
22162210
*
22172211
* This function builds an skb around an existing Rx buffer, taking care
22182212
* to set up the skb correctly and avoid any memcpy overhead.
22192213
*/
22202214
static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
2221-
struct xdp_buff *xdp,
2222-
u32 nr_frags)
2215+
struct xdp_buff *xdp)
22232216
{
22242217
unsigned int metasize = xdp->data - xdp->data_meta;
2218+
struct skb_shared_info *sinfo;
22252219
struct sk_buff *skb;
2220+
u32 nr_frags;
22262221

22272222
/* Prefetch first cache line of first page. If xdp->data_meta
22282223
* is unused, this points exactly as xdp->data, otherwise we
@@ -2231,6 +2226,11 @@ static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
22312226
*/
22322227
net_prefetch(xdp->data_meta);
22332228

2229+
if (unlikely(xdp_buff_has_frags(xdp))) {
2230+
sinfo = xdp_get_shared_info_from_buff(xdp);
2231+
nr_frags = sinfo->nr_frags;
2232+
}
2233+
22342234
/* build an skb around the page buffer */
22352235
skb = napi_build_skb(xdp->data_hard_start, xdp->frame_sz);
22362236
if (unlikely(!skb))
@@ -2243,9 +2243,6 @@ static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
22432243
skb_metadata_set(skb, metasize);
22442244

22452245
if (unlikely(xdp_buff_has_frags(xdp))) {
2246-
struct skb_shared_info *sinfo;
2247-
2248-
sinfo = xdp_get_shared_info_from_buff(xdp);
22492246
xdp_update_skb_shared_info(skb, nr_frags,
22502247
sinfo->xdp_frags_size,
22512248
nr_frags * xdp->frame_sz,
@@ -2589,9 +2586,9 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget,
25892586
total_rx_bytes += size;
25902587
} else {
25912588
if (ring_uses_build_skb(rx_ring))
2592-
skb = i40e_build_skb(rx_ring, xdp, nfrags);
2589+
skb = i40e_build_skb(rx_ring, xdp);
25932590
else
2594-
skb = i40e_construct_skb(rx_ring, xdp, nfrags);
2591+
skb = i40e_construct_skb(rx_ring, xdp);
25952592

25962593
/* drop if we failed to retrieve a buffer */
25972594
if (!skb) {

‎drivers/net/ethernet/intel/i40e/i40e_xsk.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,8 @@ i40e_add_xsk_frag(struct i40e_ring *rx_ring, struct xdp_buff *first,
414414
}
415415

416416
__skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++,
417-
virt_to_page(xdp->data_hard_start), 0, size);
417+
virt_to_page(xdp->data_hard_start),
418+
XDP_PACKET_HEADROOM, size);
418419
sinfo->xdp_frags_size += size;
419420
xsk_buff_add_frag(xdp);
420421

@@ -498,7 +499,6 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
498499
xdp_res = i40e_run_xdp_zc(rx_ring, first, xdp_prog);
499500
i40e_handle_xdp_result_zc(rx_ring, first, rx_desc, &rx_packets,
500501
&rx_bytes, xdp_res, &failure);
501-
first->flags = 0;
502502
next_to_clean = next_to_process;
503503
if (failure)
504504
break;

‎drivers/net/ethernet/intel/ice/ice_base.c

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -547,19 +547,27 @@ int ice_vsi_cfg_rxq(struct ice_rx_ring *ring)
547547
ring->rx_buf_len = ring->vsi->rx_buf_len;
548548

549549
if (ring->vsi->type == ICE_VSI_PF) {
550-
if (!xdp_rxq_info_is_reg(&ring->xdp_rxq))
551-
/* coverity[check_return] */
552-
__xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
553-
ring->q_index,
554-
ring->q_vector->napi.napi_id,
555-
ring->vsi->rx_buf_len);
550+
if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) {
551+
err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
552+
ring->q_index,
553+
ring->q_vector->napi.napi_id,
554+
ring->rx_buf_len);
555+
if (err)
556+
return err;
557+
}
556558

557559
ring->xsk_pool = ice_xsk_pool(ring);
558560
if (ring->xsk_pool) {
559-
xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
561+
xdp_rxq_info_unreg(&ring->xdp_rxq);
560562

561563
ring->rx_buf_len =
562564
xsk_pool_get_rx_frame_size(ring->xsk_pool);
565+
err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
566+
ring->q_index,
567+
ring->q_vector->napi.napi_id,
568+
ring->rx_buf_len);
569+
if (err)
570+
return err;
563571
err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
564572
MEM_TYPE_XSK_BUFF_POOL,
565573
NULL);
@@ -571,13 +579,14 @@ int ice_vsi_cfg_rxq(struct ice_rx_ring *ring)
571579
dev_info(dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
572580
ring->q_index);
573581
} else {
574-
if (!xdp_rxq_info_is_reg(&ring->xdp_rxq))
575-
/* coverity[check_return] */
576-
__xdp_rxq_info_reg(&ring->xdp_rxq,
577-
ring->netdev,
578-
ring->q_index,
579-
ring->q_vector->napi.napi_id,
580-
ring->vsi->rx_buf_len);
582+
if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) {
583+
err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
584+
ring->q_index,
585+
ring->q_vector->napi.napi_id,
586+
ring->rx_buf_len);
587+
if (err)
588+
return err;
589+
}
581590

582591
err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
583592
MEM_TYPE_PAGE_SHARED,

‎drivers/net/ethernet/intel/ice/ice_txrx.c

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -513,11 +513,6 @@ int ice_setup_rx_ring(struct ice_rx_ring *rx_ring)
513513
if (ice_is_xdp_ena_vsi(rx_ring->vsi))
514514
WRITE_ONCE(rx_ring->xdp_prog, rx_ring->vsi->xdp_prog);
515515

516-
if (rx_ring->vsi->type == ICE_VSI_PF &&
517-
!xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
518-
if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
519-
rx_ring->q_index, rx_ring->q_vector->napi.napi_id))
520-
goto err;
521516
return 0;
522517

523518
err:
@@ -603,9 +598,7 @@ ice_run_xdp(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,
603598
ret = ICE_XDP_CONSUMED;
604599
}
605600
exit:
606-
rx_buf->act = ret;
607-
if (unlikely(xdp_buff_has_frags(xdp)))
608-
ice_set_rx_bufs_act(xdp, rx_ring, ret);
601+
ice_set_rx_bufs_act(xdp, rx_ring, ret);
609602
}
610603

611604
/**
@@ -893,14 +886,17 @@ ice_add_xdp_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,
893886
}
894887

895888
if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) {
896-
if (unlikely(xdp_buff_has_frags(xdp)))
897-
ice_set_rx_bufs_act(xdp, rx_ring, ICE_XDP_CONSUMED);
889+
ice_set_rx_bufs_act(xdp, rx_ring, ICE_XDP_CONSUMED);
898890
return -ENOMEM;
899891
}
900892

901893
__skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, rx_buf->page,
902894
rx_buf->page_offset, size);
903895
sinfo->xdp_frags_size += size;
896+
/* remember frag count before XDP prog execution; bpf_xdp_adjust_tail()
897+
* can pop off frags but driver has to handle it on its own
898+
*/
899+
rx_ring->nr_frags = sinfo->nr_frags;
904900

905901
if (page_is_pfmemalloc(rx_buf->page))
906902
xdp_buff_set_frag_pfmemalloc(xdp);
@@ -1251,6 +1247,7 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget)
12511247

12521248
xdp->data = NULL;
12531249
rx_ring->first_desc = ntc;
1250+
rx_ring->nr_frags = 0;
12541251
continue;
12551252
construct_skb:
12561253
if (likely(ice_ring_uses_build_skb(rx_ring)))
@@ -1266,10 +1263,12 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget)
12661263
ICE_XDP_CONSUMED);
12671264
xdp->data = NULL;
12681265
rx_ring->first_desc = ntc;
1266+
rx_ring->nr_frags = 0;
12691267
break;
12701268
}
12711269
xdp->data = NULL;
12721270
rx_ring->first_desc = ntc;
1271+
rx_ring->nr_frags = 0;
12731272

12741273
stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S);
12751274
if (unlikely(ice_test_staterr(rx_desc->wb.status_error0,

‎drivers/net/ethernet/intel/ice/ice_txrx.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,7 @@ struct ice_rx_ring {
358358
struct ice_tx_ring *xdp_ring;
359359
struct ice_rx_ring *next; /* pointer to next ring in q_vector */
360360
struct xsk_buff_pool *xsk_pool;
361+
u32 nr_frags;
361362
dma_addr_t dma; /* physical address of ring */
362363
u16 rx_buf_len;
363364
u8 dcb_tc; /* Traffic class of ring */

0 commit comments

Comments
 (0)