Skip to content

Commit 9e3b47a

Browse files
author
Alexei Starovoitov
committed
Merge branch 'add-support-cpu-v4-insns-for-rv64'
Pu Lehui says: ==================== Add support cpu v4 insns for RV64 Add support cpu v4 instructions for RV64. The relevant tests have passed as show bellow: Summary: 6/166 PASSED, 0 SKIPPED, 0 FAILED NOTE: ldsx_insn testcase uses fentry and needs to rely on ftrace direct call [0]. [0] https://lore.kernel.org/all/20230627111612.761164-1-suagrfillet@gmail.com/ v2: - Use temporary reg to avoid clobbering the source reg in movs_8/16 insns. (Björn) - Add Acked-by v1: https://lore.kernel.org/bpf/20230823231059.3363698-1-pulehui@huaweicloud.com ==================== Tested-by: Björn Töpel <bjorn@rivosinc.com> Link: https://lore.kernel.org/r/20230824095001.3408573-1-pulehui@huaweicloud.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2 parents 1b580c9 + 0209fd5 commit 9e3b47a

File tree

8 files changed

+122
-28
lines changed

8 files changed

+122
-28
lines changed

‎arch/riscv/net/bpf_jit.h

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,11 +431,21 @@ static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
431431
return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
432432
}
433433

434+
static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2)
435+
{
436+
return rv_r_insn(1, rs2, rs1, 4, rd, 0x33);
437+
}
438+
434439
static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
435440
{
436441
return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
437442
}
438443

444+
static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2)
445+
{
446+
return rv_r_insn(1, rs2, rs1, 6, rd, 0x33);
447+
}
448+
439449
static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
440450
{
441451
return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
@@ -501,6 +511,16 @@ static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
501511
return rv_bge(rs2, rs1, imm12_1);
502512
}
503513

514+
static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1)
515+
{
516+
return rv_i_insn(imm11_0, rs1, 0, rd, 0x03);
517+
}
518+
519+
static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1)
520+
{
521+
return rv_i_insn(imm11_0, rs1, 1, rd, 0x03);
522+
}
523+
504524
static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
505525
{
506526
return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
@@ -766,11 +786,21 @@ static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
766786
return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
767787
}
768788

789+
static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2)
790+
{
791+
return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b);
792+
}
793+
769794
static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
770795
{
771796
return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
772797
}
773798

799+
static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2)
800+
{
801+
return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b);
802+
}
803+
774804
static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
775805
{
776806
return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);

‎arch/riscv/net/bpf_jit_comp64.c

Lines changed: 80 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -580,7 +580,8 @@ static int add_exception_handler(const struct bpf_insn *insn,
580580
unsigned long pc;
581581
off_t offset;
582582

583-
if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM)
583+
if (!ctx->insns || !ctx->prog->aux->extable ||
584+
(BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX))
584585
return 0;
585586

586587
if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries))
@@ -1046,7 +1047,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
10461047
emit_zext_32(rd, ctx);
10471048
break;
10481049
}
1049-
emit_mv(rd, rs, ctx);
1050+
switch (insn->off) {
1051+
case 0:
1052+
emit_mv(rd, rs, ctx);
1053+
break;
1054+
case 8:
1055+
case 16:
1056+
emit_slli(RV_REG_T1, rs, 64 - insn->off, ctx);
1057+
emit_srai(rd, RV_REG_T1, 64 - insn->off, ctx);
1058+
break;
1059+
case 32:
1060+
emit_addiw(rd, rs, 0, ctx);
1061+
break;
1062+
}
10501063
if (!is64 && !aux->verifier_zext)
10511064
emit_zext_32(rd, ctx);
10521065
break;
@@ -1094,13 +1107,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
10941107
break;
10951108
case BPF_ALU | BPF_DIV | BPF_X:
10961109
case BPF_ALU64 | BPF_DIV | BPF_X:
1097-
emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
1110+
if (off)
1111+
emit(is64 ? rv_div(rd, rd, rs) : rv_divw(rd, rd, rs), ctx);
1112+
else
1113+
emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
10981114
if (!is64 && !aux->verifier_zext)
10991115
emit_zext_32(rd, ctx);
11001116
break;
11011117
case BPF_ALU | BPF_MOD | BPF_X:
11021118
case BPF_ALU64 | BPF_MOD | BPF_X:
1103-
emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
1119+
if (off)
1120+
emit(is64 ? rv_rem(rd, rd, rs) : rv_remw(rd, rd, rs), ctx);
1121+
else
1122+
emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
11041123
if (!is64 && !aux->verifier_zext)
11051124
emit_zext_32(rd, ctx);
11061125
break;
@@ -1149,6 +1168,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
11491168
break;
11501169

11511170
case BPF_ALU | BPF_END | BPF_FROM_BE:
1171+
case BPF_ALU64 | BPF_END | BPF_FROM_LE:
11521172
emit_li(RV_REG_T2, 0, ctx);
11531173

11541174
emit_andi(RV_REG_T1, rd, 0xff, ctx);
@@ -1271,16 +1291,24 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
12711291
case BPF_ALU | BPF_DIV | BPF_K:
12721292
case BPF_ALU64 | BPF_DIV | BPF_K:
12731293
emit_imm(RV_REG_T1, imm, ctx);
1274-
emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
1275-
rv_divuw(rd, rd, RV_REG_T1), ctx);
1294+
if (off)
1295+
emit(is64 ? rv_div(rd, rd, RV_REG_T1) :
1296+
rv_divw(rd, rd, RV_REG_T1), ctx);
1297+
else
1298+
emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
1299+
rv_divuw(rd, rd, RV_REG_T1), ctx);
12761300
if (!is64 && !aux->verifier_zext)
12771301
emit_zext_32(rd, ctx);
12781302
break;
12791303
case BPF_ALU | BPF_MOD | BPF_K:
12801304
case BPF_ALU64 | BPF_MOD | BPF_K:
12811305
emit_imm(RV_REG_T1, imm, ctx);
1282-
emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
1283-
rv_remuw(rd, rd, RV_REG_T1), ctx);
1306+
if (off)
1307+
emit(is64 ? rv_rem(rd, rd, RV_REG_T1) :
1308+
rv_remw(rd, rd, RV_REG_T1), ctx);
1309+
else
1310+
emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
1311+
rv_remuw(rd, rd, RV_REG_T1), ctx);
12841312
if (!is64 && !aux->verifier_zext)
12851313
emit_zext_32(rd, ctx);
12861314
break;
@@ -1314,7 +1342,11 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
13141342

13151343
/* JUMP off */
13161344
case BPF_JMP | BPF_JA:
1317-
rvoff = rv_offset(i, off, ctx);
1345+
case BPF_JMP32 | BPF_JA:
1346+
if (BPF_CLASS(code) == BPF_JMP)
1347+
rvoff = rv_offset(i, off, ctx);
1348+
else
1349+
rvoff = rv_offset(i, imm, ctx);
13181350
ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);
13191351
if (ret)
13201352
return ret;
@@ -1486,7 +1518,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
14861518
return 1;
14871519
}
14881520

1489-
/* LDX: dst = *(size *)(src + off) */
1521+
/* LDX: dst = *(unsigned size *)(src + off) */
14901522
case BPF_LDX | BPF_MEM | BPF_B:
14911523
case BPF_LDX | BPF_MEM | BPF_H:
14921524
case BPF_LDX | BPF_MEM | BPF_W:
@@ -1495,57 +1527,80 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
14951527
case BPF_LDX | BPF_PROBE_MEM | BPF_H:
14961528
case BPF_LDX | BPF_PROBE_MEM | BPF_W:
14971529
case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1530+
/* LDSX: dst = *(signed size *)(src + off) */
1531+
case BPF_LDX | BPF_MEMSX | BPF_B:
1532+
case BPF_LDX | BPF_MEMSX | BPF_H:
1533+
case BPF_LDX | BPF_MEMSX | BPF_W:
1534+
case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1535+
case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1536+
case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
14981537
{
14991538
int insn_len, insns_start;
1539+
bool sign_ext;
1540+
1541+
sign_ext = BPF_MODE(insn->code) == BPF_MEMSX ||
1542+
BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
15001543

15011544
switch (BPF_SIZE(code)) {
15021545
case BPF_B:
15031546
if (is_12b_int(off)) {
15041547
insns_start = ctx->ninsns;
1505-
emit(rv_lbu(rd, off, rs), ctx);
1548+
if (sign_ext)
1549+
emit(rv_lb(rd, off, rs), ctx);
1550+
else
1551+
emit(rv_lbu(rd, off, rs), ctx);
15061552
insn_len = ctx->ninsns - insns_start;
15071553
break;
15081554
}
15091555

15101556
emit_imm(RV_REG_T1, off, ctx);
15111557
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
15121558
insns_start = ctx->ninsns;
1513-
emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
1559+
if (sign_ext)
1560+
emit(rv_lb(rd, 0, RV_REG_T1), ctx);
1561+
else
1562+
emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
15141563
insn_len = ctx->ninsns - insns_start;
1515-
if (insn_is_zext(&insn[1]))
1516-
return 1;
15171564
break;
15181565
case BPF_H:
15191566
if (is_12b_int(off)) {
15201567
insns_start = ctx->ninsns;
1521-
emit(rv_lhu(rd, off, rs), ctx);
1568+
if (sign_ext)
1569+
emit(rv_lh(rd, off, rs), ctx);
1570+
else
1571+
emit(rv_lhu(rd, off, rs), ctx);
15221572
insn_len = ctx->ninsns - insns_start;
15231573
break;
15241574
}
15251575

15261576
emit_imm(RV_REG_T1, off, ctx);
15271577
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
15281578
insns_start = ctx->ninsns;
1529-
emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
1579+
if (sign_ext)
1580+
emit(rv_lh(rd, 0, RV_REG_T1), ctx);
1581+
else
1582+
emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
15301583
insn_len = ctx->ninsns - insns_start;
1531-
if (insn_is_zext(&insn[1]))
1532-
return 1;
15331584
break;
15341585
case BPF_W:
15351586
if (is_12b_int(off)) {
15361587
insns_start = ctx->ninsns;
1537-
emit(rv_lwu(rd, off, rs), ctx);
1588+
if (sign_ext)
1589+
emit(rv_lw(rd, off, rs), ctx);
1590+
else
1591+
emit(rv_lwu(rd, off, rs), ctx);
15381592
insn_len = ctx->ninsns - insns_start;
15391593
break;
15401594
}
15411595

15421596
emit_imm(RV_REG_T1, off, ctx);
15431597
emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
15441598
insns_start = ctx->ninsns;
1545-
emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
1599+
if (sign_ext)
1600+
emit(rv_lw(rd, 0, RV_REG_T1), ctx);
1601+
else
1602+
emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
15461603
insn_len = ctx->ninsns - insns_start;
1547-
if (insn_is_zext(&insn[1]))
1548-
return 1;
15491604
break;
15501605
case BPF_DW:
15511606
if (is_12b_int(off)) {
@@ -1566,6 +1621,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
15661621
ret = add_exception_handler(insn, ctx, rd, insn_len);
15671622
if (ret)
15681623
return ret;
1624+
1625+
if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1]))
1626+
return 1;
15691627
break;
15701628
}
15711629
/* speculation barrier */

‎tools/testing/selftests/bpf/progs/test_ldsx_insn.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
#include <bpf/bpf_helpers.h>
66
#include <bpf/bpf_tracing.h>
77

8-
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
8+
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
9+
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
910
const volatile int skip = 0;
1011
#else
1112
const volatile int skip = 1;

‎tools/testing/selftests/bpf/progs/verifier_bswap.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
#include <bpf/bpf_helpers.h>
55
#include "bpf_misc.h"
66

7-
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
7+
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
8+
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
89

910
SEC("socket")
1011
__description("BSWAP, 16")

‎tools/testing/selftests/bpf/progs/verifier_gotol.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
#include <bpf/bpf_helpers.h>
55
#include "bpf_misc.h"
66

7-
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
7+
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
8+
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
89

910
SEC("socket")
1011
__description("gotol, small_imm")

‎tools/testing/selftests/bpf/progs/verifier_ldsx.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
#include <bpf/bpf_helpers.h>
55
#include "bpf_misc.h"
66

7-
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
7+
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
8+
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
89

910
SEC("socket")
1011
__description("LDSX, S8")

‎tools/testing/selftests/bpf/progs/verifier_movsx.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
#include <bpf/bpf_helpers.h>
55
#include "bpf_misc.h"
66

7-
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
7+
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
8+
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
89

910
SEC("socket")
1011
__description("MOV32SX, S8")

‎tools/testing/selftests/bpf/progs/verifier_sdiv.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
#include <bpf/bpf_helpers.h>
55
#include "bpf_misc.h"
66

7-
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86)) && __clang_major__ >= 18
7+
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
8+
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && __clang_major__ >= 18
89

910
SEC("socket")
1011
__description("SDIV32, non-zero imm divisor, check 1")

0 commit comments

Comments
 (0)