Skip to content

Commit d2102f2

Browse files
danobiAlexei Starovoitov
authored andcommitted
bpf: verifier: Support eliding map lookup nullness
This commit allows progs to elide a null check on statically known map lookup keys. In other words, if the verifier can statically prove that the lookup will be in-bounds, allow the prog to drop the null check. This is useful for two reasons: 1. Large numbers of nullness checks (especially when they cannot fail) unnecessarily pushes prog towards BPF_COMPLEXITY_LIMIT_JMP_SEQ. 2. It forms a tighter contract between programmer and verifier. For (1), bpftrace is starting to make heavier use of percpu scratch maps. As a result, for user scripts with large number of unrolled loops, we are starting to hit jump complexity verification errors. These percpu lookups cannot fail anyways, as we only use static key values. Eliding nullness probably results in less work for verifier as well. For (2), percpu scratch maps are often used as a larger stack, as the currrent stack is limited to 512 bytes. In these situations, it is desirable for the programmer to express: "this lookup should never fail, and if it does, it means I messed up the code". By omitting the null check, the programmer can "ask" the verifier to double check the logic. Tests also have to be updated in sync with these changes, as the verifier is more efficient with this change. Notable, iters.c tests had to be changed to use a map type that still requires null checks, as it's exercising verifier tracking logic w.r.t iterators. Signed-off-by: Daniel Xu <dxu@dxuuu.xyz> Link: https://lore.kernel.org/r/68f3ea96ff3809a87e502a11a4bd30177fc5823e.1736886479.git.dxu@dxuuu.xyz Signed-off-by: Alexei Starovoitov <ast@kernel.org>
1 parent 37cce22 commit d2102f2

File tree

5 files changed

+99
-13
lines changed

5 files changed

+99
-13
lines changed

kernel/bpf/verifier.c

Lines changed: 89 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,7 @@ struct bpf_call_arg_meta {
287287
u32 ret_btf_id;
288288
u32 subprogno;
289289
struct btf_field *kptr_field;
290+
s64 const_map_key;
290291
};
291292

292293
struct bpf_kfunc_call_arg_meta {
@@ -9148,6 +9149,63 @@ static int check_reg_const_str(struct bpf_verifier_env *env,
91489149
return 0;
91499150
}
91509151

9152+
/* Returns constant key value if possible, else negative error */
9153+
static s64 get_constant_map_key(struct bpf_verifier_env *env,
9154+
struct bpf_reg_state *key,
9155+
u32 key_size)
9156+
{
9157+
struct bpf_func_state *state = func(env, key);
9158+
struct bpf_reg_state *reg;
9159+
int slot, spi, off;
9160+
int spill_size = 0;
9161+
int zero_size = 0;
9162+
int stack_off;
9163+
int i, err;
9164+
u8 *stype;
9165+
9166+
if (!env->bpf_capable)
9167+
return -EOPNOTSUPP;
9168+
if (key->type != PTR_TO_STACK)
9169+
return -EOPNOTSUPP;
9170+
if (!tnum_is_const(key->var_off))
9171+
return -EOPNOTSUPP;
9172+
9173+
stack_off = key->off + key->var_off.value;
9174+
slot = -stack_off - 1;
9175+
spi = slot / BPF_REG_SIZE;
9176+
off = slot % BPF_REG_SIZE;
9177+
stype = state->stack[spi].slot_type;
9178+
9179+
/* First handle precisely tracked STACK_ZERO */
9180+
for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--)
9181+
zero_size++;
9182+
if (zero_size >= key_size)
9183+
return 0;
9184+
9185+
/* Check that stack contains a scalar spill of expected size */
9186+
if (!is_spilled_scalar_reg(&state->stack[spi]))
9187+
return -EOPNOTSUPP;
9188+
for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--)
9189+
spill_size++;
9190+
if (spill_size != key_size)
9191+
return -EOPNOTSUPP;
9192+
9193+
reg = &state->stack[spi].spilled_ptr;
9194+
if (!tnum_is_const(reg->var_off))
9195+
/* Stack value not statically known */
9196+
return -EOPNOTSUPP;
9197+
9198+
/* We are relying on a constant value. So mark as precise
9199+
* to prevent pruning on it.
9200+
*/
9201+
bt_set_frame_slot(&env->bt, key->frameno, spi);
9202+
err = mark_chain_precision_batch(env);
9203+
if (err < 0)
9204+
return err;
9205+
9206+
return reg->var_off.value;
9207+
}
9208+
91519209
static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
91529210
struct bpf_call_arg_meta *meta,
91539211
const struct bpf_func_proto *fn,
@@ -9158,6 +9216,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
91589216
enum bpf_arg_type arg_type = fn->arg_type[arg];
91599217
enum bpf_reg_type type = reg->type;
91609218
u32 *arg_btf_id = NULL;
9219+
u32 key_size;
91619220
int err = 0;
91629221

91639222
if (arg_type == ARG_DONTCARE)
@@ -9291,8 +9350,13 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
92919350
verbose(env, "invalid map_ptr to access map->key\n");
92929351
return -EACCES;
92939352
}
9294-
err = check_helper_mem_access(env, regno, meta->map_ptr->key_size,
9295-
BPF_READ, false, NULL);
9353+
key_size = meta->map_ptr->key_size;
9354+
err = check_helper_mem_access(env, regno, key_size, BPF_READ, false, NULL);
9355+
if (err)
9356+
return err;
9357+
meta->const_map_key = get_constant_map_key(env, reg, key_size);
9358+
if (meta->const_map_key < 0 && meta->const_map_key != -EOPNOTSUPP)
9359+
return meta->const_map_key;
92969360
break;
92979361
case ARG_PTR_TO_MAP_VALUE:
92989362
if (type_may_be_null(arg_type) && register_is_null(reg))
@@ -10816,6 +10880,21 @@ static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno
1081610880
state->callback_subprogno == subprogno);
1081710881
}
1081810882

10883+
/* Returns whether or not the given map type can potentially elide
10884+
* lookup return value nullness check. This is possible if the key
10885+
* is statically known.
10886+
*/
10887+
static bool can_elide_value_nullness(enum bpf_map_type type)
10888+
{
10889+
switch (type) {
10890+
case BPF_MAP_TYPE_ARRAY:
10891+
case BPF_MAP_TYPE_PERCPU_ARRAY:
10892+
return true;
10893+
default:
10894+
return false;
10895+
}
10896+
}
10897+
1081910898
static int get_helper_proto(struct bpf_verifier_env *env, int func_id,
1082010899
const struct bpf_func_proto **ptr)
1082110900
{
@@ -11182,10 +11261,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
1118211261
"kernel subsystem misconfigured verifier\n");
1118311262
return -EINVAL;
1118411263
}
11264+
11265+
if (func_id == BPF_FUNC_map_lookup_elem &&
11266+
can_elide_value_nullness(meta.map_ptr->map_type) &&
11267+
meta.const_map_key >= 0 &&
11268+
meta.const_map_key < meta.map_ptr->max_entries)
11269+
ret_flag &= ~PTR_MAYBE_NULL;
11270+
1118511271
regs[BPF_REG_0].map_ptr = meta.map_ptr;
1118611272
regs[BPF_REG_0].map_uid = meta.map_uid;
1118711273
regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag;
11188-
if (!type_may_be_null(ret_type) &&
11274+
if (!type_may_be_null(ret_flag) &&
1118911275
btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK)) {
1119011276
regs[BPF_REG_0].id = ++env->id_gen;
1119111277
}

tools/testing/selftests/bpf/progs/iters.c

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -524,11 +524,11 @@ int iter_subprog_iters(const void *ctx)
524524
}
525525

526526
struct {
527-
__uint(type, BPF_MAP_TYPE_ARRAY);
527+
__uint(type, BPF_MAP_TYPE_HASH);
528528
__type(key, int);
529529
__type(value, int);
530530
__uint(max_entries, 1000);
531-
} arr_map SEC(".maps");
531+
} hash_map SEC(".maps");
532532

533533
SEC("?raw_tp")
534534
__failure __msg("invalid mem access 'scalar'")
@@ -539,7 +539,7 @@ int iter_err_too_permissive1(const void *ctx)
539539

540540
MY_PID_GUARD();
541541

542-
map_val = bpf_map_lookup_elem(&arr_map, &key);
542+
map_val = bpf_map_lookup_elem(&hash_map, &key);
543543
if (!map_val)
544544
return 0;
545545

@@ -561,12 +561,12 @@ int iter_err_too_permissive2(const void *ctx)
561561

562562
MY_PID_GUARD();
563563

564-
map_val = bpf_map_lookup_elem(&arr_map, &key);
564+
map_val = bpf_map_lookup_elem(&hash_map, &key);
565565
if (!map_val)
566566
return 0;
567567

568568
bpf_repeat(1000000) {
569-
map_val = bpf_map_lookup_elem(&arr_map, &key);
569+
map_val = bpf_map_lookup_elem(&hash_map, &key);
570570
}
571571

572572
*map_val = 123;
@@ -585,7 +585,7 @@ int iter_err_too_permissive3(const void *ctx)
585585
MY_PID_GUARD();
586586

587587
bpf_repeat(1000000) {
588-
map_val = bpf_map_lookup_elem(&arr_map, &key);
588+
map_val = bpf_map_lookup_elem(&hash_map, &key);
589589
found = true;
590590
}
591591

@@ -606,7 +606,7 @@ int iter_tricky_but_fine(const void *ctx)
606606
MY_PID_GUARD();
607607

608608
bpf_repeat(1000000) {
609-
map_val = bpf_map_lookup_elem(&arr_map, &key);
609+
map_val = bpf_map_lookup_elem(&hash_map, &key);
610610
if (map_val) {
611611
found = true;
612612
break;

tools/testing/selftests/bpf/progs/map_kptr_fail.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,7 @@ int reject_indirect_global_func_access(struct __sk_buff *ctx)
345345
}
346346

347347
SEC("?tc")
348-
__failure __msg("Unreleased reference id=5 alloc_insn=")
348+
__failure __msg("Unreleased reference id=4 alloc_insn=")
349349
int kptr_xchg_ref_state(struct __sk_buff *ctx)
350350
{
351351
struct prog_test_ref_kfunc *p;

tools/testing/selftests/bpf/progs/verifier_map_in_map.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ l0_%=: r0 = 0; \
4747

4848
SEC("xdp")
4949
__description("map in map state pruning")
50-
__success __msg("processed 26 insns")
50+
__success __msg("processed 15 insns")
5151
__log_level(2) __retval(0) __flag(BPF_F_TEST_STATE_FREQ)
5252
__naked void map_in_map_state_pruning(void)
5353
{

tools/testing/selftests/bpf/verifier/map_kptr.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -373,7 +373,7 @@
373373
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
374374
.fixup_map_kptr = { 1 },
375375
.result = REJECT,
376-
.errstr = "Unreleased reference id=5 alloc_insn=20",
376+
.errstr = "Unreleased reference id=4 alloc_insn=20",
377377
.fixup_kfunc_btf_id = {
378378
{ "bpf_kfunc_call_test_acquire", 15 },
379379
}

0 commit comments

Comments
 (0)