Commit cf8b8763 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf-track-changes_pkt_data-property-for-global-functions' 

Eduard Zingerman says:

====================
bpf: track changes_pkt_data property for global functions

Nick Zavaritsky reported [0] a bug in verifier, where the following
unsafe program is not rejected:

    __attribute__((__noinline__))
    long skb_pull_data(struct __sk_buff *sk, __u32 len)
    {
        return bpf_skb_pull_data(sk, len);
    }

    SEC("tc")
    int test_invalidate_checks(struct __sk_buff *sk)
    {
        int *p = (void *)(long)sk->data;
        if ((void *)(p + 1) > (void *)(long)sk->data_end) return TCX_DROP;
        skb_pull_data(sk, 0);
        /* not safe, p is invalid after bpf_skb_pull_data call */
        *p = 42;
        return TCX_PASS;
    }

This happens because verifier does not track package invalidation
effect of global sub-programs.

This patch-set fixes the issue by modifying check_cfg() to compute
whether or not each sub-program calls (directly or indirectly)
helper invalidating packet pointers.

As global functions could be replaced with extension programs,
a new field 'changes_pkt_data' is added to struct bpf_prog_aux.
Verifier only allows replacing functions that do not change packet
data with functions that do not change packet data.

In case if there is a need to a have a global function that does not
change packet data, but allow replacing it with function that does,
the recommendation is to add a noop call to a helper, e.g.:
- for skb do 'bpf_skb_change_proto(skb, 0, 0)';
- for xdp do 'bpf_xdp_adjust_meta(xdp, 0)'.

Functions also can do tail calls. Effects of the tail call cannot be
analyzed before-hand, thus verifier assumes that tail calls always
change packet data.

Changes v1 [1] -> v2:
- added handling of extension programs and tail calls
  (thanks, Alexei, for all the input).

[0] https://lore.kernel.org/bpf/0498CA22-5779-4767-9C0C-A9515CEA711F@gmail.com/
[1] https://lore.kernel.org/bpf/20241206040307.568065-1-eddyz87@gmail.com/
====================

Link: https://patch.msgid.link/20241210041100.1898468-1-eddyz87@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 978c4486 d9706b56

include/linux/bpf.h

+1 −0
Original line number Diff line number Diff line
@@ -1527,6 +1527,7 @@ struct bpf_prog_aux { 
	bool is_extended; /* true if extended by freplace program */

	bool jits_use_priv_stack;

	bool priv_stack_requested;

	bool changes_pkt_data;

	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */

	struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */

	struct bpf_arena *arena;

include/linux/bpf_verifier.h

+1 −0
Original line number Diff line number Diff line
@@ -659,6 +659,7 @@ struct bpf_subprog_info { 
	bool args_cached: 1;

	/* true if bpf_fastcall stack region is used by functions that can't be inlined */

	bool keep_fastcall_stack: 1;

	bool changes_pkt_data: 1;



	enum priv_stack_mode priv_stack_mode;

	u8 arg_cnt;

include/linux/filter.h

+1 −1
Original line number Diff line number Diff line
@@ -1122,7 +1122,7 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); 
bool bpf_jit_supports_private_stack(void);

u64 bpf_arch_uaddress_limit(void);

void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);

bool bpf_helper_changes_pkt_data(void *func);

bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id);



static inline bool bpf_dump_raw_ok(const struct cred *cred)

{

kernel/bpf/core.c

+1 −1
Original line number Diff line number Diff line
@@ -2936,7 +2936,7 @@ void __weak bpf_jit_compile(struct bpf_prog *prog) 
{

}



bool __weak bpf_helper_changes_pkt_data(void *func)

bool __weak bpf_helper_changes_pkt_data(enum bpf_func_id func_id)

{

	return false;

}

kernel/bpf/verifier.c

+68 −10
Original line number Diff line number Diff line
@@ -2597,16 +2597,36 @@ static int cmp_subprogs(const void *a, const void *b) 
	       ((struct bpf_subprog_info *)b)->start;

}









/* Find subprogram that contains instruction at 'off' */









static struct bpf_subprog_info *find_containing_subprog(struct bpf_verifier_env *env, int off)









{









	struct bpf_subprog_info *vals = env->subprog_info;









	int l, r, m;

















	if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0)









		return NULL;

















	l = 0;









	r = env->subprog_cnt - 1;









	while (l < r) {









		m = l + (r - l + 1) / 2;









		if (vals[m].start <= off)









			l = m;









		else









			r = m - 1;









	}









	return &vals[l];









}

















/* Find subprogram that starts exactly at 'off' */









static int find_subprog(struct bpf_verifier_env *env, int off)









{









	struct bpf_subprog_info *p;

















	p = bsearch(&off, env->subprog_info, env->subprog_cnt,









		    sizeof(env->subprog_info[0]), cmp_subprogs);









	if (!p)









	p = find_containing_subprog(env, off);









	if (!p || p->start != off)









		return -ENOENT;









	return p - env->subprog_info;

















}

















static int add_subprog(struct bpf_verifier_env *env, int off)








@@ -10022,6 +10042,8 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,















		verbose(env, "Func#%d ('%s') is global and assumed valid.\n",









			subprog, sub_name);









		if (env->subprog_info[subprog].changes_pkt_data)









			clear_all_pkt_pointers(env);









		/* mark global subprog for verifying after main prog */









		subprog_aux(env, subprog)->called = true;









		clear_caller_saved_regs(env, caller->regs);








@@ -10708,7 +10730,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn







	}

















	/* With LD_ABS/IND some JITs save/restore skb from r1. */









	changes_data = bpf_helper_changes_pkt_data(fn->func);









	changes_data = bpf_helper_changes_pkt_data(func_id);









	if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {









		verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",









			func_id_name(func_id), func_id);








@@ -16226,6 +16248,29 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char







	return 0;









}

















static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off)









{









	struct bpf_subprog_info *subprog;

















	subprog = find_containing_subprog(env, off);









	subprog->changes_pkt_data = true;









}

















/* 't' is an index of a call-site.









 * 'w' is a callee entry point.









 * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.









 * Rely on DFS traversal order and absence of recursive calls to guarantee that









 * callee's change_pkt_data marks would be correct at that moment.









 */









static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)









{









	struct bpf_subprog_info *caller, *callee;

















	caller = find_containing_subprog(env, t);









	callee = find_containing_subprog(env, w);









	caller->changes_pkt_data |= callee->changes_pkt_data;









}

















/* non-recursive DFS pseudo code









 * 1  procedure DFS-iterative(G,v):









 * 2      label v as discovered








@@ -16359,6 +16404,7 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns,







				bool visit_callee)









{









	int ret, insn_sz;









	int w;

















	insn_sz = bpf_is_ldimm64(&insns[t]) ? 2 : 1;









	ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);








@@ -16370,8 +16416,10 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns,







	mark_jmp_point(env, t + insn_sz);

















	if (visit_callee) {









		w = t + insns[t].imm + 1;









		mark_prune_point(env, t);









		ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);









		merge_callee_effects(env, t, w);









		ret = push_insn(t, w, BRANCH, env);









	}









	return ret;









}








@@ -16688,6 +16736,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env)







			mark_prune_point(env, t);









			mark_jmp_point(env, t);









		}









		if (bpf_helper_call(insn) && bpf_helper_changes_pkt_data(insn->imm))









			mark_subprog_changes_pkt_data(env, t);









		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {









			struct bpf_kfunc_call_arg_meta meta;
















@@ -16822,6 +16872,7 @@ static int check_cfg(struct bpf_verifier_env *env)







		}









	}









	ret = 0; /* cfg looks good */









	env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;

















err_free:









	kvfree(insn_state);








@@ -20311,6 +20362,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)







		func[i]->aux->num_exentries = num_exentries;









		func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;









		func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;









		func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;









		if (!i)









			func[i]->aux->exception_boundary = env->seen_exception;









		func[i] = bpf_int_jit_compile(func[i]);








@@ -22175,6 +22227,12 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,







					"Extension programs should be JITed\n");









				return -EINVAL;









			}









			if (prog->aux->changes_pkt_data &&









			    !aux->func[subprog]->aux->changes_pkt_data) {









				bpf_log(log,









					"Extension program changes packet data, while original does not\n");









				return -EINVAL;









			}









		}









		if (!tgt_prog->jited) {









			bpf_log(log, "Can attach to only JITed progs\n");








@@ -22640,10 +22698,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3







	if (ret < 0)









		goto skip_full_check;

















	ret = check_attach_btf_id(env);









	if (ret)









		goto skip_full_check;

















	ret = resolve_pseudo_ldimm64(env);









	if (ret < 0)









		goto skip_full_check;








@@ -22658,6 +22712,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3







	if (ret < 0)









		goto skip_full_check;

















	ret = check_attach_btf_id(env);









	if (ret)









		goto skip_full_check;

















	ret = mark_fastcall_patterns(env);









	if (ret < 0)









		goto skip_full_check;