Commit d352eca2 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'support-bpf_fastcall-patterns-for-calls-to-kfuncs' 

Eduard Zingerman says:

====================
support bpf_fastcall patterns for calls to kfuncs

As an extension of [1], allow bpf_fastcall patterns for kfuncs:
- pattern rules are the same as for helpers;
- spill/fill removal is allowed only for kfuncs listed in the
  is_fastcall_kfunc_call (under assumption that such kfuncs would
  always be members of special_kfunc_list).

Allow bpf_fastcall rewrite for bpf_cast_to_kern_ctx() and
bpf_rdonly_cast() in order to conjure selftests for this feature.

After this patch-set verifier would rewrite the program below:

  r2 = 1
  *(u64 *)(r10 - 32) = r2
  call %[bpf_cast_to_kern_ctx]
  r2 = *(u64 *)(r10 - 32)
  r0 = r2;"

As follows:

  r2 = 1   /* spill/fill at r10[-32] is removed */
  r0 = r1  /* replacement for bpf_cast_to_kern_ctx() */
  r0 = r2
  exit

Also, attribute used by LLVM implementation of the feature had been
changed from no_caller_saved_registers to bpf_fastcall (see [2]).
This patch-set replaces references to nocsr by references to
bpf_fastcall to keep LLVM and Kernel parts in sync.

[1] no_caller_saved_registers attribute for helper calls
    https://lore.kernel.org/bpf/20240722233844.1406874-1-eddyz87@gmail.com/
[2] [BPF] introduce __attribute__((bpf_fastcall))
    https://github.com/llvm/llvm-project/pull/105417

Changes v2->v3:
- added a patch fixing arch_mask handling in test_loader,
  otherwise newly added tests for the feature were skipped
  (a fix for regression introduced by a recent commit);
- fixed warning regarding unused 'params' variable;
- applied stylistical fixes suggested by Yonghong;
- added acks from Yonghong;

Changes v1->v2:
- added two patches replacing all mentions of nocsr by bpf_fastcall
  (suggested by Andrii);
- removed KF_NOCSR flag (suggested by Yonghong).

v1: https://lore.kernel.org/bpf/20240812234356.2089263-1-eddyz87@gmail.com/
v2: https://lore.kernel.org/bpf/20240817015140.1039351-1-eddyz87@gmail.com/
====================

Link: https://lore.kernel.org/r/20240822084112.3257995-1-eddyz87@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 6d641ca5 8c2e043d

include/linux/bpf.h

+3 −3
Original line number Diff line number Diff line
@@ -808,12 +808,12 @@ struct bpf_func_proto { 
	bool gpl_only;

	bool pkt_access;

	bool might_sleep;

	/* set to true if helper follows contract for gcc/llvm

	 * attribute no_caller_saved_registers:

	/* set to true if helper follows contract for llvm

	 * attribute bpf_fastcall:

	 * - void functions do not scratch r0

	 * - functions taking N arguments scratch only registers r1-rN

	 */

	bool allow_nocsr;

	bool allow_fastcall;

	enum bpf_return_type ret_type;

	union {

		struct {

include/linux/bpf_verifier.h

+9 −9
Original line number Diff line number Diff line
@@ -577,13 +577,13 @@ struct bpf_insn_aux_data { 
	bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */

	u8 alu_state; /* used in combination with alu_limit */

	/* true if STX or LDX instruction is a part of a spill/fill

	 * pattern for a no_caller_saved_registers call.

	 * pattern for a bpf_fastcall call.

	 */

	u8 nocsr_pattern:1;

	u8 fastcall_pattern:1;

	/* for CALL instructions, a number of spill/fill pairs in the

	 * no_caller_saved_registers pattern.

	 * bpf_fastcall pattern.

	 */

	u8 nocsr_spills_num:3;

	u8 fastcall_spills_num:3;



	/* below fields are initialized once */

	unsigned int orig_idx; /* original instruction index */
@@ -653,10 +653,10 @@ struct bpf_subprog_info { 
	u32 linfo_idx; /* The idx to the main_prog->aux->linfo */

	u16 stack_depth; /* max. stack depth used by this function */

	u16 stack_extra;

	/* offsets in range [stack_depth .. nocsr_stack_off)

	 * are used for no_caller_saved_registers spills and fills.

	/* offsets in range [stack_depth .. fastcall_stack_off)

	 * are used for bpf_fastcall spills and fills.

	 */

	s16 nocsr_stack_off;

	s16 fastcall_stack_off;

	bool has_tail_call: 1;

	bool tail_call_reachable: 1;

	bool has_ld_abs: 1;
@@ -664,8 +664,8 @@ struct bpf_subprog_info { 
	bool is_async_cb: 1;

	bool is_exception_cb: 1;

	bool args_cached: 1;

	/* true if nocsr stack region is used by functions that can't be inlined */

	bool keep_nocsr_stack: 1;

	/* true if bpf_fastcall stack region is used by functions that can't be inlined */

	bool keep_fastcall_stack: 1;



	u8 arg_cnt;

	struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];

kernel/bpf/helpers.c

+1 −1
Original line number Diff line number Diff line
@@ -158,7 +158,7 @@ const struct bpf_func_proto bpf_get_smp_processor_id_proto = { 
	.func		= bpf_get_smp_processor_id,

	.gpl_only	= false,

	.ret_type	= RET_INTEGER,

	.allow_nocsr	= true,

	.allow_fastcall	= true,

};



BPF_CALL_0(bpf_get_numa_node_id)

kernel/bpf/verifier.c

+108 −73
Original line number Diff line number Diff line
@@ -4579,28 +4579,28 @@ static int get_reg_width(struct bpf_reg_state *reg) 
	return fls64(reg->umax_value);

}









/* See comment for mark_nocsr_pattern_for_call() */









static void check_nocsr_stack_contract(struct bpf_verifier_env *env, struct bpf_func_state *state,









				       int insn_idx, int off)









/* See comment for mark_fastcall_pattern_for_call() */









static void check_fastcall_stack_contract(struct bpf_verifier_env *env,









					  struct bpf_func_state *state, int insn_idx, int off)









{









	struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];









	struct bpf_insn_aux_data *aux = env->insn_aux_data;









	int i;

















	if (subprog->nocsr_stack_off <= off || aux[insn_idx].nocsr_pattern)









	if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern)









		return;









	/* access to the region [max_stack_depth .. nocsr_stack_off)









	 * from something that is not a part of the nocsr pattern,









	 * disable nocsr rewrites for current subprogram by setting









	 * nocsr_stack_off to a value smaller than any possible offset.









	/* access to the region [max_stack_depth .. fastcall_stack_off)









	 * from something that is not a part of the fastcall pattern,









	 * disable fastcall rewrites for current subprogram by setting









	 * fastcall_stack_off to a value smaller than any possible offset.









	 */









	subprog->nocsr_stack_off = S16_MIN;









	/* reset nocsr aux flags within subprogram,









	subprog->fastcall_stack_off = S16_MIN;









	/* reset fastcall aux flags within subprogram,









	 * happens at most once per subprogram









	 */









	for (i = subprog->start; i < (subprog + 1)->start; ++i) {









		aux[i].nocsr_spills_num = 0;









		aux[i].nocsr_pattern = 0;









		aux[i].fastcall_spills_num = 0;









		aux[i].fastcall_pattern = 0;









	}









}
















@@ -4652,7 +4652,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,







	if (err)









		return err;

















	check_nocsr_stack_contract(env, state, insn_idx, off);









	check_fastcall_stack_contract(env, state, insn_idx, off);









	mark_stack_slot_scratched(env, spi);









	if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {









		bool reg_value_fits;








@@ -4787,7 +4787,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,







			return err;









	}

















	check_nocsr_stack_contract(env, state, insn_idx, min_off);









	check_fastcall_stack_contract(env, state, insn_idx, min_off);









	/* Variable offset writes destroy any spilled pointers in range. */









	for (i = min_off; i < max_off; i++) {









		u8 new_type, *stype;








@@ -4926,7 +4926,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,







	reg = &reg_state->stack[spi].spilled_ptr;

















	mark_stack_slot_scratched(env, spi);









	check_nocsr_stack_contract(env, state, env->insn_idx, off);









	check_fastcall_stack_contract(env, state, env->insn_idx, off);

















	if (is_spilled_reg(&reg_state->stack[spi])) {









		u8 spill_size = 1;








@@ -5087,7 +5087,7 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env,







	min_off = reg->smin_value + off;









	max_off = reg->smax_value + off;









	mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);









	check_nocsr_stack_contract(env, ptr_state, env->insn_idx, min_off);









	check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);









	return 0;









}
















@@ -6804,13 +6804,13 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,







	struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx];









	int min_valid_off, max_bpf_stack;

















	/* If accessing instruction is a spill/fill from nocsr pattern,









	/* If accessing instruction is a spill/fill from bpf_fastcall pattern,









	 * add room for all caller saved registers below MAX_BPF_STACK.









	 * In case if nocsr rewrite won't happen maximal stack depth









	 * In case if bpf_fastcall rewrite won't happen maximal stack depth









	 * would be checked by check_max_stack_depth_subprog().









	 */









	max_bpf_stack = MAX_BPF_STACK;









	if (aux->nocsr_pattern)









	if (aux->fastcall_pattern)









		max_bpf_stack += CALLER_SAVED_REGS * BPF_REG_SIZE;

















	if (t == BPF_WRITE || env->allow_uninit_stack)








@@ -16118,14 +16118,14 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns,















/* Return a bitmask specifying which caller saved registers are









 * clobbered by a call to a helper *as if* this helper follows









 * no_caller_saved_registers contract:









 * bpf_fastcall contract:









 * - includes R0 if function is non-void;









 * - includes R1-R5 if corresponding parameter has is described









 *   in the function prototype.









 */









static u32 helper_nocsr_clobber_mask(const struct bpf_func_proto *fn)









static u32 helper_fastcall_clobber_mask(const struct bpf_func_proto *fn)









{









	u8 mask;









	u32 mask;









	int i;

















	mask = 0;








@@ -16138,8 +16138,8 @@ static u32 helper_nocsr_clobber_mask(const struct bpf_func_proto *fn)







}

















/* True if do_misc_fixups() replaces calls to helper number 'imm',









 * replacement patch is presumed to follow no_caller_saved_registers contract









 * (see mark_nocsr_pattern_for_call() below).









 * replacement patch is presumed to follow bpf_fastcall contract









 * (see mark_fastcall_pattern_for_call() below).









 */









static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)









{








@@ -16153,7 +16153,30 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)







	}









}

















/* GCC and LLVM define a no_caller_saved_registers function attribute.









/* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */









static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)









{









	u32 vlen, i, mask;

















	vlen = btf_type_vlen(meta->func_proto);









	mask = 0;









	if (!btf_type_is_void(btf_type_by_id(meta->btf, meta->func_proto->type)))









		mask |= BIT(BPF_REG_0);









	for (i = 0; i < vlen; ++i)









		mask |= BIT(BPF_REG_1 + i);









	return mask;









}

















/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */









static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)









{









	if (meta->btf == btf_vmlinux)









		return meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||









		       meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast];









	return false;









}

















/* LLVM define a bpf_fastcall function attribute.









 * This attribute means that function scratches only some of









 * the caller saved registers defined by ABI.









 * For BPF the set of such registers could be defined as follows:








@@ -16163,13 +16186,12 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)







 *









 * The contract between kernel and clang allows to simultaneously use









 * such functions and maintain backwards compatibility with old









 * kernels that don't understand no_caller_saved_registers calls









 * (nocsr for short):









 * kernels that don't understand bpf_fastcall calls:









 *









 * - for nocsr calls clang allocates registers as-if relevant r0-r5









 * - for bpf_fastcall calls clang allocates registers as-if relevant r0-r5









 *   registers are not scratched by the call;









 *









 * - as a post-processing step, clang visits each nocsr call and adds









 * - as a post-processing step, clang visits each bpf_fastcall call and adds









 *   spill/fill for every live r0-r5;









 *









 * - stack offsets used for the spill/fill are allocated as lowest








@@ -16177,11 +16199,11 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)







 *   purposes;









 *









 * - when kernel loads a program, it looks for such patterns









 *   (nocsr function surrounded by spills/fills) and checks if









 *   spill/fill stack offsets are used exclusively in nocsr patterns;









 *   (bpf_fastcall function surrounded by spills/fills) and checks if









 *   spill/fill stack offsets are used exclusively in fastcall patterns;









 *









 * - if so, and if verifier or current JIT inlines the call to the









 *   nocsr function (e.g. a helper call), kernel removes unnecessary









 *   bpf_fastcall function (e.g. a helper call), kernel removes unnecessary









 *   spill/fill pairs;









 *









 * - when old kernel loads a program, presence of spill/fill pairs








@@ -16200,22 +16222,22 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)







 *   r0 += r2;









 *   exit;









 *









 * The purpose of mark_nocsr_pattern_for_call is to:









 * The purpose of mark_fastcall_pattern_for_call is to:









 * - look for such patterns;









 * - mark spill and fill instructions in env->insn_aux_data[*].nocsr_pattern;









 * - mark set env->insn_aux_data[*].nocsr_spills_num for call instruction;









 * - update env->subprog_info[*]->nocsr_stack_off to find an offset









 *   at which nocsr spill/fill stack slots start;









 * - update env->subprog_info[*]->keep_nocsr_stack.









 * - mark spill and fill instructions in env->insn_aux_data[*].fastcall_pattern;









 * - mark set env->insn_aux_data[*].fastcall_spills_num for call instruction;









 * - update env->subprog_info[*]->fastcall_stack_off to find an offset









 *   at which bpf_fastcall spill/fill stack slots start;









 * - update env->subprog_info[*]->keep_fastcall_stack.









 *









 * The .nocsr_pattern and .nocsr_stack_off are used by









 * check_nocsr_stack_contract() to check if every stack access to









 * nocsr spill/fill stack slot originates from spill/fill









 * instructions, members of nocsr patterns.









 * The .fastcall_pattern and .fastcall_stack_off are used by









 * check_fastcall_stack_contract() to check if every stack access to









 * fastcall spill/fill stack slot originates from spill/fill









 * instructions, members of fastcall patterns.









 *









 * If such condition holds true for a subprogram, nocsr patterns could









 * be rewritten by remove_nocsr_spills_fills().









 * Otherwise nocsr patterns are not changed in the subprogram









 * If such condition holds true for a subprogram, fastcall patterns could









 * be rewritten by remove_fastcall_spills_fills().









 * Otherwise bpf_fastcall patterns are not changed in the subprogram









 * (code, presumably, generated by an older clang version).









 *









 * For example, it is *not* safe to remove spill/fill below:








@@ -16228,7 +16250,7 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)







 *   r0 += r1;                           exit;









 *   exit;









 */









static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env,









static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,









					   struct bpf_subprog_info *subprog,









					   int insn_idx, s16 lowest_off)









{








@@ -16245,12 +16267,25 @@ static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env,







		if (get_helper_proto(env, call->imm, &fn) < 0)









			/* error would be reported later */









			return;









		clobbered_regs_mask = helper_nocsr_clobber_mask(fn);









		can_be_inlined = fn->allow_nocsr &&









		clobbered_regs_mask = helper_fastcall_clobber_mask(fn);









		can_be_inlined = fn->allow_fastcall &&









				 (verifier_inlines_helper_call(env, call->imm) ||









				  bpf_jit_inlines_helper_call(call->imm));









	}

















	if (bpf_pseudo_kfunc_call(call)) {









		struct bpf_kfunc_call_arg_meta meta;









		int err;

















		err = fetch_kfunc_meta(env, call, &meta, NULL);









		if (err < 0)









			/* error would be reported later */









			return;

















		clobbered_regs_mask = kfunc_fastcall_clobber_mask(&meta);









		can_be_inlined = is_fastcall_kfunc_call(&meta);









	}

















	if (clobbered_regs_mask == ALL_CALLER_SAVED_REGS)









		return;
















@@ -16289,36 +16324,36 @@ static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env,







		if (stx->off != off || ldx->off != off)









			break;









		expected_regs_mask &= ~BIT(stx->src_reg);









		env->insn_aux_data[insn_idx - i].nocsr_pattern = 1;









		env->insn_aux_data[insn_idx + i].nocsr_pattern = 1;









		env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;









		env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;









	}









	if (i == 1)









		return;

















	/* Conditionally set 'nocsr_spills_num' to allow forward









	/* Conditionally set 'fastcall_spills_num' to allow forward









	 * compatibility when more helper functions are marked as









	 * nocsr at compile time than current kernel supports, e.g:









	 * bpf_fastcall at compile time than current kernel supports, e.g:









	 *









	 *   1: *(u64 *)(r10 - 8) = r1









	 *   2: call A                  ;; assume A is nocsr for current kernel









	 *   2: call A                  ;; assume A is bpf_fastcall for current kernel









	 *   3: r1 = *(u64 *)(r10 - 8)









	 *   4: *(u64 *)(r10 - 8) = r1









	 *   5: call B                  ;; assume B is not nocsr for current kernel









	 *   5: call B                  ;; assume B is not bpf_fastcall for current kernel









	 *   6: r1 = *(u64 *)(r10 - 8)









	 *









	 * There is no need to block nocsr rewrite for such program.









	 * Set 'nocsr_pattern' for both calls to keep check_nocsr_stack_contract() happy,









	 * don't set 'nocsr_spills_num' for call B so that remove_nocsr_spills_fills()









	 * There is no need to block bpf_fastcall rewrite for such program.









	 * Set 'fastcall_pattern' for both calls to keep check_fastcall_stack_contract() happy,









	 * don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills()









	 * does not remove spill/fill pair {4,6}.









	 */









	if (can_be_inlined)









		env->insn_aux_data[insn_idx].nocsr_spills_num = i - 1;









		env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;









	else









		subprog->keep_nocsr_stack = 1;









	subprog->nocsr_stack_off = min(subprog->nocsr_stack_off, off);









		subprog->keep_fastcall_stack = 1;









	subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off);









}

















static int mark_nocsr_patterns(struct bpf_verifier_env *env)









static int mark_fastcall_patterns(struct bpf_verifier_env *env)









{









	struct bpf_subprog_info *subprog = env->subprog_info;









	struct bpf_insn *insn;








@@ -16335,12 +16370,12 @@ static int mark_nocsr_patterns(struct bpf_verifier_env *env)







				continue;









			lowest_off = min(lowest_off, insn->off);









		}









		/* use this offset to find nocsr patterns */









		/* use this offset to find fastcall patterns */









		for (i = subprog->start; i < (subprog + 1)->start; ++i) {









			insn = env->prog->insnsi + i;









			if (insn->code != (BPF_JMP | BPF_CALL))









				continue;









			mark_nocsr_pattern_for_call(env, subprog, i, lowest_off);









			mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);









		}









	}









	return 0;








@@ -21244,10 +21279,10 @@ static int optimize_bpf_loop(struct bpf_verifier_env *env)







	return 0;









}

















/* Remove unnecessary spill/fill pairs, members of nocsr pattern,









/* Remove unnecessary spill/fill pairs, members of fastcall pattern,









 * adjust subprograms stack depth when possible.









 */









static int remove_nocsr_spills_fills(struct bpf_verifier_env *env)









static int remove_fastcall_spills_fills(struct bpf_verifier_env *env)









{









	struct bpf_subprog_info *subprog = env->subprog_info;









	struct bpf_insn_aux_data *aux = env->insn_aux_data;








@@ -21258,8 +21293,8 @@ static int remove_nocsr_spills_fills(struct bpf_verifier_env *env)







	int i, j;

















	for (i = 0; i < insn_cnt; i++, insn++) {









		if (aux[i].nocsr_spills_num > 0) {









			spills_num = aux[i].nocsr_spills_num;









		if (aux[i].fastcall_spills_num > 0) {









			spills_num = aux[i].fastcall_spills_num;









			/* NOPs would be removed by opt_remove_nops() */









			for (j = 1; j <= spills_num; ++j) {









				*(insn - j) = NOP;








@@ -21268,8 +21303,8 @@ static int remove_nocsr_spills_fills(struct bpf_verifier_env *env)







			modified = true;









		}









		if ((subprog + 1)->start == i + 1) {









			if (modified && !subprog->keep_nocsr_stack)









				subprog->stack_depth = -subprog->nocsr_stack_off;









			if (modified && !subprog->keep_fastcall_stack)









				subprog->stack_depth = -subprog->fastcall_stack_off;









			subprog++;









			modified = false;









		}








@@ -22192,7 +22227,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3







	if (ret < 0)









		goto skip_full_check;

















	ret = mark_nocsr_patterns(env);









	ret = mark_fastcall_patterns(env);









	if (ret < 0)









		goto skip_full_check;
















@@ -22209,7 +22244,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3







	 * allocate additional slots.









	 */









	if (ret == 0)









		ret = remove_nocsr_spills_fills(env);









		ret = remove_fastcall_spills_fills(env);

















	if (ret == 0)









		ret = check_max_stack_depth(env);

























tools/testing/selftests/bpf/prog_tests/verifier.c



+2
−2






























Original line number
Diff line number
Diff line







@@ -53,7 +53,7 @@







#include "verifier_movsx.skel.h"









#include "verifier_netfilter_ctx.skel.h"









#include "verifier_netfilter_retcode.skel.h"









#include "verifier_nocsr.skel.h"









#include "verifier_bpf_fastcall.skel.h"









#include "verifier_or_jmp32_k.skel.h"









#include "verifier_precision.skel.h"









#include "verifier_prevent_map_lookup.skel.h"








@@ -177,7 +177,7 @@ void test_verifier_meta_access(void)          { RUN(verifier_meta_access); }







void test_verifier_movsx(void)                 { RUN(verifier_movsx); }









void test_verifier_netfilter_ctx(void)        { RUN(verifier_netfilter_ctx); }









void test_verifier_netfilter_retcode(void)    { RUN(verifier_netfilter_retcode); }









void test_verifier_nocsr(void)                { RUN(verifier_nocsr); }









void test_verifier_bpf_fastcall(void)         { RUN(verifier_bpf_fastcall); }









void test_verifier_or_jmp32_k(void)           { RUN(verifier_or_jmp32_k); }









void test_verifier_precision(void)            { RUN(verifier_precision); }









void test_verifier_prevent_map_lookup(void)   { RUN(verifier_prevent_map_lookup); }