Commit 493d9e0d authored by Anton Protopopov's avatar Anton Protopopov Committed by Alexei Starovoitov
Browse files

bpf, x86: add support for indirect jumps 



Add support for a new instruction

    BPF_JMP|BPF_X|BPF_JA, SRC=0, DST=Rx, off=0, imm=0

which does an indirect jump to a location stored in Rx.  The register
Rx should have type PTR_TO_INSN. This new type assures that the Rx
register contains a value (or a range of values) loaded from a
correct jump table – map of type instruction array.

For example, for a C switch LLVM will generate the following code:

    0:   r3 = r1                    # "switch (r3)"
    1:   if r3 > 0x13 goto +0x666   # check r3 boundaries
    2:   r3 <<= 0x3                 # adjust to an index in array of addresses
    3:   r1 = 0xbeef ll             # r1 is PTR_TO_MAP_VALUE, r1->map_ptr=M
    5:   r1 += r3                   # r1 inherits boundaries from r3
    6:   r1 = *(u64 *)(r1 + 0x0)    # r1 now has type INSN_TO_PTR
    7:   gotox r1                   # jit will generate proper code

Here the gotox instruction corresponds to one particular map. This is
possible however to have a gotox instruction which can be loaded from
different maps, e.g.

    0:   r1 &= 0x1
    1:   r2 <<= 0x3
    2:   r3 = 0x0 ll                # load from map M_1
    4:   r3 += r2
    5:   if r1 == 0x0 goto +0x4
    6:   r1 <<= 0x3
    7:   r3 = 0x0 ll                # load from map M_2
    9:   r3 += r1
    A:   r1 = *(u64 *)(r3 + 0x0)
    B:   gotox r1                   # jump to target loaded from M_1 or M_2

During check_cfg stage the verifier will collect all the maps which
point to inside the subprog being verified. When building the config,
the high 16 bytes of the insn_state are used, so this patch
(theoretically) supports jump tables of up to 2^16 slots.

During the later stage, in check_indirect_jump, it is checked that
the register Rx was loaded from a particular instruction array.

Signed-off-by: default avatarAnton Protopopov <a.s.protopopov@gmail.com>
Acked-by: default avatarEduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251105090410.1250500-9-a.s.protopopov@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 5bef46ac

arch/x86/net/bpf_jit_comp.c

+3 −0
Original line number Diff line number Diff line
@@ -2628,6 +2628,9 @@ st: if (is_imm8(insn->off)) 


			break;



		case BPF_JMP | BPF_JA | BPF_X:

			emit_indirect_jump(&prog, insn->dst_reg, image + addrs[i - 1]);

			break;

		case BPF_JMP | BPF_JA:

		case BPF_JMP32 | BPF_JA:

			if (BPF_CLASS(insn->code) == BPF_JMP) {

include/linux/bpf.h

+1 −0
Original line number Diff line number Diff line
@@ -1001,6 +1001,7 @@ enum bpf_reg_type { 
	PTR_TO_ARENA,

	PTR_TO_BUF,		 /* reg points to a read/write buffer */

	PTR_TO_FUNC,		 /* reg points to a bpf program function */

	PTR_TO_INSN,		 /* reg points to a bpf program instruction */

	CONST_PTR_TO_DYNPTR,	 /* reg points to a const struct bpf_dynptr */

	__BPF_REG_TYPE_MAX,

include/linux/bpf_verifier.h

+9 −0
Original line number Diff line number Diff line
@@ -527,6 +527,7 @@ struct bpf_insn_aux_data { 
		struct {

			u32 map_index;		/* index into used_maps[] */

			u32 map_off;		/* offset from value base address */

			struct bpf_iarray *jt;	/* jump table for gotox instruction */

		};

		struct {

			enum bpf_reg_type reg_type;	/* type of pseudo_btf_id */
@@ -840,6 +841,7 @@ struct bpf_verifier_env { 
	struct bpf_scc_info **scc_info;

	u32 scc_cnt;

	struct bpf_iarray *succ;

	struct bpf_iarray *gotox_tmp_buf;

};



static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)
@@ -1050,6 +1052,13 @@ static inline bool bpf_stack_narrow_access_ok(int off, int fill_size, int spill_ 
	return !(off % BPF_REG_SIZE);

}



static inline bool insn_is_gotox(struct bpf_insn *insn)

{

	return BPF_CLASS(insn->code) == BPF_JMP &&

	       BPF_OP(insn->code) == BPF_JA &&

	       BPF_SRC(insn->code) == BPF_X;

}



const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type);

const char *dynptr_type_str(enum bpf_dynptr_type type);

const char *iter_type_str(const struct btf *btf, u32 btf_id);

kernel/bpf/bpf_insn_array.c

+15 −0
Original line number Diff line number Diff line
@@ -114,6 +114,20 @@ static u64 insn_array_mem_usage(const struct bpf_map *map) 
	return insn_array_alloc_size(map->max_entries);

}



static int insn_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off)

{

	struct bpf_insn_array *insn_array = cast_insn_array(map);



	if ((off % sizeof(long)) != 0 ||

	    (off / sizeof(long)) >= map->max_entries)

		return -EINVAL;



	/* from BPF's point of view, this map is a jump table */

	*imm = (unsigned long)insn_array->ips + off;



	return 0;

}



BTF_ID_LIST_SINGLE(insn_array_btf_ids, struct, bpf_insn_array)



const struct bpf_map_ops insn_array_map_ops = {
@@ -126,6 +140,7 @@ const struct bpf_map_ops insn_array_map_ops = { 
	.map_delete_elem = insn_array_delete_elem,

	.map_check_btf = insn_array_check_btf,

	.map_mem_usage = insn_array_mem_usage,

	.map_direct_value_addr = insn_array_map_direct_value_addr,

	.map_btf_id = &insn_array_btf_ids[0],

};

kernel/bpf/core.c

+1 −0
Original line number Diff line number Diff line
@@ -1708,6 +1708,7 @@ bool bpf_opcode_in_insntable(u8 code) 
		[BPF_LD | BPF_IND | BPF_B] = true,

		[BPF_LD | BPF_IND | BPF_H] = true,

		[BPF_LD | BPF_IND | BPF_W] = true,

		[BPF_JMP | BPF_JA | BPF_X] = true,

		[BPF_JMP | BPF_JCOND] = true,

	};

#undef BPF_INSN_3_TBL