Commit 0ffdbce6 authored by Naveen N Rao's avatar Naveen N Rao Committed by Michael Ellerman
Browse files

powerpc/bpf: Handle large branch ranges with BPF_EXIT 



In some scenarios, it is possible that the program epilogue is outside
the branch range for a BPF_EXIT instruction. Instead of rejecting such
programs, emit epilogue as an alternate exit point from the program.
Track the location of the same so that subsequent exits can take either
of the two paths.

Reported-by: default avatarJordan Niethe <jniethe5@gmail.com>
Signed-off-by: default avatarNaveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/33aa2e92645a92712be23b18035a2c6dcb92ff8d.1644834730.git.naveen.n.rao@linux.vnet.ibm.com
parent bafb5898

arch/powerpc/net/bpf_jit.h

+2 −0
Original line number Diff line number Diff line
@@ -148,6 +148,7 @@ struct codegen_context { 
	unsigned int stack_size;

	int b2p[ARRAY_SIZE(b2p)];

	unsigned int exentry_idx;

	unsigned int alt_exit_addr;

};



#ifdef CONFIG_PPC32
@@ -183,6 +184,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * 
void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx);

void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx);

void bpf_jit_realloc_regs(struct codegen_context *ctx);

int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr);



int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,

			  int insn_idx, int jmp_off, int dst_reg);

arch/powerpc/net/bpf_jit_comp.c

+21 −1
Original line number Diff line number Diff line
@@ -89,6 +89,22 @@ static int bpf_jit_fixup_addresses(struct bpf_prog *fp, u32 *image, 
	return 0;

}



int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)

{

	if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {

		PPC_JMP(exit_addr);

	} else if (ctx->alt_exit_addr) {

		if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))

			return -1;

		PPC_JMP(ctx->alt_exit_addr);

	} else {

		ctx->alt_exit_addr = ctx->idx * 4;

		bpf_jit_build_epilogue(image, ctx);

	}



	return 0;

}



struct powerpc64_jit_data {

	struct bpf_binary_header *header;

	u32 *addrs;
@@ -177,8 +193,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 
	 * If we have seen a tail call, we need a second pass.

	 * This is because bpf_jit_emit_common_epilogue() is called

	 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.

	 * We also need a second pass if we ended up with too large

	 * a program so as to ensure BPF_EXIT branches are in range.

	 */

	if (cgctx.seen & SEEN_TAILCALL) {

	if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {

		cgctx.idx = 0;

		if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {

			fp = org_fp;
@@ -193,6 +211,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 
	 * calculate total size from idx.

	 */

	bpf_jit_build_prologue(0, &cgctx);

	addrs[fp->len] = cgctx.idx * 4;

	bpf_jit_build_epilogue(0, &cgctx);



	fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
@@ -233,6 +252,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 
	for (pass = 1; pass < 3; pass++) {

		/* Now build the prologue, body code & epilogue for real. */

		cgctx.idx = 0;

		cgctx.alt_exit_addr = 0;

		bpf_jit_build_prologue(code_base, &cgctx);

		if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass)) {

			bpf_jit_binary_free(bpf_hdr);

arch/powerpc/net/bpf_jit_comp32.c

+5 −2
Original line number Diff line number Diff line
@@ -937,8 +937,11 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * 
			 * the epilogue. If we _are_ the last instruction,

			 * we'll just fall through to the epilogue.

			 */

			if (i != flen - 1)

				PPC_JMP(exit_addr);

			if (i != flen - 1) {

				ret = bpf_jit_emit_exit_insn(image, ctx, _R0, exit_addr);

				if (ret)

					return ret;

			}

			/* else fall through to the epilogue */

			break;

arch/powerpc/net/bpf_jit_comp64.c

+5 −2
Original line number Diff line number Diff line
@@ -871,8 +871,11 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * 
			 * the epilogue. If we _are_ the last instruction,

			 * we'll just fall through to the epilogue.

			 */

			if (i != flen - 1)

				PPC_JMP(exit_addr);

			if (i != flen - 1) {

				ret = bpf_jit_emit_exit_insn(image, ctx, b2p[TMP_REG_1], exit_addr);

				if (ret)

					return ret;

			}

			/* else fall through to the epilogue */

			break;