Commit eb0d6d97 authored by Linus Torvalds's avatar Linus Torvalds
Browse files
Pull bpf fixes from Alexei Starovoitov:
 "Most of the diff stat comes from Xu Kuohai's fix to emit ENDBR/BTI,
  since all JITs had to be touched to move constant blinding out and
  pass bpf_verifier_env in.

   - Fix use-after-free in arena_vm_close on fork (Alexei Starovoitov)

   - Dissociate struct_ops program with map if map_update fails (Amery
     Hung)

   - Fix out-of-range and off-by-one bugs in arm64 JIT (Daniel Borkmann)

   - Fix precedence bug in convert_bpf_ld_abs alignment check (Daniel
     Borkmann)

   - Fix arg tracking for imprecise/multi-offset in BPF_ST/STX insns
     (Eduard Zingerman)

   - Copy token from main to subprogs to fix missing kallsyms (Eduard
     Zingerman)

   - Prevent double close and leak of btf objects in libbpf (Jiri Olsa)

   - Fix af_unix null-ptr-deref in sockmap (Michal Luczaj)

   - Fix NULL deref in map_kptr_match_type for scalar regs (Mykyta
     Yatsenko)

   - Avoid unnecessary IPIs. Remove redundant bpf_flush_icache() in
     arm64 and riscv JITs (Puranjay Mohan)

   - Fix out of bounds access. Validate node_id in arena_alloc_pages()
     (Puranjay Mohan)

   - Reject BPF-to-BPF calls and callbacks in arm32 JIT (Puranjay Mohan)

   - Refactor all JITs to pass bpf_verifier_env to emit ENDBR/BTI for
     indirect jump targets on x86-64, arm64 JITs (Xu Kuohai)

   - Allow UTF-8 literals in bpf_bprintf_prepare() (Yihan Ding)"

* tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (32 commits)
  bpf, arm32: Reject BPF-to-BPF calls and callbacks in the JIT
  bpf: Dissociate struct_ops program with map if map_update fails
  bpf: Validate node_id in arena_alloc_pages()
  libbpf: Prevent double close and leak of btf objects
  selftests/bpf: cover UTF-8 trace_printk output
  bpf: allow UTF-8 literals in bpf_bprintf_prepare()
  selftests/bpf: Reject scalar store into kptr slot
  bpf: Fix NULL deref in map_kptr_match_type for scalar regs
  bpf: Fix precedence bug in convert_bpf_ld_abs alignment check
  bpf, arm64: Emit BTI for indirect jump target
  bpf, x86: Emit ENDBR for indirect jump targets
  bpf: Add helper to detect indirect jump targets
  bpf: Pass bpf_verifier_env to JIT
  bpf: Move constants blinding out of arch-specific JITs
  bpf, sockmap: Take state lock for af_unix iter
  bpf, sockmap: Fix af_unix null-ptr-deref in proto update
  selftests/bpf: Extend bpf_iter_unix to attempt deadlocking
  bpf, sockmap: Fix af_unix iter deadlock
  bpf, sockmap: Annotate af_unix sock:: Sk_state data-races
  selftests/bpf: verify kallsyms entries for token-loaded subprograms
  ...
parents 12bffaef e1d48644
Loading
Loading
Loading
Loading
+15 −26
Original line number Diff line number Diff line
@@ -79,7 +79,6 @@ struct arc_jit_data {
 * The JIT pertinent context that is used by different functions.
 *
 * prog:		The current eBPF program being handled.
 * orig_prog:		The original eBPF program before any possible change.
 * jit:			The JIT buffer and its length.
 * bpf_header:		The JITed program header. "jit.buf" points inside it.
 * emit:		If set, opcodes are written to memory; else, a dry-run.
@@ -94,12 +93,10 @@ struct arc_jit_data {
 * need_extra_pass:	A forecast if an "extra_pass" will occur.
 * is_extra_pass:	Indicates if the current pass is an extra pass.
 * user_bpf_prog:	True, if VM opcodes come from a real program.
 * blinded:		True if "constant blinding" step returned a new "prog".
 * success:		Indicates if the whole JIT went OK.
 */
struct jit_context {
	struct bpf_prog			*prog;
	struct bpf_prog			*orig_prog;
	struct jit_buffer		jit;
	struct bpf_binary_header	*bpf_header;
	bool				emit;
@@ -114,7 +111,6 @@ struct jit_context {
	bool				need_extra_pass;
	bool				is_extra_pass;
	bool				user_bpf_prog;
	bool				blinded;
	bool				success;
};

@@ -161,13 +157,7 @@ static int jit_ctx_init(struct jit_context *ctx, struct bpf_prog *prog)
{
	memset(ctx, 0, sizeof(*ctx));

	ctx->orig_prog = prog;

	/* If constant blinding was requested but failed, scram. */
	ctx->prog = bpf_jit_blind_constants(prog);
	if (IS_ERR(ctx->prog))
		return PTR_ERR(ctx->prog);
	ctx->blinded = (ctx->prog != ctx->orig_prog);
	ctx->prog = prog;

	/* If the verifier doesn't zero-extend, then we have to do it. */
	ctx->do_zext = !ctx->prog->aux->verifier_zext;
@@ -214,14 +204,6 @@ static inline void maybe_free(struct jit_context *ctx, void **mem)
 */
static void jit_ctx_cleanup(struct jit_context *ctx)
{
	if (ctx->blinded) {
		/* if all went well, release the orig_prog. */
		if (ctx->success)
			bpf_jit_prog_release_other(ctx->prog, ctx->orig_prog);
		else
			bpf_jit_prog_release_other(ctx->orig_prog, ctx->prog);
	}

	maybe_free(ctx, (void **)&ctx->bpf2insn);
	maybe_free(ctx, (void **)&ctx->jit_data);

@@ -229,13 +211,20 @@ static void jit_ctx_cleanup(struct jit_context *ctx)
		ctx->bpf2insn_valid = false;

	/* Freeing "bpf_header" is enough. "jit.buf" is a sub-array of it. */
	if (!ctx->success && ctx->bpf_header) {
	if (!ctx->success) {
		if (ctx->bpf_header) {
			bpf_jit_binary_free(ctx->bpf_header);
			ctx->bpf_header = NULL;
			ctx->jit.buf    = NULL;
			ctx->jit.index  = 0;
			ctx->jit.len    = 0;
		}
		if (ctx->is_extra_pass) {
			ctx->prog->bpf_func = NULL;
			ctx->prog->jited = 0;
			ctx->prog->jited_len = 0;
		}
	}

	ctx->emit = false;
	ctx->do_zext = false;
@@ -1411,7 +1400,7 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
 * (re)locations involved that their addresses are not known
 * during the first run.
 */
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
	vm_dump(prog);

+14 −35
Original line number Diff line number Diff line
@@ -1852,6 +1852,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
	{
		u64 val = (u32)imm | (u64)insn[1].imm << 32;

		if (insn->src_reg == BPF_PSEUDO_FUNC)
			goto notyet;

		emit_a32_mov_i64(dst, val, ctx);

		return 1;
@@ -2055,6 +2058,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
		const s8 *r5 = bpf2a32[BPF_REG_5];
		const u32 func = (u32)__bpf_call_base + (u32)imm;

		if (insn->src_reg == BPF_PSEUDO_CALL)
			goto notyet;

		emit_a32_mov_r64(true, r0, r1, ctx);
		emit_a32_mov_r64(true, r1, r2, ctx);
		emit_push_r64(r5, ctx);
@@ -2142,11 +2148,9 @@ bool bpf_jit_needs_zext(void)
	return true;
}

struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
	struct bpf_prog *tmp, *orig_prog = prog;
	struct bpf_binary_header *header;
	bool tmp_blinded = false;
	struct jit_ctx ctx;
	unsigned int tmp_idx;
	unsigned int image_size;
@@ -2156,20 +2160,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	 * the interpreter.
	 */
	if (!prog->jit_requested)
		return orig_prog;

	/* If constant blinding was enabled and we failed during blinding
	 * then we must fall back to the interpreter. Otherwise, we save
	 * the new JITed code.
	 */
	tmp = bpf_jit_blind_constants(prog);

	if (IS_ERR(tmp))
		return orig_prog;
	if (tmp != prog) {
		tmp_blinded = true;
		prog = tmp;
	}
		return prog;

	memset(&ctx, 0, sizeof(ctx));
	ctx.prog = prog;
@@ -2179,10 +2170,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	 * we must fall back to the interpreter
	 */
	ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
	if (ctx.offsets == NULL) {
		prog = orig_prog;
		goto out;
	}
	if (ctx.offsets == NULL)
		return prog;

	/* 1) fake pass to find in the length of the JITed code,
	 * to compute ctx->offsets and other context variables
@@ -2194,10 +2183,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	 * being successful in the second pass, so just fall back
	 * to the interpreter.
	 */
	if (build_body(&ctx)) {
		prog = orig_prog;
	if (build_body(&ctx))
		goto out_off;
	}

	tmp_idx = ctx.idx;
	build_prologue(&ctx);
@@ -2213,11 +2200,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	ctx.idx += ctx.imm_count;
	if (ctx.imm_count) {
		ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL);
		if (ctx.imms == NULL) {
			prog = orig_prog;
		if (ctx.imms == NULL)
			goto out_off;
	}
	}
#else
	/* there's nothing about the epilogue on ARMv7 */
	build_epilogue(&ctx);
@@ -2238,10 +2223,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	/* Not able to allocate memory for the structure then
	 * we must fall back to the interpretation
	 */
	if (header == NULL) {
		prog = orig_prog;
	if (header == NULL)
		goto out_imms;
	}

	/* 2.) Actual pass to generate final JIT code */
	ctx.target = (u32 *) image_ptr;
@@ -2278,16 +2261,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
#endif
out_off:
	kfree(ctx.offsets);
out:
	if (tmp_blinded)
		bpf_jit_prog_release_other(prog, prog == orig_prog ?
					   tmp : orig_prog);

	return prog;

out_free:
	image_ptr = NULL;
	bpf_jit_binary_free(header);
	prog = orig_prog;
	goto out_imms;
}
+2 −0
Original line number Diff line number Diff line
@@ -338,6 +338,8 @@ u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
	long offset;

	offset = label_imm_common(pc, addr, SZ_1M);
	if (offset >= SZ_1M)
		return AARCH64_BREAK_FAULT;

	insn = aarch64_insn_get_bcond_value();

+31 −72
Original line number Diff line number Diff line
@@ -18,7 +18,6 @@

#include <asm/asm-extable.h>
#include <asm/byteorder.h>
#include <asm/cacheflush.h>
#include <asm/cpufeature.h>
#include <asm/debug-monitors.h>
#include <asm/insn.h>
@@ -35,8 +34,8 @@
#define ARENA_VM_START (MAX_BPF_JIT_REG + 5)

#define check_imm(bits, imm) do {				\
	if ((((imm) > 0) && ((imm) >> (bits))) ||		\
	    (((imm) < 0) && (~(imm) >> (bits)))) {		\
	if ((((imm) > 0) && ((imm) >> ((bits) - 1))) ||		\
	    (((imm) < 0) && (~(imm) >> ((bits) - 1)))) {	\
		pr_info("[%2d] imm=%d(0x%x) out of range\n",	\
			i, imm, imm);				\
		return -EINVAL;					\
@@ -1198,8 +1197,8 @@ static int add_exception_handler(const struct bpf_insn *insn,
 * >0 - successfully JITed a 16-byte eBPF instruction.
 * <0 - failed to JIT.
 */
static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
		      bool extra_pass)
static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn *insn,
		      struct jit_ctx *ctx, bool extra_pass)
{
	const u8 code = insn->code;
	u8 dst = bpf2a64[insn->dst_reg];
@@ -1224,6 +1223,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
	int ret;
	bool sign_extend;

	if (bpf_insn_is_indirect_target(env, ctx->prog, i))
		emit_bti(A64_BTI_J, ctx);

	switch (code) {
	/* dst = src */
	case BPF_ALU | BPF_MOV | BPF_X:
@@ -1899,7 +1901,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
	return 0;
}

static int build_body(struct jit_ctx *ctx, bool extra_pass)
static int build_body(struct bpf_verifier_env *env, struct jit_ctx *ctx, bool extra_pass)
{
	const struct bpf_prog *prog = ctx->prog;
	int i;
@@ -1918,7 +1920,7 @@ static int build_body(struct jit_ctx *ctx, bool extra_pass)
		int ret;

		ctx->offset[i] = ctx->idx;
		ret = build_insn(insn, ctx, extra_pass);
		ret = build_insn(env, insn, ctx, extra_pass);
		if (ret > 0) {
			i++;
			ctx->offset[i] = ctx->idx;
@@ -1961,11 +1963,6 @@ static int validate_ctx(struct jit_ctx *ctx)
	return 0;
}

static inline void bpf_flush_icache(void *start, void *end)
{
	flush_icache_range((unsigned long)start, (unsigned long)end);
}

static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
{
	int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
@@ -2006,17 +2003,15 @@ struct arm64_jit_data {
	struct jit_ctx ctx;
};

struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
	int image_size, prog_size, extable_size, extable_align, extable_offset;
	struct bpf_prog *tmp, *orig_prog = prog;
	struct bpf_binary_header *header;
	struct bpf_binary_header *ro_header = NULL;
	struct arm64_jit_data *jit_data;
	void __percpu *priv_stack_ptr = NULL;
	bool was_classic = bpf_prog_was_classic(prog);
	int priv_stack_alloc_sz;
	bool tmp_blinded = false;
	bool extra_pass = false;
	struct jit_ctx ctx;
	u8 *image_ptr;
@@ -2025,26 +2020,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	int exentry_idx;

	if (!prog->jit_requested)
		return orig_prog;

	tmp = bpf_jit_blind_constants(prog);
	/* If blinding was requested and we failed during blinding,
	 * we must fall back to the interpreter.
	 */
	if (IS_ERR(tmp))
		return orig_prog;
	if (tmp != prog) {
		tmp_blinded = true;
		prog = tmp;
	}
		return prog;

	jit_data = prog->aux->jit_data;
	if (!jit_data) {
		jit_data = kzalloc_obj(*jit_data);
		if (!jit_data) {
			prog = orig_prog;
			goto out;
		}
		if (!jit_data)
			return prog;
		prog->aux->jit_data = jit_data;
	}
	priv_stack_ptr = prog->aux->priv_stack_ptr;
@@ -2056,10 +2038,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
		priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 16) +
				      2 * PRIV_STACK_GUARD_SZ;
		priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 16, GFP_KERNEL);
		if (!priv_stack_ptr) {
			prog = orig_prog;
		if (!priv_stack_ptr)
			goto out_priv_stack;
		}

		priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);
		prog->aux->priv_stack_ptr = priv_stack_ptr;
@@ -2079,10 +2059,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	ctx.prog = prog;

	ctx.offset = kvzalloc_objs(int, prog->len + 1);
	if (ctx.offset == NULL) {
		prog = orig_prog;
	if (ctx.offset == NULL)
		goto out_off;
	}

	ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
	ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
@@ -2095,15 +2073,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	 * BPF line info needs ctx->offset[i] to be the offset of
	 * instruction[i] in jited image, so build prologue first.
	 */
	if (build_prologue(&ctx, was_classic)) {
		prog = orig_prog;
	if (build_prologue(&ctx, was_classic))
		goto out_off;
	}

	if (build_body(&ctx, extra_pass)) {
		prog = orig_prog;
	if (build_body(env, &ctx, extra_pass))
		goto out_off;
	}

	ctx.epilogue_offset = ctx.idx;
	build_epilogue(&ctx, was_classic);
@@ -2121,10 +2095,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr,
					      sizeof(u64), &header, &image_ptr,
					      jit_fill_hole);
	if (!ro_header) {
		prog = orig_prog;
	if (!ro_header)
		goto out_off;
	}

	/* Pass 2: Determine jited position and result for each instruction */

@@ -2152,10 +2124,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	/* Dont write body instructions to memory for now */
	ctx.write = false;

	if (build_body(&ctx, extra_pass)) {
		prog = orig_prog;
	if (build_body(env, &ctx, extra_pass))
		goto out_free_hdr;
	}

	ctx.epilogue_offset = ctx.idx;
	ctx.exentry_idx = exentry_idx;
@@ -2163,20 +2133,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	ctx.write = true;

	/* Pass 3: Adjust jump offset and write final image */
	if (build_body(&ctx, extra_pass) ||
		WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) {
		prog = orig_prog;
	if (build_body(env, &ctx, extra_pass) ||
		WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset))
		goto out_free_hdr;
	}

	build_epilogue(&ctx, was_classic);
	build_plt(&ctx);

	/* Extra pass to validate JITed code. */
	if (validate_ctx(&ctx)) {
		prog = orig_prog;
	if (validate_ctx(&ctx))
		goto out_free_hdr;
	}

	/* update the real prog size */
	prog_size = sizeof(u32) * ctx.idx;
@@ -2193,23 +2159,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
		if (extra_pass && ctx.idx > jit_data->ctx.idx) {
			pr_err_once("multi-func JIT bug %d > %d\n",
				    ctx.idx, jit_data->ctx.idx);
			prog->bpf_func = NULL;
			prog->jited = 0;
			prog->jited_len = 0;
			goto out_free_hdr;
		}
		if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {
			/* ro_header has been freed */
			/* ro_header and header has been freed */
			ro_header = NULL;
			prog = orig_prog;
			goto out_off;
			header = NULL;
			goto out_free_hdr;
		}
		/*
		 * The instructions have now been copied to the ROX region from
		 * where they will execute. Now the data cache has to be cleaned to
		 * the PoU and the I-cache has to be invalidated for the VAs.
		 */
		bpf_flush_icache(ro_header, ctx.ro_image + ctx.idx);
	} else {
		jit_data->ctx = ctx;
		jit_data->ro_image = ro_image_ptr;
@@ -2245,13 +2202,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
		kfree(jit_data);
		prog->aux->jit_data = NULL;
	}
out:
	if (tmp_blinded)
		bpf_jit_prog_release_other(prog, prog == orig_prog ?
					   tmp : orig_prog);

	return prog;

out_free_hdr:
	if (extra_pass) {
		prog->bpf_func = NULL;
		prog->jited = 0;
		prog->jited_len = 0;
	}
	if (header) {
		bpf_arch_text_copy(&ro_header->size, &header->size,
				   sizeof(header->size));
+18 −43
Original line number Diff line number Diff line
@@ -1920,45 +1920,28 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
	return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
}

struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
	bool tmp_blinded = false, extra_pass = false;
	bool extra_pass = false;
	u8 *image_ptr, *ro_image_ptr;
	int image_size, prog_size, extable_size;
	struct jit_ctx ctx;
	struct jit_data *jit_data;
	struct bpf_binary_header *header;
	struct bpf_binary_header *ro_header;
	struct bpf_prog *tmp, *orig_prog = prog;

	/*
	 * If BPF JIT was not enabled then we must fall back to
	 * the interpreter.
	 */
	if (!prog->jit_requested)
		return orig_prog;

	tmp = bpf_jit_blind_constants(prog);
	/*
	 * If blinding was requested and we failed during blinding,
	 * we must fall back to the interpreter. Otherwise, we save
	 * the new JITed code.
	 */
	if (IS_ERR(tmp))
		return orig_prog;

	if (tmp != prog) {
		tmp_blinded = true;
		prog = tmp;
	}
		return prog;

	jit_data = prog->aux->jit_data;
	if (!jit_data) {
		jit_data = kzalloc_obj(*jit_data);
		if (!jit_data) {
			prog = orig_prog;
			goto out;
		}
		if (!jit_data)
			return prog;
		prog->aux->jit_data = jit_data;
	}
	if (jit_data->ctx.offset) {
@@ -1978,17 +1961,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);

	ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
	if (ctx.offset == NULL) {
		prog = orig_prog;
	if (ctx.offset == NULL)
		goto out_offset;
	}

	/* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
	build_prologue(&ctx);
	if (build_body(&ctx, extra_pass)) {
		prog = orig_prog;
	if (build_body(&ctx, extra_pass))
		goto out_offset;
	}
	ctx.epilogue_offset = ctx.idx;
	build_epilogue(&ctx);

@@ -2004,10 +1983,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	/* Now we know the size of the structure to make */
	ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32),
					      &header, &image_ptr, jit_fill_hole);
	if (!ro_header) {
		prog = orig_prog;
	if (!ro_header)
		goto out_offset;
	}

	/* 2. Now, the actual pass to generate final JIT code */
	/*
@@ -2027,17 +2004,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
	ctx.num_exentries = 0;

	build_prologue(&ctx);
	if (build_body(&ctx, extra_pass)) {
		prog = orig_prog;
	if (build_body(&ctx, extra_pass))
		goto out_free;
	}
	build_epilogue(&ctx);

	/* 3. Extra pass to validate JITed code */
	if (validate_ctx(&ctx)) {
		prog = orig_prog;
	if (validate_ctx(&ctx))
		goto out_free;
	}

	/* And we're done */
	if (bpf_jit_enable > 1)
@@ -2050,9 +2023,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
			goto out_free;
		}
		if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {
			/* ro_header has been freed */
			/* ro_header and header have been freed */
			ro_header = NULL;
			prog = orig_prog;
			header = NULL;
			goto out_free;
		}
		/*
@@ -2084,13 +2057,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
		prog->aux->jit_data = NULL;
	}

out:
	if (tmp_blinded)
		bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog);

	return prog;

out_free:
	if (extra_pass) {
		prog->bpf_func = NULL;
		prog->jited = 0;
		prog->jited_len = 0;
	}

	if (header) {
		bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size));
		bpf_jit_binary_pack_free(ro_header, header);
Loading