Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf (4f13d0da) · Commits · git / linux-net

arch/riscv/net/bpf_jit_comp64.c

+77 −20

Original line number	Diff line number	Diff line
		@@ -926,6 +926,14 @@ static void restore_stack_args(int nr_stack_args, int args_off, int stk_arg_off,
		}
		}

		static void emit_store_stack_imm64(u8 reg, int stack_off, u64 imm64,
		struct rv_jit_context *ctx)
		{
		/* Load imm64 into reg and store it at [FP + stack_off]. */
		emit_imm(reg, (s64)imm64, ctx);
		emit_sd(RV_REG_FP, stack_off, reg, ctx);
		}

		static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off,
		int run_ctx_off, bool save_ret, struct rv_jit_context *ctx)
		{
		@@ -933,12 +941,10 @@ static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_of
		struct bpf_prog *p = l->link.prog;
		int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);

		if (l->cookie) {
		emit_imm(RV_REG_T1, l->cookie, ctx);
		emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx);
		} else {
		if (l->cookie)
		emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, l->cookie, ctx);
		else
		emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx);
		}

		/* arg1: prog */
		emit_imm(RV_REG_A0, (const s64)p, ctx);
		@@ -990,6 +996,29 @@ static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_of
		return ret;
		}

		static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off,
		int run_ctx_off, int func_meta_off, bool save_ret, u64 func_meta,
		int cookie_off, struct rv_jit_context *ctx)
		{
		int i, cur_cookie = (cookie_off - args_off) / 8;

		for (i = 0; i < tl->nr_links; i++) {
		int err;

		if (bpf_prog_calls_session_cookie(tl->links[i])) {
		u64 meta = func_meta \| ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT);

		emit_store_stack_imm64(RV_REG_T1, -func_meta_off, meta, ctx);
		cur_cookie--;
		}
		err = invoke_bpf_prog(tl->links[i], args_off, retval_off, run_ctx_off,
		save_ret, ctx);
		if (err)
		return err;
		}
		return 0;
		}

		static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
		const struct btf_func_model *m,
		struct bpf_tramp_links *tlinks,
		@@ -999,13 +1028,15 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
		int i, ret, offset;
		int *branches_off = NULL;
		int stack_size = 0, nr_arg_slots = 0;
		int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off, stk_arg_off;
		int retval_off, args_off, func_meta_off, ip_off, run_ctx_off, sreg_off, stk_arg_off;
		int cookie_off, cookie_cnt;
		struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
		struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
		struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
		bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
		void *orig_call = func_addr;
		bool save_ret;
		u64 func_meta;
		u32 insn;

		/* Two types of generated trampoline stack layout:
		@@ -1036,10 +1067,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
		* [ ... ]
		* FP - args_off [ arg1 ]
		*
		* FP - nregs_off [ regs count ]
		* FP - func_meta_off [ regs count, etc ]
		*
		* FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG
		*
		* [ stack cookie N ]
		* [ ... ]
		* FP - cookie_off [ stack cookie 1 ]
		*
		* FP - run_ctx_off [ bpf_tramp_run_ctx ]
		*
		* FP - sreg_off [ callee saved reg ]
		@@ -1071,14 +1106,20 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
		stack_size += nr_arg_slots * 8;
		args_off = stack_size;

		/* function metadata, such as regs count */
		stack_size += 8;
		nregs_off = stack_size;
		func_meta_off = stack_size;

		if (flags & BPF_TRAMP_F_IP_ARG) {
		stack_size += 8;
		ip_off = stack_size;
		}

		cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
		/* room for session cookies */
		stack_size += cookie_cnt * 8;
		cookie_off = stack_size;

		stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
		run_ctx_off = stack_size;

		@@ -1123,16 +1164,22 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
		emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx);

		/* store ip address of the traced function */
		if (flags & BPF_TRAMP_F_IP_ARG) {
		emit_imm(RV_REG_T1, (const s64)func_addr, ctx);
		emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx);
		}
		if (flags & BPF_TRAMP_F_IP_ARG)
		emit_store_stack_imm64(RV_REG_T1, -ip_off, (u64)func_addr, ctx);

		emit_li(RV_REG_T1, nr_arg_slots, ctx);
		emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx);
		func_meta = nr_arg_slots;
		emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx);

		store_args(nr_arg_slots, args_off, ctx);

		if (bpf_fsession_cnt(tlinks)) {
		/* clear all session cookies' value */
		for (i = 0; i < cookie_cnt; i++)
		emit_sd(RV_REG_FP, -cookie_off + 8 * i, RV_REG_ZERO, ctx);
		/* clear return value to make sure fentry always get 0 */
		emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx);
		}

		if (flags & BPF_TRAMP_F_CALL_ORIG) {
		emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx);
		ret = emit_call((const u64)__bpf_tramp_enter, true, ctx);
		@@ -1140,9 +1187,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
		return ret;
		}

		for (i = 0; i < fentry->nr_links; i++) {
		ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off,
		flags & BPF_TRAMP_F_RET_FENTRY_RET, ctx);
		if (fentry->nr_links) {
		ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off,
		flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off, ctx);
		if (ret)
		return ret;
		}
		@@ -1189,9 +1236,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
		(u32 )(ctx->insns + branches_off[i]) = insn;
		}

		for (i = 0; i < fexit->nr_links; i++) {
		ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off,
		run_ctx_off, false, ctx);
		/* set "is_return" flag for fsession */
		func_meta \|= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
		if (bpf_fsession_cnt(tlinks))
		emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx);

		if (fexit->nr_links) {
		ret = invoke_bpf(fexit, args_off, retval_off, run_ctx_off, func_meta_off,
		false, func_meta, cookie_off, ctx);
		if (ret)
		goto out;
		}
		@@ -2091,3 +2143,8 @@ bool bpf_jit_inlines_helper_call(s32 imm)
		return false;
		}
		}

		bool bpf_jit_supports_fsession(void)
		{
		return true;
		}

kernel/bpf/verifier.c

+25 −23

Original line number	Diff line number	Diff line
		@@ -21333,29 +21333,29 @@ static int find_btf_percpu_datasec(struct btf *btf)
		}

		/*
		* Add btf to the used_btfs array and return the index. (If the btf was
		* already added, then just return the index.) Upon successful insertion
		* increase btf refcnt, and, if present, also refcount the corresponding
		* kernel module.
		* Add btf to the env->used_btfs array. If needed, refcount the
		* corresponding kernel module. To simplify caller's logic
		* in case of error or if btf was added before the function
		* decreases the btf refcount.
		*/
		static int __add_used_btf(struct bpf_verifier_env env, struct btf btf)
		{
		struct btf_mod_pair *btf_mod;
		int ret = 0;
		int i;

		/* check whether we recorded this BTF (and maybe module) already */
		for (i = 0; i < env->used_btf_cnt; i++)
		if (env->used_btfs[i].btf == btf)
		return i;
		goto ret_put;

		if (env->used_btf_cnt >= MAX_USED_BTFS) {
		verbose(env, "The total number of btfs per program has reached the limit of %u\n",
		MAX_USED_BTFS);
		return -E2BIG;
		ret = -E2BIG;
		goto ret_put;
		}

		btf_get(btf);

		btf_mod = &env->used_btfs[env->used_btf_cnt];
		btf_mod->btf = btf;
		btf_mod->module = NULL;
		@@ -21364,12 +21364,18 @@ static int __add_used_btf(struct bpf_verifier_env env, struct btf btf)
		if (btf_is_module(btf)) {
		btf_mod->module = btf_try_get_module(btf);
		if (!btf_mod->module) {
		btf_put(btf);
		return -ENXIO;
		ret = -ENXIO;
		goto ret_put;
		}
		}

		return env->used_btf_cnt++;
		env->used_btf_cnt++;
		return 0;

		ret_put:
		/* Either error or this BTF was already added */
		btf_put(btf);
		return ret;
		}

		/* replace pseudo btf_id with kernel symbol address */
		@@ -21466,9 +21472,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env,

		btf_fd = insn[1].imm;
		if (btf_fd) {
		CLASS(fd, f)(btf_fd);

		btf = __btf_get_by_fd(f);
		btf = btf_get_by_fd(btf_fd);
		if (IS_ERR(btf)) {
		verbose(env, "invalid module BTF object FD specified.\n");
		return -EINVAL;
		@@ -21478,17 +21482,17 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env,
		verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
		return -EINVAL;
		}
		btf_get(btf_vmlinux);
		btf = btf_vmlinux;
		}

		err = __check_pseudo_btf_id(env, insn, aux, btf);
		if (err)
		if (err) {
		btf_put(btf);
		return err;
		}

		err = __add_used_btf(env, btf);
		if (err < 0)
		return err;
		return 0;
		return __add_used_btf(env, btf);
		}

		static bool is_tracing_prog_type(enum bpf_prog_type type)
		@@ -25370,10 +25374,8 @@ static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd)

		btf = __btf_get_by_fd(f);
		if (!IS_ERR(btf)) {
		err = __add_used_btf(env, btf);
		if (err < 0)
		return err;
		return 0;
		btf_get(btf);
		return __add_used_btf(env, btf);
		}

		verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd);

tools/bpf/bpftool/net.c

+4 −1

Original line number	Diff line number	Diff line
		@@ -156,7 +156,7 @@ static int netlink_recv(int sock, __u32 nl_pid, __u32 seq,
		bool multipart = true;
		struct nlmsgerr *err;
		struct nlmsghdr *nh;
		char buf[4096];
		char buf[8192];
		int len, ret;

		while (multipart) {
		@@ -201,6 +201,9 @@ static int netlink_recv(int sock, __u32 nl_pid, __u32 seq,
		return ret;
		}
		}

		if (len)
		p_err("Invalid message or trailing data in Netlink response: %d bytes left", len);
		}
		ret = 0;
		done:

tools/lib/bpf/bpf_helpers.h

+0 −3

Original line number	Diff line number	Diff line
		@@ -315,9 +315,6 @@ enum libbpf_tristate {
		___param, sizeof(___param)); \
		})

		extern int bpf_stream_vprintk(int stream_id, const char fmt__str, const void args,
		__u32 len__sz) __weak __ksym;

		#define bpf_stream_printk(stream_id, fmt, args...) \
		({ \
		static const char ___fmt[] = fmt; \

tools/lib/bpf/features.c

+65 −0

Original line number	Diff line number	Diff line
		@@ -506,6 +506,68 @@ static int probe_kern_arg_ctx_tag(int token_fd)
		return probe_fd(prog_fd);
		}

		static int probe_ldimm64_full_range_off(int token_fd)
		{
		char log_buf[1024];
		int prog_fd, map_fd;
		int ret;
		LIBBPF_OPTS(bpf_map_create_opts, map_opts,
		.token_fd = token_fd,
		.map_flags = token_fd ? BPF_F_TOKEN_FD : 0,
		);
		LIBBPF_OPTS(bpf_prog_load_opts, prog_opts,
		.token_fd = token_fd,
		.prog_flags = token_fd ? BPF_F_TOKEN_FD : 0,
		.log_buf = log_buf,
		.log_size = sizeof(log_buf),
		);
		struct bpf_insn insns[] = {
		BPF_LD_MAP_VALUE(BPF_REG_1, 0, 1UL << 30),
		BPF_EXIT_INSN(),
		};
		int insn_cnt = ARRAY_SIZE(insns);

		map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr", sizeof(int), 1, 1, &map_opts);
		if (map_fd < 0) {
		ret = -errno;
		pr_warn("Error in %s(): %s. Couldn't create simple array map.\n",
		__func__, errstr(ret));
		return ret;
		}
		insns[0].imm = map_fd;

		log_buf[0] = '\0';
		prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "global_reloc", "GPL", insns, insn_cnt, &prog_opts);
		ret = -errno;

		close(map_fd);

		if (prog_fd >= 0) {
		pr_warn("Error in %s(): Program loading unexpectedly succeeded.\n", __func__);
		close(prog_fd);
		return -EINVAL;
		}

		/*
		* Feature is allowed if we're not failing with the error message
		* "direct value offset of %u is not allowed" removed in
		* 12a1fe6e12db ("bpf/verifier: Do not limit maximum direct offset into arena map").
		* We should instead fail with "invalid access to map value pointer".
		* Ensure we match with one of the two and we're not failing with a
		* different, unexpected message.
		*/
		if (strstr(log_buf, "direct value offset of"))
		return 0;

		if (!strstr(log_buf, "invalid access to map value pointer")) {
		pr_warn("Error in %s(): Program unexpectedly failed with message: %s.\n",
		__func__, log_buf);
		return ret;
		}

		return 1;
		}

		typedef int (feature_probe_fn)(int / token_fd */);

		static struct kern_feature_cache feature_cache;
		@@ -581,6 +643,9 @@ static struct kern_feature_desc {
		[FEAT_BTF_QMARK_DATASEC] = {
		"BTF DATASEC names starting from '?'", probe_kern_btf_qmark_datasec,
		},
		[FEAT_LDIMM64_FULL_RANGE_OFF] = {
		"full range LDIMM64 support", probe_ldimm64_full_range_off,
		},
		};

		bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id)