Commit 35f301dd authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf 

Pull bpf fixes from Daniel Borkmann:

 - Fix a bug in the BPF verifier to track changes to packet data
   property for global functions (Eduard Zingerman)

 - Fix a theoretical BPF prog_array use-after-free in RCU handling of
   __uprobe_perf_func (Jann Horn)

 - Fix BPF tracing to have an explicit list of tracepoints and their
   arguments which need to be annotated as PTR_MAYBE_NULL (Kumar
   Kartikeya Dwivedi)

 - Fix a logic bug in the bpf_remove_insns code where a potential error
   would have been wrongly propagated (Anton Protopopov)

 - Avoid deadlock scenarios caused by nested kprobe and fentry BPF
   programs (Priya Bala Govindasamy)

 - Fix a bug in BPF verifier which was missing a size check for
   BTF-based context access (Kumar Kartikeya Dwivedi)

 - Fix a crash found by syzbot through an invalid BPF prog_array access
   in perf_event_detach_bpf_prog (Jiri Olsa)

 - Fix several BPF sockmap bugs including a race causing a refcount
   imbalance upon element replace (Michal Luczaj)

 - Fix a use-after-free from mismatching BPF program/attachment RCU
   flavors (Jann Horn)

* tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (23 commits)
  bpf: Avoid deadlock caused by nested kprobe and fentry bpf programs
  selftests/bpf: Add tests for raw_tp NULL args
  bpf: Augment raw_tp arguments with PTR_MAYBE_NULL
  bpf: Revert "bpf: Mark raw_tp arguments with PTR_MAYBE_NULL"
  selftests/bpf: Add test for narrow ctx load for pointer args
  bpf: Check size for BTF-based ctx access of pointer members
  selftests/bpf: extend changes_pkt_data with cases w/o subprograms
  bpf: fix null dereference when computing changes_pkt_data of prog w/o subprogs
  bpf: Fix theoretical prog_array UAF in __uprobe_perf_func()
  bpf: fix potential error return
  selftests/bpf: validate that tail call invalidates packet pointers
  bpf: consider that tail calls invalidate packet pointers
  selftests/bpf: freplace tests for tracking of changes_packet_data
  bpf: check changes_pkt_data property for extension programs
  selftests/bpf: test for changing packet data from global functions
  bpf: track changes_pkt_data property for global functions
  bpf: refactor bpf_helper_changes_pkt_data to use helper number
  bpf: add find_containing_subprog() utility function
  bpf,perf: Fix invalid prog_array access in perf_event_detach_bpf_prog
  bpf: Fix UAF via mismatching bpf_prog/attachment RCU flavors
  ...
parents a0e3919a c83508da

include/linux/bpf.h

+6 −14
Original line number Diff line number Diff line
@@ -1527,6 +1527,7 @@ struct bpf_prog_aux { 
	bool is_extended; /* true if extended by freplace program */

	bool jits_use_priv_stack;

	bool priv_stack_requested;

	bool changes_pkt_data;

	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */

	struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */

	struct bpf_arena *arena;
@@ -2193,26 +2194,25 @@ bpf_prog_run_array(const struct bpf_prog_array *array, 
 * rcu-protected dynamically sized maps.

 */

static __always_inline u32

bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu,

bpf_prog_run_array_uprobe(const struct bpf_prog_array *array,

			  const void *ctx, bpf_prog_run_fn run_prog)

{

	const struct bpf_prog_array_item *item;

	const struct bpf_prog *prog;

	const struct bpf_prog_array *array;

	struct bpf_run_ctx *old_run_ctx;

	struct bpf_trace_run_ctx run_ctx;

	u32 ret = 1;



	might_fault();

	RCU_LOCKDEP_WARN(!rcu_read_lock_trace_held(), "no rcu lock held");



	if (unlikely(!array))

		return ret;



	rcu_read_lock_trace();

	migrate_disable();



	run_ctx.is_uprobe = true;



	array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held());

	if (unlikely(!array))

		goto out;

	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);

	item = &array->items[0];

	while ((prog = READ_ONCE(item->prog))) {
@@ -2227,9 +2227,7 @@ bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu, 
			rcu_read_unlock();

	}

	bpf_reset_run_ctx(old_run_ctx);

out:

	migrate_enable();

	rcu_read_unlock_trace();

	return ret;

}
@@ -3516,10 +3514,4 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog) 
	return prog->aux->func_idx != 0;

}



static inline bool bpf_prog_is_raw_tp(const struct bpf_prog *prog)

{

	return prog->type == BPF_PROG_TYPE_TRACING &&

	       prog->expected_attach_type == BPF_TRACE_RAW_TP;

}



#endif /* _LINUX_BPF_H */

include/linux/bpf_verifier.h

+1 −0
Original line number Diff line number Diff line
@@ -659,6 +659,7 @@ struct bpf_subprog_info { 
	bool args_cached: 1;

	/* true if bpf_fastcall stack region is used by functions that can't be inlined */

	bool keep_fastcall_stack: 1;

	bool changes_pkt_data: 1;



	enum priv_stack_mode priv_stack_mode;

	u8 arg_cnt;

include/linux/filter.h

+1 −1
Original line number Diff line number Diff line
@@ -1122,7 +1122,7 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); 
bool bpf_jit_supports_private_stack(void);

u64 bpf_arch_uaddress_limit(void);

void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);

bool bpf_helper_changes_pkt_data(void *func);

bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id);



static inline bool bpf_dump_raw_ok(const struct cred *cred)

{

kernel/bpf/Makefile

+6 −0
Original line number Diff line number Diff line
@@ -53,3 +53,9 @@ obj-$(CONFIG_BPF_SYSCALL) += relo_core.o 
obj-$(CONFIG_BPF_SYSCALL) += btf_iter.o

obj-$(CONFIG_BPF_SYSCALL) += btf_relocate.o

obj-$(CONFIG_BPF_SYSCALL) += kmem_cache_iter.o



CFLAGS_REMOVE_percpu_freelist.o = $(CC_FLAGS_FTRACE)

CFLAGS_REMOVE_bpf_lru_list.o = $(CC_FLAGS_FTRACE)

CFLAGS_REMOVE_queue_stack_maps.o = $(CC_FLAGS_FTRACE)

CFLAGS_REMOVE_lpm_trie.o = $(CC_FLAGS_FTRACE)

CFLAGS_REMOVE_ringbuf.o = $(CC_FLAGS_FTRACE)

kernel/bpf/btf.c

+145 −4
Original line number Diff line number Diff line
@@ -6439,6 +6439,101 @@ int btf_ctx_arg_offset(const struct btf *btf, const struct btf_type *func_proto, 
	return off;

}



struct bpf_raw_tp_null_args {

	const char *func;

	u64 mask;

};



static const struct bpf_raw_tp_null_args raw_tp_null_args[] = {

	/* sched */

	{ "sched_pi_setprio", 0x10 },

	/* ... from sched_numa_pair_template event class */

	{ "sched_stick_numa", 0x100 },

	{ "sched_swap_numa", 0x100 },

	/* afs */

	{ "afs_make_fs_call", 0x10 },

	{ "afs_make_fs_calli", 0x10 },

	{ "afs_make_fs_call1", 0x10 },

	{ "afs_make_fs_call2", 0x10 },

	{ "afs_protocol_error", 0x1 },

	{ "afs_flock_ev", 0x10 },

	/* cachefiles */

	{ "cachefiles_lookup", 0x1 | 0x200 },

	{ "cachefiles_unlink", 0x1 },

	{ "cachefiles_rename", 0x1 },

	{ "cachefiles_prep_read", 0x1 },

	{ "cachefiles_mark_active", 0x1 },

	{ "cachefiles_mark_failed", 0x1 },

	{ "cachefiles_mark_inactive", 0x1 },

	{ "cachefiles_vfs_error", 0x1 },

	{ "cachefiles_io_error", 0x1 },

	{ "cachefiles_ondemand_open", 0x1 },

	{ "cachefiles_ondemand_copen", 0x1 },

	{ "cachefiles_ondemand_close", 0x1 },

	{ "cachefiles_ondemand_read", 0x1 },

	{ "cachefiles_ondemand_cread", 0x1 },

	{ "cachefiles_ondemand_fd_write", 0x1 },

	{ "cachefiles_ondemand_fd_release", 0x1 },

	/* ext4, from ext4__mballoc event class */

	{ "ext4_mballoc_discard", 0x10 },

	{ "ext4_mballoc_free", 0x10 },

	/* fib */

	{ "fib_table_lookup", 0x100 },

	/* filelock */

	/* ... from filelock_lock event class */

	{ "posix_lock_inode", 0x10 },

	{ "fcntl_setlk", 0x10 },

	{ "locks_remove_posix", 0x10 },

	{ "flock_lock_inode", 0x10 },

	/* ... from filelock_lease event class */

	{ "break_lease_noblock", 0x10 },

	{ "break_lease_block", 0x10 },

	{ "break_lease_unblock", 0x10 },

	{ "generic_delete_lease", 0x10 },

	{ "time_out_leases", 0x10 },

	/* host1x */

	{ "host1x_cdma_push_gather", 0x10000 },

	/* huge_memory */

	{ "mm_khugepaged_scan_pmd", 0x10 },

	{ "mm_collapse_huge_page_isolate", 0x1 },

	{ "mm_khugepaged_scan_file", 0x10 },

	{ "mm_khugepaged_collapse_file", 0x10 },

	/* kmem */

	{ "mm_page_alloc", 0x1 },

	{ "mm_page_pcpu_drain", 0x1 },

	/* .. from mm_page event class */

	{ "mm_page_alloc_zone_locked", 0x1 },

	/* netfs */

	{ "netfs_failure", 0x10 },

	/* power */

	{ "device_pm_callback_start", 0x10 },

	/* qdisc */

	{ "qdisc_dequeue", 0x1000 },

	/* rxrpc */

	{ "rxrpc_recvdata", 0x1 },

	{ "rxrpc_resend", 0x10 },

	/* sunrpc */

	{ "xs_stream_read_data", 0x1 },

	/* ... from xprt_cong_event event class */

	{ "xprt_reserve_cong", 0x10 },

	{ "xprt_release_cong", 0x10 },

	{ "xprt_get_cong", 0x10 },

	{ "xprt_put_cong", 0x10 },

	/* tcp */

	{ "tcp_send_reset", 0x11 },

	/* tegra_apb_dma */

	{ "tegra_dma_tx_status", 0x100 },

	/* timer_migration */

	{ "tmigr_update_events", 0x1 },

	/* writeback, from writeback_folio_template event class */

	{ "writeback_dirty_folio", 0x10 },

	{ "folio_wait_writeback", 0x10 },

	/* rdma */

	{ "mr_integ_alloc", 0x2000 },

	/* bpf_testmod */

	{ "bpf_testmod_test_read", 0x0 },

};



bool btf_ctx_access(int off, int size, enum bpf_access_type type,

		    const struct bpf_prog *prog,

		    struct bpf_insn_access_aux *info)
@@ -6449,6 +6544,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, 
	const char *tname = prog->aux->attach_func_name;

	struct bpf_verifier_log *log = info->log;

	const struct btf_param *args;

	bool ptr_err_raw_tp = false;

	const char *tag_value;

	u32 nr_args, arg;

	int i, ret;
@@ -6543,6 +6639,12 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, 
		return false;

	}



	if (size != sizeof(u64)) {

		bpf_log(log, "func '%s' size %d must be 8\n",

			tname, size);

		return false;

	}



	/* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */

	for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {

		const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
@@ -6588,11 +6690,41 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, 
	if (prog_args_trusted(prog))

		info->reg_type |= PTR_TRUSTED;



	/* Raw tracepoint arguments always get marked as maybe NULL */

	if (bpf_prog_is_raw_tp(prog))

	if (btf_param_match_suffix(btf, &args[arg], "__nullable"))

		info->reg_type |= PTR_MAYBE_NULL;



	if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {

		struct btf *btf = prog->aux->attach_btf;

		const struct btf_type *t;

		const char *tname;



		/* BTF lookups cannot fail, return false on error */

		t = btf_type_by_id(btf, prog->aux->attach_btf_id);

		if (!t)

			return false;

		tname = btf_name_by_offset(btf, t->name_off);

		if (!tname)

			return false;

		/* Checked by bpf_check_attach_target */

		tname += sizeof("btf_trace_") - 1;

		for (i = 0; i < ARRAY_SIZE(raw_tp_null_args); i++) {

			/* Is this a func with potential NULL args? */

			if (strcmp(tname, raw_tp_null_args[i].func))

				continue;

			if (raw_tp_null_args[i].mask & (0x1 << (arg * 4)))

				info->reg_type |= PTR_MAYBE_NULL;

	else if (btf_param_match_suffix(btf, &args[arg], "__nullable"))

			/* Is the current arg IS_ERR? */

			if (raw_tp_null_args[i].mask & (0x2 << (arg * 4)))

				ptr_err_raw_tp = true;

			break;

		}

		/* If we don't know NULL-ness specification and the tracepoint

		 * is coming from a loadable module, be conservative and mark

		 * argument as PTR_MAYBE_NULL.

		 */

		if (i == ARRAY_SIZE(raw_tp_null_args) && btf_is_module(btf))

			info->reg_type |= PTR_MAYBE_NULL;

	}



	if (tgt_prog) {

		enum bpf_prog_type tgt_type;
@@ -6638,6 +6770,15 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, 
	bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n",

		tname, arg, info->btf_id, btf_type_str(t),

		__btf_name_by_offset(btf, t->name_off));



	/* Perform all checks on the validity of type for this argument, but if

	 * we know it can be IS_ERR at runtime, scrub pointer type and mark as

	 * scalar.

	 */

	if (ptr_err_raw_tp) {

		bpf_log(log, "marking pointer arg%d as scalar as it may encode error", arg);

		info->reg_type = SCALAR_VALUE;

	}

	return true;

}

EXPORT_SYMBOL_GPL(btf_ctx_access);