Commit d303caf5 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf 

Pull bpf fixes from Alexei Starovoitov:

 - Replace bpf_map_kmalloc_node() with kmalloc_nolock() to fix kmemleak
   imbalance in tracking of bpf_async_cb structures (Alexei Starovoitov)

 - Make selftests/bpf arg_parsing.c more robust to errors (Andrii
   Nakryiko)

 - Fix redefinition of 'off' as different kind of symbol when I40E
   driver is builtin (Brahmajit Das)

 - Do not disable preemption in bpf_test_run (Sahil Chandna)

 - Fix memory leak in __lookup_instance error path (Shardul Bankar)

 - Ensure test data is flushed to disk before reading it (Xing Guo)

* tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf:
  selftests/bpf: Fix redefinition of 'off' as different kind of symbol
  bpf: Do not disable preemption in bpf_test_run().
  bpf: Fix memory leak in __lookup_instance error path
  selftests: arg_parsing: Ensure data is flushed to disk before reading.
  bpf: Replace bpf_map_kmalloc_node() with kmalloc_nolock() to allocate bpf_async_cb structures.
  selftests/bpf: make arg_parsing.c more robust to crashes
  bpf: test_run: Fix ctx leak in bpf_prog_test_run_xdp error path
parents 847f242f a1e83d4c

include/linux/bpf.h

+4 −0
Original line number Diff line number Diff line
@@ -2499,6 +2499,8 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid, 
#ifdef CONFIG_MEMCG

void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,

			   int node);

void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags,

			     int node);

void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags);

void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,

		       gfp_t flags);
@@ -2511,6 +2513,8 @@ void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, 
 */

#define bpf_map_kmalloc_node(_map, _size, _flags, _node)	\

		kmalloc_node(_size, _flags, _node)

#define bpf_map_kmalloc_nolock(_map, _size, _flags, _node)	\

		kmalloc_nolock(_size, _flags, _node)

#define bpf_map_kzalloc(_map, _size, _flags)			\

		kzalloc(_size, _flags)

#define bpf_map_kvcalloc(_map, _n, _size, _flags)		\

kernel/bpf/helpers.c

+14 −11
Original line number Diff line number Diff line
@@ -1215,13 +1215,20 @@ static void bpf_wq_work(struct work_struct *work) 
	rcu_read_unlock_trace();

}



static void bpf_async_cb_rcu_free(struct rcu_head *rcu)

{

	struct bpf_async_cb *cb = container_of(rcu, struct bpf_async_cb, rcu);



	kfree_nolock(cb);

}



static void bpf_wq_delete_work(struct work_struct *work)

{

	struct bpf_work *w = container_of(work, struct bpf_work, delete_work);



	cancel_work_sync(&w->work);



	kfree_rcu(w, cb.rcu);

	call_rcu(&w->cb.rcu, bpf_async_cb_rcu_free);

}



static void bpf_timer_delete_work(struct work_struct *work)
@@ -1230,13 +1237,13 @@ static void bpf_timer_delete_work(struct work_struct *work) 


	/* Cancel the timer and wait for callback to complete if it was running.

	 * If hrtimer_cancel() can be safely called it's safe to call

	 * kfree_rcu(t) right after for both preallocated and non-preallocated

	 * call_rcu() right after for both preallocated and non-preallocated

	 * maps.  The async->cb = NULL was already done and no code path can see

	 * address 't' anymore. Timer if armed for existing bpf_hrtimer before

	 * bpf_timer_cancel_and_free will have been cancelled.

	 */

	hrtimer_cancel(&t->timer);

	kfree_rcu(t, cb.rcu);

	call_rcu(&t->cb.rcu, bpf_async_cb_rcu_free);

}



static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u64 flags,
@@ -1270,11 +1277,7 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u 
		goto out;

	}



	/* Allocate via bpf_map_kmalloc_node() for memcg accounting. Until

	 * kmalloc_nolock() is available, avoid locking issues by using

	 * __GFP_HIGH (GFP_ATOMIC & ~__GFP_RECLAIM).

	 */

	cb = bpf_map_kmalloc_node(map, size, __GFP_HIGH, map->numa_node);

	cb = bpf_map_kmalloc_nolock(map, size, 0, map->numa_node);

	if (!cb) {

		ret = -ENOMEM;

		goto out;
@@ -1315,7 +1318,7 @@ static int __bpf_async_init(struct bpf_async_kern *async, struct bpf_map *map, u 
		 * or pinned in bpffs.

		 */

		WRITE_ONCE(async->cb, NULL);

		kfree(cb);

		kfree_nolock(cb);

		ret = -EPERM;

	}

out:
@@ -1580,7 +1583,7 @@ void bpf_timer_cancel_and_free(void *val) 
	 * timer _before_ calling us, such that failing to cancel it here will

	 * cause it to possibly use struct hrtimer after freeing bpf_hrtimer.

	 * Therefore, we _need_ to cancel any outstanding timers before we do

	 * kfree_rcu, even though no more timers can be armed.

	 * call_rcu, even though no more timers can be armed.

	 *

	 * Moreover, we need to schedule work even if timer does not belong to

	 * the calling callback_fn, as on two different CPUs, we can end up in a
@@ -1607,7 +1610,7 @@ void bpf_timer_cancel_and_free(void *val) 
		 * completion.

		 */

		if (hrtimer_try_to_cancel(&t->timer) >= 0)

			kfree_rcu(t, cb.rcu);

			call_rcu(&t->cb.rcu, bpf_async_cb_rcu_free);

		else

			queue_work(system_dfl_wq, &t->cb.delete_work);

	} else {

kernel/bpf/liveness.c

+3 −1
Original line number Diff line number Diff line
@@ -195,8 +195,10 @@ static struct func_instance *__lookup_instance(struct bpf_verifier_env *env, 
		return ERR_PTR(-ENOMEM);

	result->must_write_set = kvcalloc(subprog_sz, sizeof(*result->must_write_set),

					  GFP_KERNEL_ACCOUNT);

	if (!result->must_write_set)

	if (!result->must_write_set) {

		kvfree(result);

		return ERR_PTR(-ENOMEM);

	}

	memcpy(&result->callchain, callchain, sizeof(*callchain));

	result->insn_cnt = subprog_sz;

	hash_add(liveness->func_instances, &result->hl_node, key);

kernel/bpf/syscall.c

+15 −0
Original line number Diff line number Diff line
@@ -520,6 +520,21 @@ void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 
	return ptr;

}



void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags,

			     int node)

{

	struct mem_cgroup *memcg, *old_memcg;

	void *ptr;



	memcg = bpf_map_get_memcg(map);

	old_memcg = set_active_memcg(memcg);

	ptr = kmalloc_nolock(size, flags | __GFP_ACCOUNT, node);

	set_active_memcg(old_memcg);

	mem_cgroup_put(memcg);



	return ptr;

}



void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)

{

	struct mem_cgroup *memcg, *old_memcg;

net/bpf/test_run.c

+7 −18
Original line number Diff line number Diff line
@@ -29,7 +29,6 @@ 
#include <trace/events/bpf_test_run.h>



struct bpf_test_timer {

	enum { NO_PREEMPT, NO_MIGRATE } mode;

	u32 i;

	u64 time_start, time_spent;

};
@@ -37,12 +36,7 @@ struct bpf_test_timer { 
static void bpf_test_timer_enter(struct bpf_test_timer *t)

	__acquires(rcu)

{

	rcu_read_lock();

	if (t->mode == NO_PREEMPT)

		preempt_disable();

	else

		migrate_disable();



	rcu_read_lock_dont_migrate();

	t->time_start = ktime_get_ns();

}
@@ -50,12 +44,7 @@ static void bpf_test_timer_leave(struct bpf_test_timer *t) 
	__releases(rcu)

{

	t->time_start = 0;



	if (t->mode == NO_PREEMPT)

		preempt_enable();

	else

		migrate_enable();

	rcu_read_unlock();

	rcu_read_unlock_migrate();

}



static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations,
@@ -374,7 +363,7 @@ static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx, 


{

	struct xdp_test_data xdp = { .batch_size = batch_size };

	struct bpf_test_timer t = { .mode = NO_MIGRATE };

	struct bpf_test_timer t = {};

	int ret;



	if (!repeat)
@@ -404,7 +393,7 @@ static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, 
	struct bpf_prog_array_item item = {.prog = prog};

	struct bpf_run_ctx *old_ctx;

	struct bpf_cg_run_ctx run_ctx;

	struct bpf_test_timer t = { NO_MIGRATE };

	struct bpf_test_timer t = {};

	enum bpf_cgroup_storage_type stype;

	int ret;
@@ -1269,7 +1258,7 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 
		goto free_ctx;



	if (kattr->test.data_size_in - meta_sz < ETH_HLEN)

		return -EINVAL;

		goto free_ctx;



	data = bpf_test_init(kattr, linear_sz, max_linear_sz, headroom, tailroom);

	if (IS_ERR(data)) {
@@ -1377,7 +1366,7 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 
				     const union bpf_attr *kattr,

				     union bpf_attr __user *uattr)

{

	struct bpf_test_timer t = { NO_PREEMPT };

	struct bpf_test_timer t = {};

	u32 size = kattr->test.data_size_in;

	struct bpf_flow_dissector ctx = {};

	u32 repeat = kattr->test.repeat;
@@ -1445,7 +1434,7 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 
int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,

				union bpf_attr __user *uattr)

{

	struct bpf_test_timer t = { NO_PREEMPT };

	struct bpf_test_timer t = {};

	struct bpf_prog_array *progs = NULL;

	struct bpf_sk_lookup_kern ctx = {};

	u32 repeat = kattr->test.repeat;