Commit 246331e3 authored by Benjamin Tissoires's avatar Benjamin Tissoires Committed by Alexei Starovoitov
Browse files

bpf: allow struct bpf_wq to be embedded in arraymaps and hashmaps 



Currently bpf_wq_cancel_and_free() is just a placeholder as there is
no memory allocation for bpf_wq just yet.

Again, duplication of the bpf_timer approach

Signed-off-by: default avatarBenjamin Tissoires <bentiss@kernel.org>
Link: https://lore.kernel.org/r/20240420-bpf_wq-v2-9-6c986a5a741f@kernel.org


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent d940c9b9

include/linux/bpf.h

+2 −0
Original line number Diff line number Diff line
@@ -534,6 +534,7 @@ static inline void zero_map_value(struct bpf_map *map, void *dst) 
void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,

			   bool lock_src);

void bpf_timer_cancel_and_free(void *timer);

void bpf_wq_cancel_and_free(void *timer);

void bpf_list_head_free(const struct btf_field *field, void *list_head,

			struct bpf_spin_lock *spin_lock);

void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
@@ -2204,6 +2205,7 @@ void bpf_map_free_record(struct bpf_map *map); 
struct btf_record *btf_record_dup(const struct btf_record *rec);

bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);

void bpf_obj_free_timer(const struct btf_record *rec, void *obj);

void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);

void bpf_obj_free_fields(const struct btf_record *rec, void *obj);

void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);

kernel/bpf/arraymap.c

+11 −7
Original line number Diff line number Diff line
@@ -428,17 +428,21 @@ static void *array_map_vmalloc_addr(struct bpf_array *array) 
	return (void *)round_down((unsigned long)array, PAGE_SIZE);

}



static void array_map_free_timers(struct bpf_map *map)

static void array_map_free_timers_wq(struct bpf_map *map)

{

	struct bpf_array *array = container_of(map, struct bpf_array, map);

	int i;



	/* We don't reset or free fields other than timer on uref dropping to zero. */

	if (!btf_record_has_field(map->record, BPF_TIMER))

		return;



	/* We don't reset or free fields other than timer and workqueue

	 * on uref dropping to zero.

	 */

	if (btf_record_has_field(map->record, BPF_TIMER))

		for (i = 0; i < array->map.max_entries; i++)

			bpf_obj_free_timer(map->record, array_map_elem_ptr(array, i));



	if (btf_record_has_field(map->record, BPF_WORKQUEUE))

		for (i = 0; i < array->map.max_entries; i++)

			bpf_obj_free_workqueue(map->record, array_map_elem_ptr(array, i));

}



/* Called when map->refcnt goes to zero, either from workqueue or from syscall */
@@ -782,7 +786,7 @@ const struct bpf_map_ops array_map_ops = { 
	.map_alloc = array_map_alloc,

	.map_free = array_map_free,

	.map_get_next_key = array_map_get_next_key,

	.map_release_uref = array_map_free_timers,

	.map_release_uref = array_map_free_timers_wq,

	.map_lookup_elem = array_map_lookup_elem,

	.map_update_elem = array_map_update_elem,

	.map_delete_elem = array_map_delete_elem,

kernel/bpf/hashtab.c

+43 −12
Original line number Diff line number Diff line
@@ -240,6 +240,26 @@ static void htab_free_prealloced_timers(struct bpf_htab *htab) 
	}

}



static void htab_free_prealloced_wq(struct bpf_htab *htab)

{

	u32 num_entries = htab->map.max_entries;

	int i;



	if (!btf_record_has_field(htab->map.record, BPF_WORKQUEUE))

		return;

	if (htab_has_extra_elems(htab))

		num_entries += num_possible_cpus();



	for (i = 0; i < num_entries; i++) {

		struct htab_elem *elem;



		elem = get_htab_elem(htab, i);

		bpf_obj_free_workqueue(htab->map.record,

				       elem->key + round_up(htab->map.key_size, 8));

		cond_resched();

	}

}



static void htab_free_prealloced_fields(struct bpf_htab *htab)

{

	u32 num_entries = htab->map.max_entries;
@@ -1495,7 +1515,7 @@ static void delete_all_elements(struct bpf_htab *htab) 
	migrate_enable();

}



static void htab_free_malloced_timers(struct bpf_htab *htab)

static void htab_free_malloced_timers_or_wq(struct bpf_htab *htab, bool is_timer)

{

	int i;
@@ -1507,25 +1527,36 @@ static void htab_free_malloced_timers(struct bpf_htab *htab) 


		hlist_nulls_for_each_entry(l, n, head, hash_node) {

			/* We only free timer on uref dropping to zero */

			bpf_obj_free_timer(htab->map.record, l->key + round_up(htab->map.key_size, 8));

			if (is_timer)

				bpf_obj_free_timer(htab->map.record,

						   l->key + round_up(htab->map.key_size, 8));

			else

				bpf_obj_free_workqueue(htab->map.record,

						       l->key + round_up(htab->map.key_size, 8));

		}

		cond_resched_rcu();

	}

	rcu_read_unlock();

}



static void htab_map_free_timers(struct bpf_map *map)

static void htab_map_free_timers_and_wq(struct bpf_map *map)

{

	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);



	/* We only free timer on uref dropping to zero */

	if (!btf_record_has_field(htab->map.record, BPF_TIMER))

		return;

	/* We only free timer and workqueue on uref dropping to zero */

	if (btf_record_has_field(htab->map.record, BPF_TIMER)) {

		if (!htab_is_prealloc(htab))

		htab_free_malloced_timers(htab);

			htab_free_malloced_timers_or_wq(htab, true);

		else

			htab_free_prealloced_timers(htab);

	}

	if (btf_record_has_field(htab->map.record, BPF_WORKQUEUE)) {

		if (!htab_is_prealloc(htab))

			htab_free_malloced_timers_or_wq(htab, false);

		else

			htab_free_prealloced_wq(htab);

	}

}



/* Called when map->refcnt goes to zero, either from workqueue or from syscall */

static void htab_map_free(struct bpf_map *map)
@@ -2260,7 +2291,7 @@ const struct bpf_map_ops htab_map_ops = { 
	.map_alloc = htab_map_alloc,

	.map_free = htab_map_free,

	.map_get_next_key = htab_map_get_next_key,

	.map_release_uref = htab_map_free_timers,

	.map_release_uref = htab_map_free_timers_and_wq,

	.map_lookup_elem = htab_map_lookup_elem,

	.map_lookup_and_delete_elem = htab_map_lookup_and_delete_elem,

	.map_update_elem = htab_map_update_elem,
@@ -2281,7 +2312,7 @@ const struct bpf_map_ops htab_lru_map_ops = { 
	.map_alloc = htab_map_alloc,

	.map_free = htab_map_free,

	.map_get_next_key = htab_map_get_next_key,

	.map_release_uref = htab_map_free_timers,

	.map_release_uref = htab_map_free_timers_and_wq,

	.map_lookup_elem = htab_lru_map_lookup_elem,

	.map_lookup_and_delete_elem = htab_lru_map_lookup_and_delete_elem,

	.map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys,

kernel/bpf/helpers.c

+8 −0
Original line number Diff line number Diff line
@@ -1468,6 +1468,14 @@ void bpf_timer_cancel_and_free(void *val) 
	kfree_rcu(t, cb.rcu);

}



/* This function is called by map_delete/update_elem for individual element and

 * by ops->map_release_uref when the user space reference to a map reaches zero.

 */

void bpf_wq_cancel_and_free(void *val)

{

	BTF_TYPE_EMIT(struct bpf_wq);

}



BPF_CALL_2(bpf_kptr_xchg, void *, map_value, void *, ptr)

{

	unsigned long *kptr = map_value;

kernel/bpf/syscall.c

+9 −0
Original line number Diff line number Diff line
@@ -661,6 +661,13 @@ void bpf_obj_free_timer(const struct btf_record *rec, void *obj) 
	bpf_timer_cancel_and_free(obj + rec->timer_off);

}



void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj)

{

	if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_WORKQUEUE)))

		return;

	bpf_wq_cancel_and_free(obj + rec->wq_off);

}



void bpf_obj_free_fields(const struct btf_record *rec, void *obj)

{

	const struct btf_field *fields;
@@ -682,6 +689,7 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) 
			bpf_timer_cancel_and_free(field_ptr);

			break;

		case BPF_WORKQUEUE:

			bpf_wq_cancel_and_free(field_ptr);

			break;

		case BPF_KPTR_UNREF:

			WRITE_ONCE(*(u64 *)field_ptr, 0);
@@ -1119,6 +1127,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, 
				}

				break;

			case BPF_TIMER:

			case BPF_WORKQUEUE:

				if (map->map_type != BPF_MAP_TYPE_HASH &&

				    map->map_type != BPF_MAP_TYPE_LRU_HASH &&

				    map->map_type != BPF_MAP_TYPE_ARRAY) {