Commit 136deea4 authored by Amery Hung's avatar Amery Hung Committed by Alexei Starovoitov
Browse files

bpf: Remove gfp_flags plumbing from bpf_local_storage_update() 



Remove the check that rejects sleepable BPF programs from doing
BPF_ANY/BPF_EXIST updates on local storage. This restriction was added
in commit b00fa38a ("bpf: Enable non-atomic allocations in local
storage") because kzalloc(GFP_KERNEL) could sleep inside
local_storage->lock. This is no longer a concern: all local storage
allocations now use kmalloc_nolock() which never sleeps.

In addition, since kmalloc_nolock() only accepts __GFP_ACCOUNT,
__GFP_ZERO and __GFP_NO_OBJ_EXT, the gfp_flags parameter plumbing from
bpf_*_storage_get() to bpf_local_storage_update() becomes dead code.
Remove gfp_flags from bpf_selem_alloc(), bpf_local_storage_alloc() and
bpf_local_storage_update(). Drop the hidden 5th argument from
bpf_*_storage_get helpers, and remove the verifier patching that
injected GFP_KERNEL/GFP_ATOMIC into the fifth argument.

Signed-off-by: default avatarAmery Hung <ameryhung@gmail.com>
Link: https://lore.kernel.org/r/20260411015419.114016-4-ameryhung@gmail.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 5063e775

include/linux/bpf_local_storage.h

+3 −4
Original line number Diff line number Diff line
@@ -188,7 +188,7 @@ int bpf_selem_link_map(struct bpf_local_storage_map *smap, 


struct bpf_local_storage_elem *

bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,

		bool swap_uptrs, gfp_t gfp_flags);

		bool swap_uptrs);



void bpf_selem_free(struct bpf_local_storage_elem *selem,

		    bool reuse_now);
@@ -196,12 +196,11 @@ void bpf_selem_free(struct bpf_local_storage_elem *selem, 
int

bpf_local_storage_alloc(void *owner,

			struct bpf_local_storage_map *smap,

			struct bpf_local_storage_elem *first_selem,

			gfp_t gfp_flags);

			struct bpf_local_storage_elem *first_selem);



struct bpf_local_storage_data *

bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,

			 void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags);

			 void *value, u64 map_flags, bool swap_uptrs);



u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map);

kernel/bpf/bpf_cgrp_storage.c

+4 −5
Original line number Diff line number Diff line
@@ -76,7 +76,7 @@ static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key, 
		return PTR_ERR(cgroup);



	sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,

					 value, map_flags, false, GFP_ATOMIC);

					 value, map_flags, false);

	cgroup_put(cgroup);

	return PTR_ERR_OR_ZERO(sdata);

}
@@ -122,9 +122,8 @@ static void cgroup_storage_map_free(struct bpf_map *map) 
	bpf_local_storage_map_free(map, &cgroup_cache);

}



/* *gfp_flags* is a hidden argument provided by the verifier */

BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,

	   void *, value, u64, flags, gfp_t, gfp_flags)

BPF_CALL_4(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,

	   void *, value, u64, flags)

{

	struct bpf_local_storage_data *sdata;
@@ -143,7 +142,7 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup, 
	if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&

	    (flags & BPF_LOCAL_STORAGE_GET_F_CREATE))

		sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,

						 value, BPF_NOEXIST, false, gfp_flags);

						 value, BPF_NOEXIST, false);



out:

	return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;

kernel/bpf/bpf_inode_storage.c

+4 −5
Original line number Diff line number Diff line
@@ -98,7 +98,7 @@ static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, 


	sdata = bpf_local_storage_update(file_inode(fd_file(f)),

					 (struct bpf_local_storage_map *)map,

					 value, map_flags, false, GFP_ATOMIC);

					 value, map_flags, false);

	return PTR_ERR_OR_ZERO(sdata);

}
@@ -122,9 +122,8 @@ static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key) 
	return inode_storage_delete(file_inode(fd_file(f)), map);

}



/* *gfp_flags* is a hidden argument provided by the verifier */

BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,

	   void *, value, u64, flags, gfp_t, gfp_flags)

BPF_CALL_4(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,

	   void *, value, u64, flags)

{

	struct bpf_local_storage_data *sdata;
@@ -150,7 +149,7 @@ BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, 
	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {

		sdata = bpf_local_storage_update(

			inode, (struct bpf_local_storage_map *)map, value,

			BPF_NOEXIST, false, gfp_flags);

			BPF_NOEXIST, false);

		return IS_ERR(sdata) ? (unsigned long)NULL :

					     (unsigned long)sdata->data;

	}

kernel/bpf/bpf_local_storage.c

+6 −10
Original line number Diff line number Diff line
@@ -68,7 +68,7 @@ static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem) 


struct bpf_local_storage_elem *

bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,

		void *value, bool swap_uptrs, gfp_t gfp_flags)

		void *value, bool swap_uptrs)

{

	struct bpf_local_storage_elem *selem;
@@ -475,8 +475,7 @@ static int check_flags(const struct bpf_local_storage_data *old_sdata, 


int bpf_local_storage_alloc(void *owner,

			    struct bpf_local_storage_map *smap,

			    struct bpf_local_storage_elem *first_selem,

			    gfp_t gfp_flags)

			    struct bpf_local_storage_elem *first_selem)

{

	struct bpf_local_storage *prev_storage, *storage;

	struct bpf_local_storage **owner_storage_ptr;
@@ -546,7 +545,7 @@ int bpf_local_storage_alloc(void *owner, 
 */

struct bpf_local_storage_data *

bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,

			 void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags)

			 void *value, u64 map_flags, bool swap_uptrs)

{

	struct bpf_local_storage_data *old_sdata = NULL;

	struct bpf_local_storage_elem *alloc_selem, *selem = NULL;
@@ -563,9 +562,6 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, 
		     !btf_record_has_field(smap->map.record, BPF_SPIN_LOCK)))

		return ERR_PTR(-EINVAL);



	if (gfp_flags == GFP_KERNEL && (map_flags & ~BPF_F_LOCK) != BPF_NOEXIST)

		return ERR_PTR(-EINVAL);



	local_storage = rcu_dereference_check(*owner_storage(smap, owner),

					      bpf_rcu_lock_held());

	if (!local_storage || hlist_empty(&local_storage->list)) {
@@ -574,11 +570,11 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, 
		if (err)

			return ERR_PTR(err);



		selem = bpf_selem_alloc(smap, owner, value, swap_uptrs, gfp_flags);

		selem = bpf_selem_alloc(smap, owner, value, swap_uptrs);

		if (!selem)

			return ERR_PTR(-ENOMEM);



		err = bpf_local_storage_alloc(owner, smap, selem, gfp_flags);

		err = bpf_local_storage_alloc(owner, smap, selem);

		if (err) {

			bpf_selem_free(selem, true);

			mem_uncharge(smap, owner, smap->elem_size);
@@ -608,7 +604,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, 
	/* A lookup has just been done before and concluded a new selem is

	 * needed. The chance of an unnecessary alloc is unlikely.

	 */

	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, swap_uptrs, gfp_flags);

	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, swap_uptrs);

	if (!alloc_selem)

		return ERR_PTR(-ENOMEM);

kernel/bpf/bpf_task_storage.c

+4 −5
Original line number Diff line number Diff line
@@ -118,7 +118,7 @@ static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, 


	sdata = bpf_local_storage_update(

		task, (struct bpf_local_storage_map *)map, value, map_flags,

		true, GFP_ATOMIC);

		true);



	err = PTR_ERR_OR_ZERO(sdata);

out:
@@ -165,9 +165,8 @@ static long bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key) 
	return err;

}



/* *gfp_flags* is a hidden argument provided by the verifier */

BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,

	   task, void *, value, u64, flags, gfp_t, gfp_flags)

BPF_CALL_4(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,

	   task, void *, value, u64, flags)

{

	struct bpf_local_storage_data *sdata;
@@ -184,7 +183,7 @@ BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *, 
	    (flags & BPF_LOCAL_STORAGE_GET_F_CREATE)) {

		sdata = bpf_local_storage_update(

			task, (struct bpf_local_storage_map *)map, value,

			BPF_NOEXIST, false, gfp_flags);

			BPF_NOEXIST, false);

		return IS_ERR(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;

	}