Commit b9a5a07a authored by Martin KaFai Lau's avatar Martin KaFai Lau Committed by Alexei Starovoitov
Browse files

bpf: Add "bool swap_uptrs" arg to bpf_local_storage_update() and bpf_selem_alloc() 



In a later patch, the task local storage will only accept uptr
from the syscall update_elem and will not accept uptr from
the bpf prog. The reason is the bpf prog does not have a way
to provide a valid user space address.

bpf_local_storage_update() and bpf_selem_alloc() are used by
both bpf prog bpf_task_storage_get(BPF_LOCAL_STORAGE_GET_F_CREATE)
and bpf syscall update_elem. "bool swap_uptrs" arg is added
to bpf_local_storage_update() and bpf_selem_alloc() to tell if
it is called by the bpf prog or by the bpf syscall. When
swap_uptrs==true, it is called by the syscall.

The arg is named (swap_)uptrs because the later patch will swap
the uptrs between the newly allocated selem and the user space
provided map_value. It will make error handling easier in case
map->ops->map_update_elem() fails and the caller can decide
if it needs to unpin the uptr in the user space provided
map_value or the bpf_local_storage_update() has already
taken the uptr ownership and will take care of unpinning it also.

Only swap_uptrs==false is passed now. The logic to handle
the true case will be added in a later patch.

Signed-off-by: default avatarMartin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20241023234759.860539-4-martin.lau@linux.dev


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 99dde42e

include/linux/bpf_local_storage.h

+2 −2
Original line number Diff line number Diff line
@@ -181,7 +181,7 @@ void bpf_selem_link_map(struct bpf_local_storage_map *smap, 


struct bpf_local_storage_elem *

bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,

		bool charge_mem, gfp_t gfp_flags);

		bool charge_mem, bool swap_uptrs, gfp_t gfp_flags);



void bpf_selem_free(struct bpf_local_storage_elem *selem,

		    struct bpf_local_storage_map *smap,
@@ -195,7 +195,7 @@ bpf_local_storage_alloc(void *owner, 


struct bpf_local_storage_data *

bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,

			 void *value, u64 map_flags, gfp_t gfp_flags);

			 void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags);



u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map);

kernel/bpf/bpf_cgrp_storage.c

+2 −2
Original line number Diff line number Diff line
@@ -107,7 +107,7 @@ static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key, 


	bpf_cgrp_storage_lock();

	sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,

					 value, map_flags, GFP_ATOMIC);

					 value, map_flags, false, GFP_ATOMIC);

	bpf_cgrp_storage_unlock();

	cgroup_put(cgroup);

	return PTR_ERR_OR_ZERO(sdata);
@@ -181,7 +181,7 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup, 
	if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&

	    (flags & BPF_LOCAL_STORAGE_GET_F_CREATE))

		sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,

						 value, BPF_NOEXIST, gfp_flags);

						 value, BPF_NOEXIST, false, gfp_flags);



unlock:

	bpf_cgrp_storage_unlock();

kernel/bpf/bpf_inode_storage.c

+2 −2
Original line number Diff line number Diff line
@@ -100,7 +100,7 @@ static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, 


	sdata = bpf_local_storage_update(file_inode(fd_file(f)),

					 (struct bpf_local_storage_map *)map,

					 value, map_flags, GFP_ATOMIC);

					 value, map_flags, false, GFP_ATOMIC);

	return PTR_ERR_OR_ZERO(sdata);

}
@@ -154,7 +154,7 @@ BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, 
	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {

		sdata = bpf_local_storage_update(

			inode, (struct bpf_local_storage_map *)map, value,

			BPF_NOEXIST, gfp_flags);

			BPF_NOEXIST, false, gfp_flags);

		return IS_ERR(sdata) ? (unsigned long)NULL :

					     (unsigned long)sdata->data;

	}

kernel/bpf/bpf_local_storage.c

+4 −4
Original line number Diff line number Diff line
@@ -73,7 +73,7 @@ static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem) 


struct bpf_local_storage_elem *

bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,

		void *value, bool charge_mem, gfp_t gfp_flags)

		void *value, bool charge_mem, bool swap_uptrs, gfp_t gfp_flags)

{

	struct bpf_local_storage_elem *selem;
@@ -524,7 +524,7 @@ int bpf_local_storage_alloc(void *owner, 
 */

struct bpf_local_storage_data *

bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,

			 void *value, u64 map_flags, gfp_t gfp_flags)

			 void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags)

{

	struct bpf_local_storage_data *old_sdata = NULL;

	struct bpf_local_storage_elem *alloc_selem, *selem = NULL;
@@ -550,7 +550,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, 
		if (err)

			return ERR_PTR(err);



		selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);

		selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);

		if (!selem)

			return ERR_PTR(-ENOMEM);
@@ -584,7 +584,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, 
	/* A lookup has just been done before and concluded a new selem is

	 * needed. The chance of an unnecessary alloc is unlikely.

	 */

	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);

	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);

	if (!alloc_selem)

		return ERR_PTR(-ENOMEM);

kernel/bpf/bpf_task_storage.c

+2 −2
Original line number Diff line number Diff line
@@ -147,7 +147,7 @@ static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, 
	bpf_task_storage_lock();

	sdata = bpf_local_storage_update(

		task, (struct bpf_local_storage_map *)map, value, map_flags,

		GFP_ATOMIC);

		false, GFP_ATOMIC);

	bpf_task_storage_unlock();



	err = PTR_ERR_OR_ZERO(sdata);
@@ -219,7 +219,7 @@ static void *__bpf_task_storage_get(struct bpf_map *map, 
	    (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) {

		sdata = bpf_local_storage_update(

			task, (struct bpf_local_storage_map *)map, value,

			BPF_NOEXIST, gfp_flags);

			BPF_NOEXIST, false, gfp_flags);

		return IS_ERR(sdata) ? NULL : sdata->data;

	}