Commit 2b421662 authored by Leon Hwang's avatar Leon Hwang Committed by Alexei Starovoitov
Browse files

bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags 



Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags and check them for
following APIs:

* 'map_lookup_elem()'
* 'map_update_elem()'
* 'generic_map_lookup_batch()'
* 'generic_map_update_batch()'

And, get the correct value size for these APIs.

Acked-by: default avatarAndrii Nakryiko <andrii@kernel.org>
Signed-off-by: default avatarLeon Hwang <leon.hwang@linux.dev>
Link: https://lore.kernel.org/r/20260107022022.12843-2-leon.hwang@linux.dev


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent a8d50675

include/linux/bpf.h

+22 −1
Original line number Diff line number Diff line
@@ -3915,14 +3915,35 @@ bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image) 
}

#endif



static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type)

{

	return false;

}



static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags)

{

	if (flags & ~allowed_flags)

	u32 cpu;



	if ((u32)flags & ~allowed_flags)

		return -EINVAL;



	if ((flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK))

		return -EINVAL;



	if (!(flags & BPF_F_CPU) && flags >> 32)

		return -EINVAL;



	if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) {

		if (!bpf_map_supports_cpu_flags(map->map_type))

			return -EINVAL;

		if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS))

			return -EINVAL;



		cpu = flags >> 32;

		if ((flags & BPF_F_CPU) && cpu >= num_possible_cpus())

			return -ERANGE;

	}



	return 0;

}

include/uapi/linux/bpf.h

+2 −0
Original line number Diff line number Diff line
@@ -1384,6 +1384,8 @@ enum { 
	BPF_NOEXIST	= 1, /* create new element if it didn't exist */

	BPF_EXIST	= 2, /* update existing element */

	BPF_F_LOCK	= 4, /* spin_lock-ed map_lookup/map_update */

	BPF_F_CPU	= 8, /* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */

	BPF_F_ALL_CPUS	= 16, /* update value across all CPUs for percpu maps */

};



/* flags for BPF_MAP_CREATE command */

kernel/bpf/syscall.c

+17 −14
Original line number Diff line number Diff line
@@ -133,9 +133,11 @@ bool bpf_map_write_active(const struct bpf_map *map) 
	return atomic64_read(&map->writecnt) != 0;

}



static u32 bpf_map_value_size(const struct bpf_map *map)

static u32 bpf_map_value_size(const struct bpf_map *map, u64 flags)

{

	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

	if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS))

		return map->value_size;

	else if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||

		 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||

		 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||

		 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
@@ -1729,7 +1731,7 @@ static int map_lookup_elem(union bpf_attr *attr) 
	if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ))

		return -EPERM;



	err = bpf_map_check_op_flags(map, attr->flags, BPF_F_LOCK);

	err = bpf_map_check_op_flags(map, attr->flags, BPF_F_LOCK | BPF_F_CPU);

	if (err)

		return err;
@@ -1737,7 +1739,7 @@ static int map_lookup_elem(union bpf_attr *attr) 
	if (IS_ERR(key))

		return PTR_ERR(key);



	value_size = bpf_map_value_size(map);

	value_size = bpf_map_value_size(map, attr->flags);



	err = -ENOMEM;

	value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
@@ -1804,7 +1806,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) 
		goto err_put;

	}



	value_size = bpf_map_value_size(map);

	value_size = bpf_map_value_size(map, attr->flags);

	value = kvmemdup_bpfptr(uvalue, value_size);

	if (IS_ERR(value)) {

		err = PTR_ERR(value);
@@ -2000,11 +2002,12 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, 
	void *key, *value;

	int err = 0;



	err = bpf_map_check_op_flags(map, attr->batch.elem_flags, BPF_F_LOCK);

	err = bpf_map_check_op_flags(map, attr->batch.elem_flags,

				     BPF_F_LOCK | BPF_F_CPU | BPF_F_ALL_CPUS);

	if (err)

		return err;



	value_size = bpf_map_value_size(map);

	value_size = bpf_map_value_size(map, attr->batch.elem_flags);



	max_count = attr->batch.count;

	if (!max_count)
@@ -2059,11 +2062,11 @@ int generic_map_lookup_batch(struct bpf_map *map, 
	u32 value_size, cp, max_count;

	int err;



	err = bpf_map_check_op_flags(map, attr->batch.elem_flags, BPF_F_LOCK);

	err = bpf_map_check_op_flags(map, attr->batch.elem_flags, BPF_F_LOCK | BPF_F_CPU);

	if (err)

		return err;



	value_size = bpf_map_value_size(map);

	value_size = bpf_map_value_size(map, attr->batch.elem_flags);



	max_count = attr->batch.count;

	if (!max_count)
@@ -2185,7 +2188,7 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) 
		goto err_put;

	}



	value_size = bpf_map_value_size(map);

	value_size = bpf_map_value_size(map, 0);



	err = -ENOMEM;

	value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);

tools/include/uapi/linux/bpf.h

+2 −0
Original line number Diff line number Diff line
@@ -1384,6 +1384,8 @@ enum { 
	BPF_NOEXIST	= 1, /* create new element if it didn't exist */

	BPF_EXIST	= 2, /* update existing element */

	BPF_F_LOCK	= 4, /* spin_lock-ed map_lookup/map_update */

	BPF_F_CPU	= 8, /* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */

	BPF_F_ALL_CPUS	= 16, /* update value across all CPUs for percpu maps */

};



/* flags for BPF_MAP_CREATE command */