Commit 488a8544 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'bpf-introduce-helper-for-populating-bpf_cpumask' 

Emil Tsalapatis says:

====================
bpf: introduce helper for populating bpf_cpumask

Some BPF programs like scx schedulers have their own internal CPU mask types,
mask types, which they must transform into struct bpf_cpumask instances
before passing them to scheduling-related kfuncs. There is currently no
way to efficiently populate the bitfield of a bpf_cpumask from BPF memory,
and programs must use multiple bpf_cpumask_[set, clear] calls to do so.
Introduce a kfunc helper to populate the bitfield of a bpf_cpumask from valid
BPF memory with a single call.

Changelog :
-----------
v6->v7
v6:https://lore.kernel.org/bpf/20250307153847.8530-1-emil@etsalapatis.com/

Addressed feedback by Hou Tao:
	* Removed RUN_TESTS invocation causing tests to run twice
	* Added is_test_task guard to new selftests
	* Removed extraneous __success attribute from existing selftests

v5->v6
v5:https://lore.kernel.org/bpf/20250307041738.6665-1-emil@etsalapatis.com/

Addressed feedback by Hou Tao:
	* Removed __success attributes from cpumask selftests
	* Fixed stale patch description that used old function name

v4->v5
v4: https://lore.kernel.org/bpf/20250305211235.368399-1-emil@etsalapatis.com/

Addressed feedback by Hou Tao:
	* Readded the tests in tools/selftests/bpf/prog_tests/cpumask.c,
	turns out the selftest entries were not duplicates.
	* Removed stray whitespace in selftest.
	* Add patch the missing selftest to prog_tests/cpumask.c
	* Explicitly annotate all cpumask selftests with __success

The last patch could very well be its own cleanup patch, but I rolled it into
this series because it came up in the discussion. If the last patch in the
series has any issues I'd be fine with applying the first 3 patches and dealing
with it separately.

v3->v4
v3: https://lore.kernel.org/bpf/20250305161327.203396-1-emil@etsalapatis.com/

	* Removed new tests from tools/selftests/bpf/prog_tests/cpumask.c because
they were being run twice.

Addressed feedback by Alexei Starovoitov:
	* Added missing return value in function kdoc
	* Added an additional patch fixing some missing kdoc fields in
	kernel/bpf/cpumask.c

Addressed feedback by Tejun Heo:
	* Renamed the kfunc to bpf_cpumask_populate to avoid confusion
	w/ bitmap_fill()

v2->v3
v2: https://lore.kernel.org/bpf/20250305021020.1004858-1-emil@etsalapatis.com/

Addressed feedback by Alexei Starovoitov:
	* Added back patch descriptions dropped from v1->v2
	* Elide the alignment check for archs with efficient
	  unaligned accesses

v1->v2
v1: https://lore.kernel.org/bpf/20250228003321.1409285-1-emil@etsalapatis.com/



Addressed feedback by Hou Tao:
	* Add check that the input buffer is aligned to sizeof(long)
	* Adjust input buffer size check to use bitmap_size()
	* Add selftest for checking the bit pattern of the bpf_cpumask
	* Moved all selftests into existing files

Signed-off-by: default avatarEmil Tsalapatis (Meta) <emil@etsalapatis.com>
====================

Link: https://patch.msgid.link/20250309230427.26603-1-emil@etsalapatis.com


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents 103b9ab9 c06707ff

kernel/bpf/cpumask.c

+53 −0
Original line number Diff line number Diff line
@@ -45,6 +45,10 @@ __bpf_kfunc_start_defs(); 
 *

 * bpf_cpumask_create() allocates memory using the BPF memory allocator, and

 * will not block. It may return NULL if no memory is available.

 *

 * Return:

 * * A pointer to a new struct bpf_cpumask instance on success.

 * * NULL if the BPF memory allocator is out of memory.

 */

__bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void)

{
@@ -71,6 +75,10 @@ __bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void) 
 * Acquires a reference to a BPF cpumask. The cpumask returned by this function

 * must either be embedded in a map as a kptr, or freed with

 * bpf_cpumask_release().

 *

 * Return:

 * * The struct bpf_cpumask pointer passed to the function.

 *

 */

__bpf_kfunc struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask)

{
@@ -106,6 +114,9 @@ CFI_NOSEAL(bpf_cpumask_release_dtor); 
 *

 * Find the index of the first nonzero bit of the cpumask. A struct bpf_cpumask

 * pointer may be safely passed to this function.

 *

 * Return:

 * * The index of the first nonzero bit in the struct cpumask.

 */

__bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask)

{
@@ -119,6 +130,9 @@ __bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask) 
 *

 * Find the index of the first unset bit of the cpumask. A struct bpf_cpumask

 * pointer may be safely passed to this function.

 *

 * Return:

 * * The index of the first zero bit in the struct cpumask.

 */

__bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask)

{
@@ -133,6 +147,9 @@ __bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) 
 *

 * Find the index of the first nonzero bit of the AND of two cpumasks.

 * struct bpf_cpumask pointers may be safely passed to @src1 and @src2.

 *

 * Return:

 * * The index of the first bit that is nonzero in both cpumask instances.

 */

__bpf_kfunc u32 bpf_cpumask_first_and(const struct cpumask *src1,

				      const struct cpumask *src2)
@@ -414,12 +431,47 @@ __bpf_kfunc u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, 
 * @cpumask: The cpumask being queried.

 *

 * Count the number of set bits in the given cpumask.

 *

 * Return:

 * * The number of bits set in the mask.

 */

__bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask)

{

	return cpumask_weight(cpumask);

}



/**

 * bpf_cpumask_populate() - Populate the CPU mask from the contents of

 * a BPF memory region.

 *

 * @cpumask: The cpumask being populated.

 * @src: The BPF memory holding the bit pattern.

 * @src__sz: Length of the BPF memory region in bytes.

 *

 * Return:

 * * 0 if the struct cpumask * instance was populated successfully.

 * * -EACCES if the memory region is too small to populate the cpumask.

 * * -EINVAL if the memory region is not aligned to the size of a long

 *   and the architecture does not support efficient unaligned accesses.

 */

__bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz)

{

	unsigned long source = (unsigned long)src;



	/* The memory region must be large enough to populate the entire CPU mask. */

	if (src__sz < bitmap_size(nr_cpu_ids))

		return -EACCES;



	/* If avoiding unaligned accesses, the input region must be aligned to the nearest long. */

	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&

		!IS_ALIGNED(source, sizeof(long)))

		return -EINVAL;



	bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids);



	return 0;

}



__bpf_kfunc_end_defs();



BTF_KFUNCS_START(cpumask_kfunc_btf_ids)
@@ -448,6 +500,7 @@ BTF_ID_FLAGS(func, bpf_cpumask_copy, KF_RCU) 
BTF_ID_FLAGS(func, bpf_cpumask_any_distribute, KF_RCU)

BTF_ID_FLAGS(func, bpf_cpumask_any_and_distribute, KF_RCU)

BTF_ID_FLAGS(func, bpf_cpumask_weight, KF_RCU)

BTF_ID_FLAGS(func, bpf_cpumask_populate, KF_RCU)

BTF_KFUNCS_END(cpumask_kfunc_btf_ids)



static const struct btf_kfunc_id_set cpumask_kfunc_set = {

tools/testing/selftests/bpf/prog_tests/cpumask.c

+4 −1
Original line number Diff line number Diff line
@@ -25,6 +25,10 @@ static const char * const cpumask_success_testcases[] = { 
	"test_global_mask_nested_deep_rcu",

	"test_global_mask_nested_deep_array_rcu",

	"test_cpumask_weight",

	"test_refcount_null_tracking",

	"test_populate_reject_small_mask",

	"test_populate_reject_unaligned",

	"test_populate",

};



static void verify_success(const char *prog_name)
@@ -78,6 +82,5 @@ void test_cpumask(void) 
		verify_success(cpumask_success_testcases[i]);

	}



	RUN_TESTS(cpumask_success);

	RUN_TESTS(cpumask_failure);

}

tools/testing/selftests/bpf/progs/cpumask_common.h

+1 −0
Original line number Diff line number Diff line
@@ -61,6 +61,7 @@ u32 bpf_cpumask_any_distribute(const struct cpumask *src) __ksym __weak; 
u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1,

				   const struct cpumask *src2) __ksym __weak;

u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym __weak;

int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) __ksym __weak;



void bpf_rcu_read_lock(void) __ksym __weak;

void bpf_rcu_read_unlock(void) __ksym __weak;

tools/testing/selftests/bpf/progs/cpumask_failure.c

+38 −0
Original line number Diff line number Diff line
@@ -222,3 +222,41 @@ int BPF_PROG(test_invalid_nested_array, struct task_struct *task, u64 clone_flag 


	return 0;

}



SEC("tp_btf/task_newtask")

__failure __msg("type=scalar expected=fp")

int BPF_PROG(test_populate_invalid_destination, struct task_struct *task, u64 clone_flags)

{

	struct bpf_cpumask *invalid = (struct bpf_cpumask *)0x123456;

	u64 bits;

	int ret;



	ret = bpf_cpumask_populate((struct cpumask *)invalid, &bits, sizeof(bits));

	if (!ret)

		err = 2;



	return 0;

}



SEC("tp_btf/task_newtask")

__failure __msg("leads to invalid memory access")

int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_flags)

{

	void *garbage = (void *)0x123456;

	struct bpf_cpumask *local;

	int ret;



	local = create_cpumask();

	if (!local) {

		err = 1;

		return 0;

	}



	ret = bpf_cpumask_populate((struct cpumask *)local, garbage, 8);

	if (!ret)

		err = 2;



	bpf_cpumask_release(local);



	return 0;

}

tools/testing/selftests/bpf/progs/cpumask_success.c

+119 −1
Original line number Diff line number Diff line
@@ -749,7 +749,6 @@ int BPF_PROG(test_cpumask_weight, struct task_struct *task, u64 clone_flags) 
}



SEC("tp_btf/task_newtask")

__success

int BPF_PROG(test_refcount_null_tracking, struct task_struct *task, u64 clone_flags)

{

	struct bpf_cpumask *mask1, *mask2;
@@ -770,3 +769,122 @@ int BPF_PROG(test_refcount_null_tracking, struct task_struct *task, u64 clone_fl 
		bpf_cpumask_release(mask2);

	return 0;

}



SEC("tp_btf/task_newtask")

int BPF_PROG(test_populate_reject_small_mask, struct task_struct *task, u64 clone_flags)

{

	struct bpf_cpumask *local;

	u8 toofewbits;

	int ret;



	if (!is_test_task())

		return 0;



	local = create_cpumask();

	if (!local)

		return 0;



	/* The kfunc should prevent this operation */

	ret = bpf_cpumask_populate((struct cpumask *)local, &toofewbits, sizeof(toofewbits));

	if (ret != -EACCES)

		err = 2;



	bpf_cpumask_release(local);



	return 0;

}



/* Mask is guaranteed to be large enough for bpf_cpumask_t. */

#define CPUMASK_TEST_MASKLEN (sizeof(cpumask_t))



/* Add an extra word for the test_populate_reject_unaligned test. */

u64 bits[CPUMASK_TEST_MASKLEN / 8 + 1];

extern bool CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS __kconfig __weak;



SEC("tp_btf/task_newtask")

int BPF_PROG(test_populate_reject_unaligned, struct task_struct *task, u64 clone_flags)

{

	struct bpf_cpumask *mask;

	char *src;

	int ret;



	if (!is_test_task())

		return 0;



	/* Skip if unaligned accesses are fine for this arch.  */

	if (CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)

		return 0;



	mask = bpf_cpumask_create();

	if (!mask) {

		err = 1;

		return 0;

	}



	/* Misalign the source array by a byte. */

	src = &((char *)bits)[1];



	ret = bpf_cpumask_populate((struct cpumask *)mask, src, CPUMASK_TEST_MASKLEN);

	if (ret != -EINVAL)

		err = 2;



	bpf_cpumask_release(mask);



	return 0;

}





SEC("tp_btf/task_newtask")

int BPF_PROG(test_populate, struct task_struct *task, u64 clone_flags)

{

	struct bpf_cpumask *mask;

	bool bit;

	int ret;

	int i;



	if (!is_test_task())

		return 0;



	/* Set only odd bits. */

	__builtin_memset(bits, 0xaa, CPUMASK_TEST_MASKLEN);



	mask = bpf_cpumask_create();

	if (!mask) {

		err = 1;

		return 0;

	}



	/* Pass the entire bits array, the kfunc will only copy the valid bits. */

	ret = bpf_cpumask_populate((struct cpumask *)mask, bits, CPUMASK_TEST_MASKLEN);

	if (ret) {

		err = 2;

		goto out;

	}



	/*

	 * Test is there to appease the verifier. We cannot directly

	 * access NR_CPUS, the upper bound for nr_cpus, so we infer

	 * it from the size of cpumask_t.

	 */

	if (nr_cpus < 0 || nr_cpus >= CPUMASK_TEST_MASKLEN * 8) {

		err = 3;

		goto out;

	}



	bpf_for(i, 0, nr_cpus) {

		/* Odd-numbered bits should be set, even ones unset. */

		bit = bpf_cpumask_test_cpu(i, (const struct cpumask *)mask);

		if (bit == (i % 2 != 0))

			continue;



		err = 4;

		break;

	}



out:

	bpf_cpumask_release(mask);



	return 0;

}



#undef CPUMASK_TEST_MASKLEN