Commit b4560055 authored by Andrii Nakryiko's avatar Andrii Nakryiko
Browse files

Merge branch 'bpf-volatile-compare' 

Alexei Starovoitov says:

====================
bpf: volatile compare

From: Alexei Starovoitov <ast@kernel.org>

v2->v3:
Debugged profiler.c regression. It was caused by basic block layout.
Introduce bpf_cmp_likely() and bpf_cmp_unlikely() macros.
Debugged redundant <<=32, >>=32 with u32 variables. Added cast workaround.

v1->v2:
Fixed issues pointed out by Daniel, added more tests, attempted to convert profiler.c,
but barrier_var() wins vs bpf_cmp(). To be investigated.
Patches 1-4 are good to go, but 5 needs more work.
====================

Link: https://lore.kernel.org/r/20231226191148.48536-1-alexei.starovoitov@gmail.com


Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
parents a640de4c 7e3811cb

tools/testing/selftests/bpf/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -383,6 +383,7 @@ CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) 
BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN)	\

	     -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR)			\

	     -I$(abspath $(OUTPUT)/../usr/include)

# TODO: enable me -Wsign-compare



CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) \

	       -Wno-compare-distinct-pointer-types

tools/testing/selftests/bpf/bpf_experimental.h

+72 −148
Original line number Diff line number Diff line
@@ -254,173 +254,97 @@ extern void bpf_throw(u64 cookie) __ksym; 
		}									\

	 })



/* Description

 *	Assert that a conditional expression is true.

 * Returns

 *	Void.

 * Throws

 *	An exception with the value zero when the assertion fails.

 */

#define bpf_assert(cond) if (!(cond)) bpf_throw(0);



/* Description

 *	Assert that a conditional expression is true.

 * Returns

 *	Void.

 * Throws

 *	An exception with the specified value when the assertion fails.

 */

#define bpf_assert_with(cond, value) if (!(cond)) bpf_throw(value);



/* Description

 *	Assert that LHS is equal to RHS. This statement updates the known value

 *	of LHS during verification. Note that RHS must be a constant value, and

 *	must fit within the data type of LHS.

 * Returns

 *	Void.

 * Throws

 *	An exception with the value zero when the assertion fails.

 */

#define bpf_assert_eq(LHS, RHS)						\

	({								\

		barrier_var(LHS);					\

		__bpf_assert_op(LHS, ==, RHS, 0, true);			\

	})

#define __cmp_cannot_be_signed(x) \

	__builtin_strcmp(#x, "==") == 0 || __builtin_strcmp(#x, "!=") == 0 || \

	__builtin_strcmp(#x, "&") == 0



/* Description

 *	Assert that LHS is equal to RHS. This statement updates the known value

 *	of LHS during verification. Note that RHS must be a constant value, and

 *	must fit within the data type of LHS.

 * Returns

 *	Void.

 * Throws

 *	An exception with the specified value when the assertion fails.

 */

#define bpf_assert_eq_with(LHS, RHS, value)				\

	({								\

		barrier_var(LHS);					\

		__bpf_assert_op(LHS, ==, RHS, value, true);		\

	})

#define __is_signed_type(type) (((type)(-1)) < (type)1)



/* Description

 *	Assert that LHS is less than RHS. This statement updates the known

 *	bounds of LHS during verification. Note that RHS must be a constant

 *	value, and must fit within the data type of LHS.

 * Returns

 *	Void.

 * Throws

 *	An exception with the value zero when the assertion fails.

 */

#define bpf_assert_lt(LHS, RHS)						\

#define __bpf_cmp(LHS, OP, SIGN, PRED, RHS, DEFAULT)						\

	({											\

		barrier_var(LHS);					\

		__bpf_assert_op(LHS, <, RHS, 0, false);			\

		__label__ l_true;								\

		bool ret = DEFAULT;								\

		asm volatile goto("if %[lhs] " SIGN #OP " %[rhs] goto %l[l_true]"		\

				  :: [lhs] "r"((short)LHS), [rhs] PRED (RHS) :: l_true);	\

		ret = !DEFAULT;									\

l_true:												\

		ret;										\

       })



/* Description

 *	Assert that LHS is less than RHS. This statement updates the known

 *	bounds of LHS during verification. Note that RHS must be a constant

 *	value, and must fit within the data type of LHS.

 * Returns

 *	Void.

 * Throws

 *	An exception with the specified value when the assertion fails.

/* C type conversions coupled with comparison operator are tricky.

 * Make sure BPF program is compiled with -Wsign-compare then

 * __lhs OP __rhs below will catch the mistake.

 * Be aware that we check only __lhs to figure out the sign of compare.

 */

#define bpf_assert_lt_with(LHS, RHS, value)				\

#define _bpf_cmp(LHS, OP, RHS, NOFLIP)								\

	({											\

		barrier_var(LHS);					\

		__bpf_assert_op(LHS, <, RHS, value, false);		\

	})



/* Description

 *	Assert that LHS is greater than RHS. This statement updates the known

 *	bounds of LHS during verification. Note that RHS must be a constant

 *	value, and must fit within the data type of LHS.

 * Returns

 *	Void.

 * Throws

 *	An exception with the value zero when the assertion fails.

 */

#define bpf_assert_gt(LHS, RHS)						\

	({								\

		barrier_var(LHS);					\

		__bpf_assert_op(LHS, >, RHS, 0, false);			\

		typeof(LHS) __lhs = (LHS);							\

		typeof(RHS) __rhs = (RHS);							\

		bool ret;									\

		_Static_assert(sizeof(&(LHS)), "1st argument must be an lvalue expression");	\

		(void)(__lhs OP __rhs);								\

		if (__cmp_cannot_be_signed(OP) || !__is_signed_type(typeof(__lhs))) {		\

			if (sizeof(__rhs) == 8)							\

				ret = __bpf_cmp(__lhs, OP, "", "r", __rhs, NOFLIP);		\

			else									\

				ret = __bpf_cmp(__lhs, OP, "", "i", __rhs, NOFLIP);		\

		} else {									\

			if (sizeof(__rhs) == 8)							\

				ret = __bpf_cmp(__lhs, OP, "s", "r", __rhs, NOFLIP);		\

			else									\

				ret = __bpf_cmp(__lhs, OP, "s", "i", __rhs, NOFLIP);		\

		}										\

		ret;										\

       })



/* Description

 *	Assert that LHS is greater than RHS. This statement updates the known

 *	bounds of LHS during verification. Note that RHS must be a constant

 *	value, and must fit within the data type of LHS.

 * Returns

 *	Void.

 * Throws

 *	An exception with the specified value when the assertion fails.

 */

#define bpf_assert_gt_with(LHS, RHS, value)				\

	({								\

		barrier_var(LHS);					\

		__bpf_assert_op(LHS, >, RHS, value, false);		\

	})

#ifndef bpf_cmp_unlikely

#define bpf_cmp_unlikely(LHS, OP, RHS) _bpf_cmp(LHS, OP, RHS, true)

#endif



/* Description

 *	Assert that LHS is less than or equal to RHS. This statement updates the

 *	known bounds of LHS during verification. Note that RHS must be a

 *	constant value, and must fit within the data type of LHS.

 * Returns

 *	Void.

 * Throws

 *	An exception with the value zero when the assertion fails.

 */

#define bpf_assert_le(LHS, RHS)						\

#ifndef bpf_cmp_likely

#define bpf_cmp_likely(LHS, OP, RHS)								\

	({											\

		barrier_var(LHS);					\

		__bpf_assert_op(LHS, <=, RHS, 0, false);		\

		bool ret;									\

		if (__builtin_strcmp(#OP, "==") == 0)						\

			ret = _bpf_cmp(LHS, !=, RHS, false);					\

		else if (__builtin_strcmp(#OP, "!=") == 0)					\

			ret = _bpf_cmp(LHS, ==, RHS, false);					\

		else if (__builtin_strcmp(#OP, "<=") == 0)					\

			ret = _bpf_cmp(LHS, >, RHS, false);					\

		else if (__builtin_strcmp(#OP, "<") == 0)					\

			ret = _bpf_cmp(LHS, >=, RHS, false);					\

		else if (__builtin_strcmp(#OP, ">") == 0)					\

			ret = _bpf_cmp(LHS, <=, RHS, false);					\

		else if (__builtin_strcmp(#OP, ">=") == 0)					\

			ret = _bpf_cmp(LHS, <, RHS, false);					\

		else										\

			(void) "bug";								\

		ret;										\

       })

#endif



/* Description

 *	Assert that LHS is less than or equal to RHS. This statement updates the

 *	known bounds of LHS during verification. Note that RHS must be a

 *	constant value, and must fit within the data type of LHS.

 * Returns

 *	Void.

 * Throws

 *	An exception with the specified value when the assertion fails.

 */

#define bpf_assert_le_with(LHS, RHS, value)				\

	({								\

		barrier_var(LHS);					\

		__bpf_assert_op(LHS, <=, RHS, value, false);		\

	})

#ifndef bpf_nop_mov

#define bpf_nop_mov(var) \

	asm volatile("%[reg]=%[reg]"::[reg]"r"((short)var))

#endif



/* Description

 *	Assert that LHS is greater than or equal to RHS. This statement updates

 *	the known bounds of LHS during verification. Note that RHS must be a

 *	constant value, and must fit within the data type of LHS.

 *	Assert that a conditional expression is true.

 * Returns

 *	Void.

 * Throws

 *	An exception with the value zero when the assertion fails.

 */

#define bpf_assert_ge(LHS, RHS)						\

	({								\

		barrier_var(LHS);					\

		__bpf_assert_op(LHS, >=, RHS, 0, false);		\

	})

#define bpf_assert(cond) if (!(cond)) bpf_throw(0);



/* Description

 *	Assert that LHS is greater than or equal to RHS. This statement updates

 *	the known bounds of LHS during verification. Note that RHS must be a

 *	constant value, and must fit within the data type of LHS.

 *	Assert that a conditional expression is true.

 * Returns

 *	Void.

 * Throws

 *	An exception with the specified value when the assertion fails.

 */

#define bpf_assert_ge_with(LHS, RHS, value)				\

	({								\

		barrier_var(LHS);					\

		__bpf_assert_op(LHS, >=, RHS, value, false);		\

	})

#define bpf_assert_with(cond, value) if (!(cond)) bpf_throw(value);



/* Description

 *	Assert that LHS is in the range [BEG, END] (inclusive of both). This

tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_hash_map.c

+1 −1
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ struct { 
} hashmap1 SEC(".maps");



/* will set before prog run */

volatile const __u32 num_cpus = 0;

volatile const __s32 num_cpus = 0;



/* will collect results during prog run */

__u32 key_sum_a = 0, key_sum_b = 0, key_sum_c = 0;

tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c

+1 −1
Original line number Diff line number Diff line
@@ -35,7 +35,7 @@ SEC("iter/task_vma") int proc_maps(struct bpf_iter__task_vma *ctx) 
		return 0;



	file = vma->vm_file;

	if (task->tgid != pid) {

	if (task->tgid != (pid_t)pid) {

		if (one_task)

			one_task_error = 1;

		return 0;

tools/testing/selftests/bpf/progs/bpf_iter_tasks.c

+1 −1
Original line number Diff line number Diff line
@@ -22,7 +22,7 @@ int dump_task(struct bpf_iter__task *ctx) 
		return 0;

	}



	if (task->pid != tid)

	if (task->pid != (pid_t)tid)

		num_unknown_tid++;

	else

		num_known_tid++;