Commit 780818a6 authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Boqun Feng
Browse files

srcu: Rename srcu_check_read_flavor_lite() to srcu_check_read_flavor_force() 



This commit renames the srcu_check_read_flavor_lite() function to
srcu_check_read_flavor_force() and adds a read_flavor argument in order to
support an srcu_read_lock_fast() variant that is to avoid array indexing
in both the lock and unlock primitives.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: <bpf@vger.kernel.org>
Signed-off-by: default avatarBoqun Feng <boqun.feng@gmail.com>
parent d31e3136

include/linux/srcu.h

+1 −1
Original line number Diff line number Diff line
@@ -279,7 +279,7 @@ static inline int srcu_read_lock_lite(struct srcu_struct *ssp) __acquires(ssp) 
{

	int retval;



	srcu_check_read_flavor_lite(ssp);

	srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_LITE);

	retval = __srcu_read_lock_lite(ssp);

	rcu_try_lock_acquire(&ssp->dep_map);

	return retval;

include/linux/srcutiny.h

+1 −1
Original line number Diff line number Diff line
@@ -82,7 +82,7 @@ static inline void srcu_barrier(struct srcu_struct *ssp) 
}



#define srcu_check_read_flavor(ssp, read_flavor) do { } while (0)

#define srcu_check_read_flavor_lite(ssp) do { } while (0)

#define srcu_check_read_flavor_force(ssp, read_flavor) do { } while (0)



/* Defined here to avoid size increase for non-torture kernels. */

static inline void srcu_torture_stats_print(struct srcu_struct *ssp,

include/linux/srcutree.h

+6 −4
Original line number Diff line number Diff line
@@ -251,16 +251,18 @@ static inline void __srcu_read_unlock_lite(struct srcu_struct *ssp, int idx) 


void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor);



// Record _lite() usage even for CONFIG_PROVE_RCU=n kernels.

static inline void srcu_check_read_flavor_lite(struct srcu_struct *ssp)

// Record reader usage even for CONFIG_PROVE_RCU=n kernels.  This is

// needed only for flavors that require grace-period smp_mb() calls to be

// promoted to synchronize_rcu().

static inline void srcu_check_read_flavor_force(struct srcu_struct *ssp, int read_flavor)

{

	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);



	if (likely(READ_ONCE(sdp->srcu_reader_flavor) & SRCU_READ_FLAVOR_LITE))

	if (likely(READ_ONCE(sdp->srcu_reader_flavor) & read_flavor))

		return;



	// Note that the cmpxchg() in __srcu_check_read_flavor() is fully ordered.

	__srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_LITE);

	__srcu_check_read_flavor(ssp, read_flavor);

}



// Record non-_lite() usage only for CONFIG_PROVE_RCU=y kernels.