Commit 8c111f1b authored by Jeff Layton's avatar Jeff Layton Committed by Thomas Gleixner
Browse files

timekeeping: Don't use seqcount loop in ktime_mono_to_any() on 64-bit systems 



ktime_mono_to_any() only fetches the offset inside the loop. This is a
single word on 64-bit CPUs, and seqcount_read_begin() implies a full SMP
barrier.

Use READ_ONCE() to fetch the offset instead of doing a seqcount loop on
64-bit and add the matching WRITE_ONCE()'s to update the offsets in
tk_set_wall_to_mono() and tk_update_sleep_time().

[ tglx: Get rid of the #ifdeffery ]

Signed-off-by: default avatarJeff Layton <jlayton@kernel.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/all/20240910-mgtime-v3-1-84406ed53fad@kernel.org
parent b98b2768

kernel/time/timekeeping.c

+13 −3
Original line number Diff line number Diff line
@@ -161,13 +161,15 @@ static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm) 
	WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));

	tk->wall_to_monotonic = wtm;

	set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);

	tk->offs_real = timespec64_to_ktime(tmp);

	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));

	/* Paired with READ_ONCE() in ktime_mono_to_any() */

	WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp));

	WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)));

}



static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)

{

	tk->offs_boot = ktime_add(tk->offs_boot, delta);

	/* Paired with READ_ONCE() in ktime_mono_to_any() */

	WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta));

	/*

	 * Timespec representation for VDSO update to avoid 64bit division

	 * on every update.
@@ -930,6 +932,14 @@ ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs) 
	unsigned int seq;

	ktime_t tconv;



	if (IS_ENABLED(CONFIG_64BIT)) {

		/*

		 * Paired with WRITE_ONCE()s in tk_set_wall_to_mono() and

		 * tk_update_sleep_time().

		 */

		return ktime_add(tmono, READ_ONCE(*offset));

	}



	do {

		seq = read_seqcount_begin(&tk_core.seq);

		tconv = ktime_add(tmono, *offset);