rust: sync: atomic: Add atomic {cmp,}xchg operations (b606a532) · Commits · git / linux-net

rust/kernel/sync/atomic.rs

+167 −1

Original line number	Diff line number	Diff line
		@@ -25,7 +25,7 @@
		pub use ordering::{Acquire, Full, Relaxed, Release};

		use crate::build_error;
		use internal::{AtomicBasicOps, AtomicRepr};
		use internal::{AtomicBasicOps, AtomicExchangeOps, AtomicRepr};
		use ordering::OrderingType;

		/// A memory location which can be safely modified from multiple execution contexts.
		@@ -293,3 +293,169 @@ pub fn store<Ordering: ordering::ReleaseOrRelaxed>(&self, v: T, _: Ordering) {
		}
		}
		}

		impl<T: AtomicType> Atomic<T>
		where
		T::Repr: AtomicExchangeOps,
		{
		/// Atomic exchange.
		///
		/// Atomically updates `self` to `v` and returns the old value of `self`.
		///
		/// # Examples
		///
		/// ```
		/// use kernel::sync::atomic::{Atomic, Acquire, Relaxed};
		///
		/// let x = Atomic::new(42);
		///
		/// assert_eq!(42, x.xchg(52, Acquire));
		/// assert_eq!(52, x.load(Relaxed));
		/// ```
		#[doc(alias("atomic_xchg", "atomic64_xchg", "swap"))]
		#[inline(always)]
		pub fn xchg<Ordering: ordering::Ordering>(&self, v: T, _: Ordering) -> T {
		let v = into_repr(v);

		// INVARIANT: `self.0` is a valid `T` after `atomic_xchg*()` because `v` is transmutable to
		// `T`.
		let ret = {
		match Ordering::TYPE {
		OrderingType::Full => T::Repr::atomic_xchg(&self.0, v),
		OrderingType::Acquire => T::Repr::atomic_xchg_acquire(&self.0, v),
		OrderingType::Release => T::Repr::atomic_xchg_release(&self.0, v),
		OrderingType::Relaxed => T::Repr::atomic_xchg_relaxed(&self.0, v),
		}
		};

		// SAFETY: `ret` comes from reading `*self`, which is a valid `T` per type invariants.
		unsafe { from_repr(ret) }
		}

		/// Atomic compare and exchange.
		///
		/// If `self` == `old`, atomically updates `self` to `new`. Otherwise, `*self` is not
		/// modified.
		///
		/// Compare: The comparison is done via the byte level comparison between `*self` and `old`.
		///
		/// Ordering: When succeeds, provides the corresponding ordering as the `Ordering` type
		/// parameter indicates, and a failed one doesn't provide any ordering, the load part of a
		/// failed cmpxchg is a [`Relaxed`] load.
		///
		/// Returns `Ok(value)` if cmpxchg succeeds, and `value` is guaranteed to be equal to `old`,
		/// otherwise returns `Err(value)`, and `value` is the current value of `*self`.
		///
		/// # Examples
		///
		/// ```
		/// use kernel::sync::atomic::{Atomic, Full, Relaxed};
		///
		/// let x = Atomic::new(42);
		///
		/// // Checks whether cmpxchg succeeded.
		/// let success = x.cmpxchg(52, 64, Relaxed).is_ok();
		/// # assert!(!success);
		///
		/// // Checks whether cmpxchg failed.
		/// let failure = x.cmpxchg(52, 64, Relaxed).is_err();
		/// # assert!(failure);
		///
		/// // Uses the old value if failed, probably re-try cmpxchg.
		/// match x.cmpxchg(52, 64, Relaxed) {
		/// Ok(_) => { },
		/// Err(old) => {
		/// // do something with `old`.
		/// # assert_eq!(old, 42);
		/// }
		/// }
		///
		/// // Uses the latest value regardlessly, same as atomic_cmpxchg() in C.
		/// let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(\|old\| old);
		/// # assert_eq!(42, latest);
		/// assert_eq!(64, x.load(Relaxed));
		/// ```
		///
		/// [`Relaxed`]: ordering::Relaxed
		#[doc(alias(
		"atomic_cmpxchg",
		"atomic64_cmpxchg",
		"atomic_try_cmpxchg",
		"atomic64_try_cmpxchg",
		"compare_exchange"
		))]
		#[inline(always)]
		pub fn cmpxchg<Ordering: ordering::Ordering>(
		&self,
		mut old: T,
		new: T,
		o: Ordering,
		) -> Result<T, T> {
		// Note on code generation:
		//
		// try_cmpxchg() is used to implement cmpxchg(), and if the helper functions are inlined,
		// the compiler is able to figure out that branch is not needed if the users don't care
		// about whether the operation succeeds or not. One exception is on x86, due to commit
		// 44fe84459faf ("locking/atomic: Fix atomic_try_cmpxchg() semantics"), the
		// atomic_try_cmpxchg() on x86 has a branch even if the caller doesn't care about the
		// success of cmpxchg and only wants to use the old value. For example, for code like:
		//
		// let latest = x.cmpxchg(42, 64, Full).unwrap_or_else(\|old\| old);
		//
		// It will still generate code:
		//
		// movl $0x40, %ecx
		// movl $0x34, %eax
		// lock
		// cmpxchgl %ecx, 0x4(%rsp)
		// jne 1f
		// 2:
		// ...
		// 1: movl %eax, %ecx
		// jmp 2b
		//
		// This might be "fixed" by introducing a try_cmpxchg_exclusive() that knows the "*old"
		// location in the C function is always safe to write.
		if self.try_cmpxchg(&mut old, new, o) {
		Ok(old)
		} else {
		Err(old)
		}
		}

		/// Atomic compare and exchange and returns whether the operation succeeds.
		///
		/// If `self` == `old`, atomically updates `self` to `new`. Otherwise, `*self` is not
		/// modified, `old` is updated to the current value of `self`.
		///
		/// "Compare" and "Ordering" part are the same as [`Atomic::cmpxchg()`].
		///
		/// Returns `true` means the cmpxchg succeeds otherwise returns `false`.
		#[inline(always)]
		fn try_cmpxchg<Ordering: ordering::Ordering>(&self, old: &mut T, new: T, _: Ordering) -> bool {
		let mut tmp = into_repr(*old);
		let new = into_repr(new);

		// INVARIANT: `self.0` is a valid `T` after `atomic_try_cmpxchg*()` because `new` is
		// transmutable to `T`.
		let ret = {
		match Ordering::TYPE {
		OrderingType::Full => T::Repr::atomic_try_cmpxchg(&self.0, &mut tmp, new),
		OrderingType::Acquire => {
		T::Repr::atomic_try_cmpxchg_acquire(&self.0, &mut tmp, new)
		}
		OrderingType::Release => {
		T::Repr::atomic_try_cmpxchg_release(&self.0, &mut tmp, new)
		}
		OrderingType::Relaxed => {
		T::Repr::atomic_try_cmpxchg_relaxed(&self.0, &mut tmp, new)
		}
		}
		};

		// SAFETY: `tmp` comes from reading `*self`, which is a valid `T` per type invariants.
		*old = unsafe { from_repr(tmp) };

		ret
		}
		}