Commit e2f9c86f authored by Boqun Feng's avatar Boqun Feng Committed by Peter Zijlstra
Browse files

rust: sync: atomic: Add atomic operation helpers over raw pointers 



In order to synchronize with C or external memory, atomic operations
over raw pointers are need. Although there is already an
`Atomic::from_ptr()` to provide a `&Atomic<T>`, it's more convenient to
have helpers that directly perform atomic operations on raw pointers.
Hence a few are added, which are basically an `Atomic::from_ptr().op()`
wrapper.

Note: for naming, since `atomic_xchg()` and `atomic_cmpxchg()` have a
conflict naming to 32bit C atomic xchg/cmpxchg, hence the helpers are
just named as `xchg()` and `cmpxchg()`. For `atomic_load()` and
`atomic_store()`, their 32bit C counterparts are `atomic_read()` and
`atomic_set()`, so keep the `atomic_` prefix.

[boqun: Fix typo spotted by Alice and fix broken sentence spotted by
Gary]

Signed-off-by: default avatarBoqun Feng <boqun.feng@gmail.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: default avatarAlice Ryhl <aliceryhl@google.com>
Reviewed-by: default avatarGary Guo <gary@garyguo.net>
Link: https://patch.msgid.link/20260120115207.55318-3-boqun.feng@gmail.com
Link: https://patch.msgid.link/20260303201701.12204-11-boqun@kernel.org
parent 28286620

rust/kernel/sync/atomic.rs

+104 −0
Original line number Diff line number Diff line
@@ -703,3 +703,107 @@ pub fn cmpxchg<Ordering: ordering::Ordering>( 
        }

    }

}



/// Atomic load over raw pointers.

///

/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work

/// with C side on synchronizations:

///

/// - `atomic_load(.., Relaxed)` maps to `READ_ONCE()` when used for inter-thread communication.

/// - `atomic_load(.., Acquire)` maps to `smp_load_acquire()`.

///

/// # Safety

///

/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.

/// - If there is a concurrent store from kernel (C or Rust), it has to be atomic.

#[doc(alias("READ_ONCE", "smp_load_acquire"))]

#[inline(always)]

pub unsafe fn atomic_load<T: AtomicType, Ordering: ordering::AcquireOrRelaxed>(

    ptr: *mut T,

    o: Ordering,

) -> T

where

    T::Repr: AtomicBasicOps,

{

    // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to

    // `align_of::<T>()`, and all concurrent stores from kernel are atomic, hence no data race per

    // LKMM.

    unsafe { Atomic::from_ptr(ptr) }.load(o)

}



/// Atomic store over raw pointers.

///

/// This function provides a short-cut of `Atomic::from_ptr().load(..)`, and can be used to work

/// with C side on synchronizations:

///

/// - `atomic_store(.., Relaxed)` maps to `WRITE_ONCE()` when used for inter-thread communication.

/// - `atomic_load(.., Release)` maps to `smp_store_release()`.

///

/// # Safety

///

/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.

/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.

#[doc(alias("WRITE_ONCE", "smp_store_release"))]

#[inline(always)]

pub unsafe fn atomic_store<T: AtomicType, Ordering: ordering::ReleaseOrRelaxed>(

    ptr: *mut T,

    v: T,

    o: Ordering,

) where

    T::Repr: AtomicBasicOps,

{

    // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to

    // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race

    // per LKMM.

    unsafe { Atomic::from_ptr(ptr) }.store(v, o);

}



/// Atomic exchange over raw pointers.

///

/// This function provides a short-cut of `Atomic::from_ptr().xchg(..)`, and can be used to work

/// with C side on synchronizations.

///

/// # Safety

///

/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.

/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.

#[inline(always)]

pub unsafe fn xchg<T: AtomicType, Ordering: ordering::Ordering>(

    ptr: *mut T,

    new: T,

    o: Ordering,

) -> T

where

    T::Repr: AtomicExchangeOps,

{

    // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to

    // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race

    // per LKMM.

    unsafe { Atomic::from_ptr(ptr) }.xchg(new, o)

}



/// Atomic compare and exchange over raw pointers.

///

/// This function provides a short-cut of `Atomic::from_ptr().cmpxchg(..)`, and can be used to work

/// with C side on synchronizations.

///

/// # Safety

///

/// - `ptr` is a valid pointer to `T` and aligned to `align_of::<T>()`.

/// - If there is a concurrent access from kernel (C or Rust), it has to be atomic.

#[doc(alias("try_cmpxchg"))]

#[inline(always)]

pub unsafe fn cmpxchg<T: AtomicType, Ordering: ordering::Ordering>(

    ptr: *mut T,

    old: T,

    new: T,

    o: Ordering,

) -> Result<T, T>

where

    T::Repr: AtomicExchangeOps,

{

    // SAFETY: Per the function safety requirement, `ptr` is valid and aligned to

    // `align_of::<T>()`, and all concurrent accesses from kernel are atomic, hence no data race

    // per LKMM.

    unsafe { Atomic::from_ptr(ptr) }.cmpxchg(old, new, o)

}

rust/kernel/sync/atomic/predefine.rs

+46 −0
Original line number Diff line number Diff line
@@ -178,6 +178,14 @@ fn atomic_basic_tests() { 


            assert_eq!(v, x.load(Relaxed));

        });



        for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {

            let x = Atomic::new(v);

            let ptr = x.as_ptr();



            // SAFETY: `ptr` is a valid pointer and no concurrent access.

            assert_eq!(v, unsafe { atomic_load(ptr, Relaxed) });

        });

    }



    #[test]
@@ -188,6 +196,17 @@ fn atomic_acquire_release_tests() { 
            x.store(v, Release);

            assert_eq!(v, x.load(Acquire));

        });



        for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {

            let x = Atomic::new(0);

            let ptr = x.as_ptr();



            // SAFETY: `ptr` is a valid pointer and no concurrent access.

            unsafe { atomic_store(ptr, v, Release) };



            // SAFETY: `ptr` is a valid pointer and no concurrent access.

            assert_eq!(v, unsafe { atomic_load(ptr, Acquire) });

        });

    }



    #[test]
@@ -201,6 +220,18 @@ fn atomic_xchg_tests() { 
            assert_eq!(old, x.xchg(new, Full));

            assert_eq!(new, x.load(Relaxed));

        });



        for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {

            let x = Atomic::new(v);

            let ptr = x.as_ptr();



            let old = v;

            let new = v + 1;



            // SAFETY: `ptr` is a valid pointer and no concurrent access.

            assert_eq!(old, unsafe { xchg(ptr, new, Full) });

            assert_eq!(new, x.load(Relaxed));

        });

    }



    #[test]
@@ -216,6 +247,21 @@ fn atomic_cmpxchg_tests() { 
            assert_eq!(Ok(old), x.cmpxchg(old, new, Relaxed));

            assert_eq!(new, x.load(Relaxed));

        });



        for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| {

            let x = Atomic::new(v);

            let ptr = x.as_ptr();



            let old = v;

            let new = v + 1;



            // SAFETY: `ptr` is a valid pointer and no concurrent access.

            assert_eq!(Err(old), unsafe { cmpxchg(ptr, new, new, Full) });

            assert_eq!(old, x.load(Relaxed));

            // SAFETY: `ptr` is a valid pointer and no concurrent access.

            assert_eq!(Ok(old), unsafe { cmpxchg(ptr, old, new, Relaxed) });

            assert_eq!(new, x.load(Relaxed));

        });

    }



    #[test]