Commit c7e03c5c authored by Danilo Krummrich's avatar Danilo Krummrich
Browse files

Merge tag 'topic/dma-features-2025-06-23' into alloc-next 

DMA features for v6.17

- Clarify wording and be consistent in 'coherent' nomenclature.

- Convert the read!() / write!() macros to return a Result.

- Add as_slice() / write() methods in CoherentAllocation.

- Fix doc-comment of dma_handle().

- Expose count() and size() in CoherentAllocation and add the
  corresponding type invariants.

- Implement CoherentAllocation::dma_handle_with_offset().
parents f86c0036 26af8565

rust/kernel/dma.rs

+165 −34
Original line number Diff line number Diff line
@@ -89,7 +89,7 @@ pub mod attrs { 
    /// Forces contiguous allocation of the buffer in physical memory.

    pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS);



    /// This is a hint to the DMA-mapping subsystem that it's probably not worth the time to try

    /// Hints DMA-mapping subsystem that it's probably not worth the time to try

    /// to allocate memory to in a way that gives better TLB efficiency.

    pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES);
@@ -97,7 +97,7 @@ pub mod attrs { 
    /// `__GFP_NOWARN`).

    pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN);



    /// Used to indicate that the buffer is fully accessible at an elevated privilege level (and

    /// Indicates that the buffer is fully accessible at an elevated privilege level (and

    /// ideally inaccessible or at least read-only at lesser-privileged levels).

    pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED);

}
@@ -105,7 +105,7 @@ pub mod attrs { 
/// An abstraction of the `dma_alloc_coherent` API.

///

/// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map

/// large consistent DMA regions.

/// large coherent DMA regions.

///

/// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the

/// processor's virtual address space) and the device address which can be given to the device
@@ -114,9 +114,11 @@ pub mod attrs { 
///

/// # Invariants

///

/// For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer

/// to an allocated region of consistent memory and `dma_handle` is the DMA address base of

/// the region.

/// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer

///   to an allocated region of coherent memory and `dma_handle` is the DMA address base of the

///   region.

/// - The size in bytes of the allocation is equal to `size_of::<T> * count`.

/// - `size_of::<T> * count` fits into a `usize`.

// TODO

//

// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness
@@ -138,7 +140,7 @@ pub struct CoherentAllocation<T: AsBytes + FromBytes> { 
}



impl<T: AsBytes + FromBytes> CoherentAllocation<T> {

    /// Allocates a region of `size_of::<T> * count` of consistent memory.

    /// Allocates a region of `size_of::<T> * count` of coherent memory.

    ///

    /// # Examples

    ///
@@ -179,9 +181,12 @@ pub fn alloc_attrs( 
        if ret.is_null() {

            return Err(ENOMEM);

        }

        // INVARIANT: We just successfully allocated a coherent region which is accessible for

        // INVARIANT:

        // - We just successfully allocated a coherent region which is accessible for

        //   `count` elements, hence the cpu address is valid. We also hold a refcounted reference

        //   to the device.

        // - The allocated `size` is equal to `size_of::<T> * count`.

        // - The allocated `size` fits into a `usize`.

        Ok(Self {

            dev: dev.into(),

            dma_handle,
@@ -201,6 +206,21 @@ pub fn alloc_coherent( 
        CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0))

    }



    /// Returns the number of elements `T` in this allocation.

    ///

    /// Note that this is not the size of the allocation in bytes, which is provided by

    /// [`Self::size`].

    pub fn count(&self) -> usize {

        self.count

    }



    /// Returns the size in bytes of this allocation.

    pub fn size(&self) -> usize {

        // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits into

        // a `usize`.

        self.count * core::mem::size_of::<T>()

    }



    /// Returns the base address to the allocated region in the CPU's virtual address space.

    pub fn start_ptr(&self) -> *const T {

        self.cpu_addr
@@ -212,12 +232,113 @@ pub fn start_ptr_mut(&mut self) -> *mut T { 
        self.cpu_addr

    }



    /// Returns a DMA handle which may given to the device as the DMA address base of

    /// Returns a DMA handle which may be given to the device as the DMA address base of

    /// the region.

    pub fn dma_handle(&self) -> bindings::dma_addr_t {

        self.dma_handle

    }



    /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the

    /// device as the DMA address base of the region.

    ///

    /// Returns `EINVAL` if `offset` is not within the bounds of the allocation.

    pub fn dma_handle_with_offset(&self, offset: usize) -> Result<bindings::dma_addr_t> {

        if offset >= self.count {

            Err(EINVAL)

        } else {

            // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits

            // into a `usize`, and `offset` is inferior to `count`.

            Ok(self.dma_handle + (offset * core::mem::size_of::<T>()) as bindings::dma_addr_t)

        }

    }



    /// Common helper to validate a range applied from the allocated region in the CPU's virtual

    /// address space.

    fn validate_range(&self, offset: usize, count: usize) -> Result {

        if offset.checked_add(count).ok_or(EOVERFLOW)? > self.count {

            return Err(EINVAL);

        }

        Ok(())

    }



    /// Returns the data from the region starting from `offset` as a slice.

    /// `offset` and `count` are in units of `T`, not the number of bytes.

    ///

    /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed,

    /// [`CoherentAllocation::start_ptr`] or [`CoherentAllocation::start_ptr_mut`] could be used

    /// instead.

    ///

    /// # Safety

    ///

    /// * Callers must ensure that the device does not read/write to/from memory while the returned

    ///   slice is live.

    /// * Callers must ensure that this call does not race with a write to the same region while

    ///   the returned slice is live.

    pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> {

        self.validate_range(offset, count)?;

        // SAFETY:

        // - The pointer is valid due to type invariant on `CoherentAllocation`,

        //   we've just checked that the range and index is within bounds. The immutability of the

        //   data is also guaranteed by the safety requirements of the function.

        // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked

        //   that `self.count` won't overflow early in the constructor.

        Ok(unsafe { core::slice::from_raw_parts(self.cpu_addr.add(offset), count) })

    }



    /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable

    /// slice is returned.

    ///

    /// # Safety

    ///

    /// * Callers must ensure that the device does not read/write to/from memory while the returned

    ///   slice is live.

    /// * Callers must ensure that this call does not race with a read or write to the same region

    ///   while the returned slice is live.

    pub unsafe fn as_slice_mut(&self, offset: usize, count: usize) -> Result<&mut [T]> {

        self.validate_range(offset, count)?;

        // SAFETY:

        // - The pointer is valid due to type invariant on `CoherentAllocation`,

        //   we've just checked that the range and index is within bounds. The immutability of the

        //   data is also guaranteed by the safety requirements of the function.

        // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked

        //   that `self.count` won't overflow early in the constructor.

        Ok(unsafe { core::slice::from_raw_parts_mut(self.cpu_addr.add(offset), count) })

    }



    /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the

    /// number of bytes.

    ///

    /// # Safety

    ///

    /// * Callers must ensure that the device does not read/write to/from memory while the returned

    ///   slice is live.

    /// * Callers must ensure that this call does not race with a read or write to the same region

    ///   that overlaps with this write.

    ///

    /// # Examples

    ///

    /// ```

    /// # fn test(alloc: &mut kernel::dma::CoherentAllocation<u8>) -> Result {

    /// let somedata: [u8; 4] = [0xf; 4];

    /// let buf: &[u8] = &somedata;

    /// // SAFETY: There is no concurrent HW operation on the device and no other R/W access to the

    /// // region.

    /// unsafe { alloc.write(buf, 0)?; }

    /// # Ok::<(), Error>(()) }

    /// ```

    pub unsafe fn write(&self, src: &[T], offset: usize) -> Result {

        self.validate_range(offset, src.len())?;

        // SAFETY:

        // - The pointer is valid due to type invariant on `CoherentAllocation`

        //   and we've just checked that the range and index is within bounds.

        // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked

        //   that `self.count` won't overflow early in the constructor.

        unsafe {

            core::ptr::copy_nonoverlapping(src.as_ptr(), self.cpu_addr.add(offset), src.len())

        };

        Ok(())

    }



    /// Returns a pointer to an element from the region with bounds checking. `offset` is in

    /// units of `T`, not the number of bytes.

    ///
@@ -328,20 +449,24 @@ unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} 
#[macro_export]

macro_rules! dma_read {

    ($dma:expr, $idx: expr, $($field:tt)*) => {{

        (|| -> ::core::result::Result<_, $crate::error::Error> {

            let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;

            // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be

            // dereferenced. The compiler also further validates the expression on whether `field`

            // is a member of `item` when expanded by the macro.

            unsafe {

                let ptr_field = ::core::ptr::addr_of!((*item) $($field)*);

                ::core::result::Result::Ok(

                    $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field)

                )

            }

        })()

    }};

    ($dma:ident [ $idx:expr ] $($field:tt)* ) => {

        $crate::dma_read!($dma, $idx, $($field)*);

        $crate::dma_read!($dma, $idx, $($field)*)

    };

    ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {

        $crate::dma_read!($($dma).*, $idx, $($field)*);

        $crate::dma_read!($($dma).*, $idx, $($field)*)

    };

}
@@ -368,17 +493,21 @@ macro_rules! dma_read { 
#[macro_export]

macro_rules! dma_write {

    ($dma:ident [ $idx:expr ] $($field:tt)*) => {{

        $crate::dma_write!($dma, $idx, $($field)*);

        $crate::dma_write!($dma, $idx, $($field)*)

    }};

    ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{

        $crate::dma_write!($($dma).*, $idx, $($field)*);

        $crate::dma_write!($($dma).*, $idx, $($field)*)

    }};

    ($dma:expr, $idx: expr, = $val:expr) => {

        (|| -> ::core::result::Result<_, $crate::error::Error> {

            let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;

            // SAFETY: `item_from_index` ensures that `item` is always a valid item.

            unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) }

            ::core::result::Result::Ok(())

        })()

    };

    ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => {

        (|| -> ::core::result::Result<_, $crate::error::Error> {

            let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;

            // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be

            // dereferenced. The compiler also further validates the expression on whether `field`
@@ -387,5 +516,7 @@ macro_rules! dma_write { 
                let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*);

                $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val)

            }

            ::core::result::Result::Ok(())

        })()

    };

}

samples/rust/rust_dma.rs

+15 −13
Original line number Diff line number Diff line
@@ -54,14 +54,10 @@ fn probe(pdev: &pci::Device<Core>, _info: &Self::IdInfo) -> Result<Pin<KBox<Self 
        let ca: CoherentAllocation<MyStruct> =

            CoherentAllocation::alloc_coherent(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?;



        || -> Result {

        for (i, value) in TEST_VALUES.into_iter().enumerate() {

                kernel::dma_write!(ca[i] = MyStruct::new(value.0, value.1));

            kernel::dma_write!(ca[i] = MyStruct::new(value.0, value.1))?;

        }



            Ok(())

        }()?;



        let drvdata = KBox::new(

            Self {

                pdev: pdev.into(),
@@ -78,13 +74,19 @@ impl Drop for DmaSampleDriver { 
    fn drop(&mut self) {

        dev_info!(self.pdev.as_ref(), "Unload DMA test driver.\n");



        let _ = || -> Result {

        for (i, value) in TEST_VALUES.into_iter().enumerate() {

                assert_eq!(kernel::dma_read!(self.ca[i].h), value.0);

                assert_eq!(kernel::dma_read!(self.ca[i].b), value.1);

            let val0 = kernel::dma_read!(self.ca[i].h);

            let val1 = kernel::dma_read!(self.ca[i].b);

            assert!(val0.is_ok());

            assert!(val1.is_ok());



            if let Ok(val0) = val0 {

                assert_eq!(val0, value.0);

            }

            if let Ok(val1) = val1 {

                assert_eq!(val1, value.1);

            }

        }

            Ok(())

        }();

    }

}