Merge tag 'drm-rust-fixes-2026-03-12' of... (b28913e8) · Commits · git / linux-net

drivers/gpu/nova-core/gsp.rs

+24 −22

Original line number	Diff line number	Diff line
		@@ -47,16 +47,12 @@
		unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {}

		impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> {
		/// Creates a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`.
		fn new(start: DmaAddress) -> Result<Self> {
		let mut ptes = [0u64; NUM_PAGES];
		for (i, pte) in ptes.iter_mut().enumerate() {
		*pte = start
		.checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT)
		.ok_or(EOVERFLOW)?;
		}

		Ok(Self(ptes))
		/// Returns the page table entry for `index`, for a mapping starting at `start`.
		// TODO: Replace with `IoView` projection once available.
		fn entry(start: DmaAddress, index: usize) -> Result<u64> {
		start
		.checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT)
		.ok_or(EOVERFLOW)
		}
		}

		@@ -86,16 +82,22 @@ fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
		NUM_PAGES * GSP_PAGE_SIZE,
		GFP_KERNEL \| __GFP_ZERO,
		)?);
		let ptes = PteArray::<NUM_PAGES>::new(obj.0.dma_handle())?;

		let start_addr = obj.0.dma_handle();

		// SAFETY: `obj` has just been created and we are its sole user.
		unsafe {
		// Copy the self-mapping PTE at the expected location.
		let pte_region = unsafe {
		obj.0
		.as_slice_mut(size_of::<u64>(), size_of_val(&ptes))?
		.copy_from_slice(ptes.as_bytes())
		.as_slice_mut(size_of::<u64>(), NUM_PAGES * size_of::<u64>())?
		};

		// Write values one by one to avoid an on-stack instance of `PteArray`.
		for (i, chunk) in pte_region.chunks_exact_mut(size_of::<u64>()).enumerate() {
		let pte_value = PteArray::<0>::entry(start_addr, i)?;

		chunk.copy_from_slice(&pte_value.to_ne_bytes());
		}

		Ok(obj)
		}
		}
		@@ -143,14 +145,14 @@ pub(crate) fn new(pdev: &pci::Device<device::Bound>) -> impl PinInit<Self, Error
		// _kgspInitLibosLoggingStructures (allocates memory for buffers)
		// kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)
		dma_write!(
		libos[0] = LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0)
		)?;
		libos, [0]?, LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0)
		);
		dma_write!(
		libos[1] = LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0)
		)?;
		dma_write!(libos[2] = LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0))?;
		dma_write!(rmargs[0].inner = fw::GspArgumentsCached::new(cmdq))?;
		dma_write!(libos[3] = LibosMemoryRegionInitArgument::new("RMARGS", rmargs))?;
		libos, [1]?, LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0)
		);
		dma_write!(libos, [2]?, LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0));
		dma_write!(rmargs, [0]?.inner, fw::GspArgumentsCached::new(cmdq));
		dma_write!(libos, [3]?, LibosMemoryRegionInitArgument::new("RMARGS", rmargs));
		},
		}))
		})

drivers/gpu/nova-core/gsp/boot.rs

+1 −1

Original line number	Diff line number	Diff line
		@@ -157,7 +157,7 @@ pub(crate) fn boot(

		let wpr_meta =
		CoherentAllocation::<GspFwWprMeta>::alloc_coherent(dev, 1, GFP_KERNEL \| __GFP_ZERO)?;
		dma_write!(wpr_meta[0] = GspFwWprMeta::new(&gsp_fw, &fb_layout))?;
		dma_write!(wpr_meta, [0]?, GspFwWprMeta::new(&gsp_fw, &fb_layout));

		self.cmdq
		.send_command(bar, commands::SetSystemInfo::new(pdev))?;

drivers/gpu/nova-core/gsp/cmdq.rs

+33 −60

Original line number	Diff line number	Diff line
		@@ -2,11 +2,7 @@

		use core::{
		cmp,
		mem,
		sync::atomic::{
		fence,
		Ordering, //
		}, //
		mem, //
		};

		use kernel::{
		@@ -146,30 +142,36 @@ struct MsgqData {
		#[repr(C)]
		// There is no struct defined for this in the open-gpu-kernel-source headers.
		// Instead it is defined by code in `GspMsgQueuesInit()`.
		struct Msgq {
		// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module.
		pub(super) struct Msgq {
		/// Header for sending messages, including the write pointer.
		tx: MsgqTxHeader,
		pub(super) tx: MsgqTxHeader,
		/// Header for receiving messages, including the read pointer.
		rx: MsgqRxHeader,
		pub(super) rx: MsgqRxHeader,
		/// The message queue proper.
		msgq: MsgqData,
		}

		/// Structure shared between the driver and the GSP and containing the command and message queues.
		#[repr(C)]
		struct GspMem {
		// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module.
		pub(super) struct GspMem {
		/// Self-mapping page table entries.
		ptes: PteArray<{ GSP_PAGE_SIZE / size_of::<u64>() }>,
		ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>,
		/// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the
		/// write and read pointers that the CPU updates.
		///
		/// This member is read-only for the GSP.
		cpuq: Msgq,
		pub(super) cpuq: Msgq,
		/// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the
		/// write and read pointers that the GSP updates.
		///
		/// This member is read-only for the driver.
		gspq: Msgq,
		pub(super) gspq: Msgq,
		}

		impl GspMem {
		const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>();
		}

		// SAFETY: These structs don't meet the no-padding requirements of AsBytes but
		@@ -201,9 +203,19 @@ fn new(dev: &device::Device<device::Bound>) -> Result<Self> {

		let gsp_mem =
		CoherentAllocation::<GspMem>::alloc_coherent(dev, 1, GFP_KERNEL \| __GFP_ZERO)?;
		dma_write!(gsp_mem[0].ptes = PteArray::new(gsp_mem.dma_handle())?)?;
		dma_write!(gsp_mem[0].cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES))?;
		dma_write!(gsp_mem[0].cpuq.rx = MsgqRxHeader::new())?;

		let start = gsp_mem.dma_handle();
		// Write values one by one to avoid an on-stack instance of `PteArray`.
		for i in 0..GspMem::PTE_ARRAY_SIZE {
		dma_write!(gsp_mem, [0]?.ptes.0[i], PteArray::<0>::entry(start, i)?);
		}

		dma_write!(
		gsp_mem,
		[0]?.cpuq.tx,
		MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES)
		);
		dma_write!(gsp_mem, [0]?.cpuq.rx, MsgqRxHeader::new());

		Ok(Self(gsp_mem))
		}
		@@ -317,12 +329,7 @@ fn allocate_command(&mut self, size: usize) -> Result<GspCommand<'_>> {
		//
		// - The returned value is between `0` and `MSGQ_NUM_PAGES`.
		fn gsp_write_ptr(&self) -> u32 {
		let gsp_mem = self.0.start_ptr();

		// SAFETY:
		// - The 'CoherentAllocation' contains at least one object.
		// - By the invariants of `CoherentAllocation` the pointer is valid.
		(unsafe { (*gsp_mem).gspq.tx.write_ptr() } % MSGQ_NUM_PAGES)
		super::fw::gsp_mem::gsp_write_ptr(&self.0)
		}

		// Returns the index of the memory page the GSP will read the next command from.
		@@ -331,12 +338,7 @@ fn gsp_write_ptr(&self) -> u32 {
		//
		// - The returned value is between `0` and `MSGQ_NUM_PAGES`.
		fn gsp_read_ptr(&self) -> u32 {
		let gsp_mem = self.0.start_ptr();

		// SAFETY:
		// - The 'CoherentAllocation' contains at least one object.
		// - By the invariants of `CoherentAllocation` the pointer is valid.
		(unsafe { (*gsp_mem).gspq.rx.read_ptr() } % MSGQ_NUM_PAGES)
		super::fw::gsp_mem::gsp_read_ptr(&self.0)
		}

		// Returns the index of the memory page the CPU can read the next message from.
		@@ -345,27 +347,12 @@ fn gsp_read_ptr(&self) -> u32 {
		//
		// - The returned value is between `0` and `MSGQ_NUM_PAGES`.
		fn cpu_read_ptr(&self) -> u32 {
		let gsp_mem = self.0.start_ptr();

		// SAFETY:
		// - The ['CoherentAllocation'] contains at least one object.
		// - By the invariants of CoherentAllocation the pointer is valid.
		(unsafe { (*gsp_mem).cpuq.rx.read_ptr() } % MSGQ_NUM_PAGES)
		super::fw::gsp_mem::cpu_read_ptr(&self.0)
		}

		// Informs the GSP that it can send `elem_count` new pages into the message queue.
		fn advance_cpu_read_ptr(&mut self, elem_count: u32) {
		let rptr = self.cpu_read_ptr().wrapping_add(elem_count) % MSGQ_NUM_PAGES;

		// Ensure read pointer is properly ordered.
		fence(Ordering::SeqCst);

		let gsp_mem = self.0.start_ptr_mut();

		// SAFETY:
		// - The 'CoherentAllocation' contains at least one object.
		// - By the invariants of `CoherentAllocation` the pointer is valid.
		unsafe { (*gsp_mem).cpuq.rx.set_read_ptr(rptr) };
		super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count)
		}

		// Returns the index of the memory page the CPU can write the next command to.
		@@ -374,26 +361,12 @@ fn advance_cpu_read_ptr(&mut self, elem_count: u32) {
		//
		// - The returned value is between `0` and `MSGQ_NUM_PAGES`.
		fn cpu_write_ptr(&self) -> u32 {
		let gsp_mem = self.0.start_ptr();

		// SAFETY:
		// - The 'CoherentAllocation' contains at least one object.
		// - By the invariants of `CoherentAllocation` the pointer is valid.
		(unsafe { (*gsp_mem).cpuq.tx.write_ptr() } % MSGQ_NUM_PAGES)
		super::fw::gsp_mem::cpu_write_ptr(&self.0)
		}

		// Informs the GSP that it can process `elem_count` new pages from the command queue.
		fn advance_cpu_write_ptr(&mut self, elem_count: u32) {
		let wptr = self.cpu_write_ptr().wrapping_add(elem_count) & MSGQ_NUM_PAGES;
		let gsp_mem = self.0.start_ptr_mut();

		// SAFETY:
		// - The 'CoherentAllocation' contains at least one object.
		// - By the invariants of `CoherentAllocation` the pointer is valid.
		unsafe { (*gsp_mem).cpuq.tx.set_write_ptr(wptr) };

		// Ensure all command data is visible before triggering the GSP read.
		fence(Ordering::SeqCst);
		super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count)
		}
		}

drivers/gpu/nova-core/gsp/fw.rs

+69 −32

Original line number	Diff line number	Diff line
		@@ -40,6 +40,75 @@
		},
		};

		// TODO: Replace with `IoView` projections once available; the `unwrap()` calls go away once we
		// switch to the new `dma::Coherent` API.
		pub(super) mod gsp_mem {
		use core::sync::atomic::{
		fence,
		Ordering, //
		};

		use kernel::{
		dma::CoherentAllocation,
		dma_read,
		dma_write,
		prelude::*, //
		};

		use crate::gsp::cmdq::{
		GspMem,
		MSGQ_NUM_PAGES, //
		};

		pub(in crate::gsp) fn gsp_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
		// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		\|\| -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap()
		}

		pub(in crate::gsp) fn gsp_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
		// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		\|\| -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap()
		}

		pub(in crate::gsp) fn cpu_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
		// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		\|\| -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap()
		}

		pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &CoherentAllocation<GspMem>, count: u32) {
		let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES;

		// Ensure read pointer is properly ordered.
		fence(Ordering::SeqCst);

		// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		\|\| -> Result {
		dma_write!(qs, [0]?.cpuq.rx.0.readPtr, rptr);
		Ok(())
		}()
		.unwrap()
		}

		pub(in crate::gsp) fn cpu_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
		// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		\|\| -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap()
		}

		pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &CoherentAllocation<GspMem>, count: u32) {
		let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES;

		// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		\|\| -> Result {
		dma_write!(qs, [0]?.cpuq.tx.0.writePtr, wptr);
		Ok(())
		}()
		.unwrap();

		// Ensure all command data is visible before triggering the GSP read.
		fence(Ordering::SeqCst);
		}
		}

		/// Empty type to group methods related to heap parameters for running the GSP firmware.
		enum GspFwHeapParams {}

		@@ -708,22 +777,6 @@ pub(crate) fn new(msgq_size: u32, rx_hdr_offset: u32, msg_count: u32) -> Self {
		entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(),
		})
		}

		/// Returns the value of the write pointer for this queue.
		pub(crate) fn write_ptr(&self) -> u32 {
		let ptr = core::ptr::from_ref(&self.0.writePtr);

		// SAFETY: `ptr` is a valid pointer to a `u32`.
		unsafe { ptr.read_volatile() }
		}

		/// Sets the value of the write pointer for this queue.
		pub(crate) fn set_write_ptr(&mut self, val: u32) {
		let ptr = core::ptr::from_mut(&mut self.0.writePtr);

		// SAFETY: `ptr` is a valid pointer to a `u32`.
		unsafe { ptr.write_volatile(val) }
		}
		}

		// SAFETY: Padding is explicit and does not contain uninitialized data.
		@@ -739,22 +792,6 @@ impl MsgqRxHeader {
		pub(crate) fn new() -> Self {
		Self(Default::default())
		}

		/// Returns the value of the read pointer for this queue.
		pub(crate) fn read_ptr(&self) -> u32 {
		let ptr = core::ptr::from_ref(&self.0.readPtr);

		// SAFETY: `ptr` is a valid pointer to a `u32`.
		unsafe { ptr.read_volatile() }
		}

		/// Sets the value of the read pointer for this queue.
		pub(crate) fn set_read_ptr(&mut self, val: u32) {
		let ptr = core::ptr::from_mut(&mut self.0.readPtr);

		// SAFETY: `ptr` is a valid pointer to a `u32`.
		unsafe { ptr.write_volatile(val) }
		}
		}

		// SAFETY: Padding is explicit and does not contain uninitialized data.

rust/kernel/dma.rs

+50 −64

Original line number	Diff line number	Diff line
		@@ -461,6 +461,19 @@ pub fn size(&self) -> usize {
		self.count * core::mem::size_of::<T>()
		}

		/// Returns the raw pointer to the allocated region in the CPU's virtual address space.
		#[inline]
		pub fn as_ptr(&self) -> *const [T] {
		core::ptr::slice_from_raw_parts(self.cpu_addr.as_ptr(), self.count)
		}

		/// Returns the raw pointer to the allocated region in the CPU's virtual address space as
		/// a mutable pointer.
		#[inline]
		pub fn as_mut_ptr(&self) -> *mut [T] {
		core::ptr::slice_from_raw_parts_mut(self.cpu_addr.as_ptr(), self.count)
		}

		/// Returns the base address to the allocated region in the CPU's virtual address space.
		pub fn start_ptr(&self) -> *const T {
		self.cpu_addr.as_ptr()
		@@ -581,23 +594,6 @@ pub unsafe fn write(&mut self, src: &[T], offset: usize) -> Result {
		Ok(())
		}

		/// Returns a pointer to an element from the region with bounds checking. `offset` is in
		/// units of `T`, not the number of bytes.
		///
		/// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros.
		#[doc(hidden)]
		pub fn item_from_index(&self, offset: usize) -> Result<*mut T> {
		if offset >= self.count {
		return Err(EINVAL);
		}
		// SAFETY:
		// - The pointer is valid due to type invariant on `CoherentAllocation`
		// and we've just checked that the range and index is within bounds.
		// - `offset` can't overflow since it is smaller than `self.count` and we've checked
		// that `self.count` won't overflow early in the constructor.
		Ok(unsafe { self.cpu_addr.as_ptr().add(offset) })
		}

		/// Reads the value of `field` and ensures that its type is [`FromBytes`].
		///
		/// # Safety
		@@ -670,6 +666,9 @@ unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {}

		/// Reads a field of an item from an allocated region of structs.
		///
		/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating
		/// to a [`CoherentAllocation`] and `proj` is a [projection specification](kernel::ptr::project!).
		///
		/// # Examples
		///
		/// ```
		@@ -684,36 +683,29 @@ unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {}
		/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
		///
		/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
		/// let whole = kernel::dma_read!(alloc[2]);
		/// let field = kernel::dma_read!(alloc[1].field);
		/// let whole = kernel::dma_read!(alloc, [2]?);
		/// let field = kernel::dma_read!(alloc, [1]?.field);
		/// # Ok::<(), Error>(()) }
		/// ```
		#[macro_export]
		macro_rules! dma_read {
		($dma:expr, $idx: expr, $($field:tt)*) => {{
		(\|\| -> ::core::result::Result<_, $crate::error::Error> {
		let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
		// SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
		// dereferenced. The compiler also further validates the expression on whether `field`
		// is a member of `item` when expanded by the macro.
		unsafe {
		let ptr_field = ::core::ptr::addr_of!((item) $($field));
		::core::result::Result::Ok(
		$crate::dma::CoherentAllocation::field_read(&$dma, ptr_field)
		)
		}
		})()
		($dma:expr, $($proj:tt)*) => {{
		let dma = &$dma;
		let ptr = $crate::ptr::project!(
		$crate::dma::CoherentAllocation::as_ptr(dma), $($proj)*
		);
		// SAFETY: The pointer created by the projection is within the DMA region.
		unsafe { $crate::dma::CoherentAllocation::field_read(dma, ptr) }
		}};
		($dma:ident [ $idx:expr ] $($field:tt)* ) => {
		$crate::dma_read!($dma, $idx, $($field)*)
		};
		($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {
		$crate::dma_read!($($dma)., $idx, $($field))
		};
		}

		/// Writes to a field of an item from an allocated region of structs.
		///
		/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression
		/// evaluating to a [`CoherentAllocation`], `proj` is a
		/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the
		/// projected location.
		///
		/// # Examples
		///
		/// ```
		@@ -728,37 +720,31 @@ macro_rules! dma_read {
		/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
		///
		/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
		/// kernel::dma_write!(alloc[2].member = 0xf);
		/// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf });
		/// kernel::dma_write!(alloc, [2]?.member, 0xf);
		/// kernel::dma_write!(alloc, [1]?, MyStruct { member: 0xf });
		/// # Ok::<(), Error>(()) }
		/// ```
		#[macro_export]
		macro_rules! dma_write {
		($dma:ident [ $idx:expr ] $($field:tt)*) => {{
		$crate::dma_write!($dma, $idx, $($field)*)
		}};
		($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{
		$crate::dma_write!($($dma)., $idx, $($field))
		(@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{
		let dma = &$dma;
		let ptr = $crate::ptr::project!(
		mut $crate::dma::CoherentAllocation::as_mut_ptr(dma), $($proj)*
		);
		let val = $val;
		// SAFETY: The pointer created by the projection is within the DMA region.
		unsafe { $crate::dma::CoherentAllocation::field_write(dma, ptr, val) }
		}};
		($dma:expr, $idx: expr, = $val:expr) => {
		(\|\| -> ::core::result::Result<_, $crate::error::Error> {
		let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
		// SAFETY: `item_from_index` ensures that `item` is always a valid item.
		unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) }
		::core::result::Result::Ok(())
		})()
		(@parse [$dma:expr] [$($proj:tt)] [.$field:tt $($rest:tt)]) => {
		$crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*])
		};
		($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => {
		(\|\| -> ::core::result::Result<_, $crate::error::Error> {
		let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
		// SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
		// dereferenced. The compiler also further validates the expression on whether `field`
		// is a member of `item` when expanded by the macro.
		unsafe {
		let ptr_field = ::core::ptr::addr_of_mut!((item) $(.$field));
		$crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val)
		}
		::core::result::Result::Ok(())
		})()
		(@parse [$dma:expr] [$($proj:tt)] [[$index:expr]? $($rest:tt)]) => {
		$crate::dma_write!(@parse [$dma] [$($proj)* [$index]?] [$($rest)*])
		};
		(@parse [$dma:expr] [$($proj:tt)] [[$index:expr] $($rest:tt)]) => {
		$crate::dma_write!(@parse [$dma] [$($proj)* [$index]] [$($rest)*])
		};
		($dma:expr, $($rest:tt)*) => {
		$crate::dma_write!(@parse [$dma] [] [$($rest)*])
		};
		}