Merge tag 'rust-timekeeping-v6.18' of https://github.com/Rust-for-Linux/linux into rust-next (cfe872eb) · Commits · git / linux-net

rust/kernel/time.rs

+162 −1

Original line number	Diff line number	Diff line
		@@ -25,6 +25,7 @@
		//! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h).

		use core::marker::PhantomData;
		use core::ops;

		pub mod delay;
		pub mod hrtimer;
		@@ -200,9 +201,31 @@ pub fn elapsed(&self) -> Delta {
		pub(crate) fn as_nanos(&self) -> i64 {
		self.inner
		}

		/// Create an [`Instant`] from a `ktime_t` without checking if it is non-negative.
		///
		/// # Panics
		///
		/// On debug builds, this function will panic if `ktime` is not in the range from 0 to
		/// `KTIME_MAX`.
		///
		/// # Safety
		///
		/// The caller promises that `ktime` is in the range from 0 to `KTIME_MAX`.
		#[inline]
		pub(crate) unsafe fn from_ktime(ktime: bindings::ktime_t) -> Self {
		debug_assert!(ktime >= 0);

		// INVARIANT: Our safety contract ensures that `ktime` is in the range from 0 to
		// `KTIME_MAX`.
		Self {
		inner: ktime,
		_c: PhantomData,
		}
		}
		}

		impl<C: ClockSource> core::ops::Sub for Instant<C> {
		impl<C: ClockSource> ops::Sub for Instant<C> {
		type Output = Delta;

		// By the type invariant, it never overflows.
		@@ -214,6 +237,46 @@ fn sub(self, other: Instant<C>) -> Delta {
		}
		}

		impl<T: ClockSource> ops::Add<Delta> for Instant<T> {
		type Output = Self;

		#[inline]
		fn add(self, rhs: Delta) -> Self::Output {
		// INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow
		// (e.g. go above `KTIME_MAX`)
		let res = self.inner + rhs.nanos;

		// INVARIANT: With overflow checks enabled, we verify here that the value is >= 0
		#[cfg(CONFIG_RUST_OVERFLOW_CHECKS)]
		assert!(res >= 0);

		Self {
		inner: res,
		_c: PhantomData,
		}
		}
		}

		impl<T: ClockSource> ops::Sub<Delta> for Instant<T> {
		type Output = Self;

		#[inline]
		fn sub(self, rhs: Delta) -> Self::Output {
		// INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow
		// (e.g. go above `KTIME_MAX`)
		let res = self.inner - rhs.nanos;

		// INVARIANT: With overflow checks enabled, we verify here that the value is >= 0
		#[cfg(CONFIG_RUST_OVERFLOW_CHECKS)]
		assert!(res >= 0);

		Self {
		inner: res,
		_c: PhantomData,
		}
		}
		}

		/// A span of time.
		///
		/// This struct represents a span of time, with its value stored as nanoseconds.
		@@ -224,6 +287,78 @@ pub struct Delta {
		nanos: i64,
		}

		impl ops::Add for Delta {
		type Output = Self;

		#[inline]
		fn add(self, rhs: Self) -> Self {
		Self {
		nanos: self.nanos + rhs.nanos,
		}
		}
		}

		impl ops::AddAssign for Delta {
		#[inline]
		fn add_assign(&mut self, rhs: Self) {
		self.nanos += rhs.nanos;
		}
		}

		impl ops::Sub for Delta {
		type Output = Self;

		#[inline]
		fn sub(self, rhs: Self) -> Self::Output {
		Self {
		nanos: self.nanos - rhs.nanos,
		}
		}
		}

		impl ops::SubAssign for Delta {
		#[inline]
		fn sub_assign(&mut self, rhs: Self) {
		self.nanos -= rhs.nanos;
		}
		}

		impl ops::Mul<i64> for Delta {
		type Output = Self;

		#[inline]
		fn mul(self, rhs: i64) -> Self::Output {
		Self {
		nanos: self.nanos * rhs,
		}
		}
		}

		impl ops::MulAssign<i64> for Delta {
		#[inline]
		fn mul_assign(&mut self, rhs: i64) {
		self.nanos *= rhs;
		}
		}

		impl ops::Div for Delta {
		type Output = i64;

		#[inline]
		fn div(self, rhs: Self) -> Self::Output {
		#[cfg(CONFIG_64BIT)]
		{
		self.nanos / rhs.nanos
		}

		#[cfg(not(CONFIG_64BIT))]
		{
		// SAFETY: This function is always safe to call regardless of the input values
		unsafe { bindings::div64_s64(self.nanos, rhs.nanos) }
		}
		}
		}

		impl Delta {
		/// A span of time equal to zero.
		pub const ZERO: Self = Self { nanos: 0 };
		@@ -312,4 +447,30 @@ pub fn as_millis(self) -> i64 {
		bindings::ktime_to_ms(self.as_nanos())
		}
		}

		/// Return `self % dividend` where `dividend` is in nanoseconds.
		///
		/// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is
		/// limited to 32 bit dividends.
		#[inline]
		pub fn rem_nanos(self, dividend: i32) -> Self {
		#[cfg(CONFIG_64BIT)]
		{
		Self {
		nanos: self.as_nanos() % i64::from(dividend),
		}
		}

		#[cfg(not(CONFIG_64BIT))]
		{
		let mut rem = 0;

		// SAFETY: `rem` is in the stack, so we can always provide a valid pointer to it.
		unsafe { bindings::div_s64_rem(self.as_nanos(), dividend, &mut rem) };

		Self {
		nanos: i64::from(rem),
		}
		}
		}
		}

rust/kernel/time/hrtimer.rs

+149 −3

Original line number	Diff line number	Diff line
		@@ -69,9 +69,14 @@

		use super::{ClockSource, Delta, Instant};
		use crate::{prelude::*, types::Opaque};
		use core::marker::PhantomData;
		use core::{marker::PhantomData, ptr::NonNull};
		use pin_init::PinInit;

		/// A type-alias to refer to the [`Instant<C>`] for a given `T` from [`HrTimer<T>`].
		///
		/// Where `C` is the [`ClockSource`] of the [`HrTimer`].
		pub type HrTimerInstant<T> = Instant<<<T as HasHrTimer<T>>::TimerMode as HrTimerMode>::Clock>;

		/// A timer backed by a C `struct hrtimer`.
		///
		/// # Invariants
		@@ -163,6 +168,84 @@ pub(crate) unsafe fn raw_cancel(this: *const Self) -> bool {
		// handled on the C side.
		unsafe { bindings::hrtimer_cancel(c_timer_ptr) != 0 }
		}

		/// Forward the timer expiry for a given timer pointer.
		///
		/// # Safety
		///
		/// - `self_ptr` must point to a valid `Self`.
		/// - The caller must either have exclusive access to the data pointed at by `self_ptr`, or be
		/// within the context of the timer callback.
		#[inline]
		unsafe fn raw_forward(self_ptr: *mut Self, now: HrTimerInstant<T>, interval: Delta) -> u64
		where
		T: HasHrTimer<T>,
		{
		// SAFETY:
		// * The C API requirements for this function are fulfilled by our safety contract.
		// * `self_ptr` is guaranteed to point to a valid `Self` via our safety contract
		unsafe {
		bindings::hrtimer_forward(Self::raw_get(self_ptr), now.as_nanos(), interval.as_nanos())
		}
		}

		/// Conditionally forward the timer.
		///
		/// If the timer expires after `now`, this function does nothing and returns 0. If the timer
		/// expired at or before `now`, this function forwards the timer by `interval` until the timer
		/// expires after `now` and then returns the number of times the timer was forwarded by
		/// `interval`.
		///
		/// This function is mainly useful for timer types which can provide exclusive access to the
		/// timer when the timer is not running. For forwarding the timer from within the timer callback
		/// context, see [`HrTimerCallbackContext::forward()`].
		///
		/// Returns the number of overruns that occurred as a result of the timer expiry change.
		pub fn forward(self: Pin<&mut Self>, now: HrTimerInstant<T>, interval: Delta) -> u64
		where
		T: HasHrTimer<T>,
		{
		// SAFETY: `raw_forward` does not move `Self`
		let this = unsafe { self.get_unchecked_mut() };

		// SAFETY: By existence of `Pin<&mut Self>`, the pointer passed to `raw_forward` points to a
		// valid `Self` that we have exclusive access to.
		unsafe { Self::raw_forward(this, now, interval) }
		}

		/// Conditionally forward the timer.
		///
		/// This is a variant of [`forward()`](Self::forward) that uses an interval after the current
		/// time of the base clock for the [`HrTimer`].
		pub fn forward_now(self: Pin<&mut Self>, interval: Delta) -> u64
		where
		T: HasHrTimer<T>,
		{
		self.forward(HrTimerInstant::<T>::now(), interval)
		}

		/// Return the time expiry for this [`HrTimer`].
		///
		/// This value should only be used as a snapshot, as the actual expiry time could change after
		/// this function is called.
		pub fn expires(&self) -> HrTimerInstant<T>
		where
		T: HasHrTimer<T>,
		{
		// SAFETY: `self` is an immutable reference and thus always points to a valid `HrTimer`.
		let c_timer_ptr = unsafe { HrTimer::raw_get(self) };

		// SAFETY:
		// - Timers cannot have negative ktime_t values as their expiration time.
		// - There's no actual locking here, a racy read is fine and expected
		unsafe {
		Instant::from_ktime(
		// This `read_volatile` is intended to correspond to a READ_ONCE call.
		// FIXME(read_once): Replace with `read_once` when available on the Rust side.
		core::ptr::read_volatile(&raw const ((*c_timer_ptr).node.expires)),
		)
		}
		}
		}

		/// Implemented by pointer types that point to structs that contain a [`HrTimer`].
		@@ -300,9 +383,13 @@ pub trait HrTimerCallback {
		type Pointer<'a>: RawHrTimerCallback;

		/// Called by the timer logic when the timer fires.
		fn run(this: <Self::Pointer<'_> as RawHrTimerCallback>::CallbackTarget<'_>) -> HrTimerRestart
		fn run(
		this: <Self::Pointer<'_> as RawHrTimerCallback>::CallbackTarget<'_>,
		ctx: HrTimerCallbackContext<'_, Self>,
		) -> HrTimerRestart
		where
		Self: Sized;
		Self: Sized,
		Self: HasHrTimer<Self>;
		}

		/// A handle representing a potentially running timer.
		@@ -324,6 +411,8 @@ pub unsafe trait HrTimerHandle {
		/// Note that the timer might be started by a concurrent start operation. If
		/// so, the timer might not be in the stopped state when this function
		/// returns.
		///
		/// Returns `true` if the timer was running.
		fn cancel(&mut self) -> bool;
		}

		@@ -585,6 +674,63 @@ impl<C: ClockSource> HrTimerMode for RelativePinnedHardMode<C> {
		type Expires = Delta;
		}

		/// Privileged smart-pointer for a [`HrTimer`] callback context.
		///
		/// Many [`HrTimer`] methods can only be called in two situations:
		///
		/// * When the caller has exclusive access to the `HrTimer` and the `HrTimer` is guaranteed not to
		/// be running.
		/// * From within the context of an `HrTimer`'s callback method.
		///
		/// This type provides access to said methods from within a timer callback context.
		///
		/// # Invariants
		///
		/// * The existence of this type means the caller is currently within the callback for an
		/// [`HrTimer`].
		/// * `self.0` always points to a live instance of [`HrTimer<T>`].
		pub struct HrTimerCallbackContext<'a, T: HasHrTimer<T>>(NonNull<HrTimer<T>>, PhantomData<&'a ()>);

		impl<'a, T: HasHrTimer<T>> HrTimerCallbackContext<'a, T> {
		/// Create a new [`HrTimerCallbackContext`].
		///
		/// # Safety
		///
		/// This function relies on the caller being within the context of a timer callback, so it must
		/// not be used anywhere except for within implementations of [`RawHrTimerCallback::run`]. The
		/// caller promises that `timer` points to a valid initialized instance of
		/// [`bindings::hrtimer`].
		///
		/// The returned `Self` must not outlive the function context of [`RawHrTimerCallback::run`]
		/// where this function is called.
		pub(crate) unsafe fn from_raw(timer: *mut HrTimer<T>) -> Self {
		// SAFETY: The caller guarantees `timer` is a valid pointer to an initialized
		// `bindings::hrtimer`
		// INVARIANT: Our safety contract ensures that we're within the context of a timer callback
		// and that `timer` points to a live instance of `HrTimer<T>`.
		Self(unsafe { NonNull::new_unchecked(timer) }, PhantomData)
		}

		/// Conditionally forward the timer.
		///
		/// This function is identical to [`HrTimer::forward()`] except that it may only be used from
		/// within the context of a [`HrTimer`] callback.
		pub fn forward(&mut self, now: HrTimerInstant<T>, interval: Delta) -> u64 {
		// SAFETY:
		// - We are guaranteed to be within the context of a timer callback by our type invariants
		// - By our type invariants, `self.0` always points to a valid `HrTimer<T>`
		unsafe { HrTimer::<T>::raw_forward(self.0.as_ptr(), now, interval) }
		}

		/// Conditionally forward the timer.
		///
		/// This is a variant of [`HrTimerCallbackContext::forward()`] that uses an interval after the
		/// current time of the base clock for the [`HrTimer`].
		pub fn forward_now(&mut self, duration: Delta) -> u64 {
		self.forward(HrTimerInstant::<T>::now(), duration)
		}
		}

		/// Use to implement the [`HasHrTimer<T>`] trait.
		///
		/// See [`module`] documentation for an example.

rust/kernel/time/hrtimer/arc.rs

+8 −1

Original line number	Diff line number	Diff line
		@@ -3,6 +3,7 @@
		use super::HasHrTimer;
		use super::HrTimer;
		use super::HrTimerCallback;
		use super::HrTimerCallbackContext;
		use super::HrTimerHandle;
		use super::HrTimerMode;
		use super::HrTimerPointer;
		@@ -99,6 +100,12 @@ impl<T> RawHrTimerCallback for Arc<T>
		// allocation from other `Arc` clones.
		let receiver = unsafe { ArcBorrow::from_raw(data_ptr) };

		T::run(receiver).into_c()
		// SAFETY:
		// - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so
		// it is a valid pointer to a `HrTimer<T>` embedded in a `T`.
		// - We are within `RawHrTimerCallback::run`
		let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) };

		T::run(receiver, context).into_c()
		}
		}

rust/kernel/time/hrtimer/pin.rs

+8 −1

Original line number	Diff line number	Diff line
		@@ -3,6 +3,7 @@
		use super::HasHrTimer;
		use super::HrTimer;
		use super::HrTimerCallback;
		use super::HrTimerCallbackContext;
		use super::HrTimerHandle;
		use super::HrTimerMode;
		use super::RawHrTimerCallback;
		@@ -103,6 +104,12 @@ impl<'a, T> RawHrTimerCallback for Pin<&'a T>
		// here.
		let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) };

		T::run(receiver_pin).into_c()
		// SAFETY:
		// - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so
		// it is a valid pointer to a `HrTimer<T>` embedded in a `T`.
		// - We are within `RawHrTimerCallback::run`
		let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) };

		T::run(receiver_pin, context).into_c()
		}
		}

rust/kernel/time/hrtimer/pin_mut.rs

+9 −3

Original line number	Diff line number	Diff line
		// SPDX-License-Identifier: GPL-2.0

		use super::{
		HasHrTimer, HrTimer, HrTimerCallback, HrTimerHandle, HrTimerMode, RawHrTimerCallback,
		UnsafeHrTimerPointer,
		HasHrTimer, HrTimer, HrTimerCallback, HrTimerCallbackContext, HrTimerHandle, HrTimerMode,
		RawHrTimerCallback, UnsafeHrTimerPointer,
		};
		use core::{marker::PhantomData, pin::Pin, ptr::NonNull};

		@@ -107,6 +107,12 @@ impl<'a, T> RawHrTimerCallback for Pin<&'a mut T>
		// here.
		let receiver_pin = unsafe { Pin::new_unchecked(receiver_ref) };

		T::run(receiver_pin).into_c()
		// SAFETY:
		// - By C API contract `timer_ptr` is the pointer that we passed when queuing the timer, so
		// it is a valid pointer to a `HrTimer<T>` embedded in a `T`.
		// - We are within `RawHrTimerCallback::run`
		let context = unsafe { HrTimerCallbackContext::from_raw(timer_ptr) };

		T::run(receiver_pin, context).into_c()
		}
		}