summaryrefslogtreecommitdiff
path: root/rust/kernel/workqueue.rs
diff options
context:
space:
mode:
Diffstat (limited to 'rust/kernel/workqueue.rs')
-rw-r--r--rust/kernel/workqueue.rs190
1 files changed, 108 insertions, 82 deletions
diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs
index 480cb292e7c2..d092112d843f 100644
--- a/rust/kernel/workqueue.rs
+++ b/rust/kernel/workqueue.rs
@@ -33,7 +33,6 @@
//! we do not need to specify ids for the fields.
//!
//! ```
-//! use kernel::prelude::*;
//! use kernel::sync::Arc;
//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem};
//!
@@ -53,7 +52,7 @@
//! Arc::pin_init(pin_init!(MyStruct {
//! value,
//! work <- new_work!("MyStruct::work"),
-//! }))
+//! }), GFP_KERNEL)
//! }
//! }
//!
@@ -61,7 +60,7 @@
//! type Pointer = Arc<MyStruct>;
//!
//! fn run(this: Arc<MyStruct>) {
-//! pr_info!("The value is: {}", this.value);
+//! pr_info!("The value is: {}\n", this.value);
//! }
//! }
//!
@@ -70,12 +69,12 @@
//! fn print_later(val: Arc<MyStruct>) {
//! let _ = workqueue::system().enqueue(val);
//! }
+//! # print_later(MyStruct::new(42).unwrap());
//! ```
//!
//! The following example shows how multiple `work_struct` fields can be used:
//!
//! ```
-//! use kernel::prelude::*;
//! use kernel::sync::Arc;
//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem};
//!
@@ -101,7 +100,7 @@
//! value_2,
//! work_1 <- new_work!("MyStruct::work_1"),
//! work_2 <- new_work!("MyStruct::work_2"),
-//! }))
+//! }), GFP_KERNEL)
//! }
//! }
//!
@@ -109,7 +108,7 @@
//! type Pointer = Arc<MyStruct>;
//!
//! fn run(this: Arc<MyStruct>) {
-//! pr_info!("The value is: {}", this.value_1);
+//! pr_info!("The value is: {}\n", this.value_1);
//! }
//! }
//!
@@ -117,7 +116,7 @@
//! type Pointer = Arc<MyStruct>;
//!
//! fn run(this: Arc<MyStruct>) {
-//! pr_info!("The second value is: {}", this.value_2);
+//! pr_info!("The second value is: {}\n", this.value_2);
//! }
//! }
//!
@@ -128,15 +127,15 @@
//! fn print_2_later(val: Arc<MyStruct>) {
//! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 2>(val);
//! }
+//! # print_1_later(MyStruct::new(24, 25).unwrap());
+//! # print_2_later(MyStruct::new(41, 42).unwrap());
//! ```
//!
//! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h)
-use crate::{bindings, prelude::*, sync::Arc, sync::LockClassKey, types::Opaque};
-use alloc::alloc::AllocError;
-use alloc::boxed::Box;
+use crate::alloc::{AllocError, Flags};
+use crate::{prelude::*, sync::Arc, sync::LockClassKey, types::Opaque};
use core::marker::PhantomData;
-use core::pin::Pin;
/// Creates a [`Work`] initialiser with the given name and a newly-created lock class.
#[macro_export]
@@ -210,13 +209,17 @@ impl Queue {
/// Tries to spawn the given function or closure as a work item.
///
/// This method can fail because it allocates memory to store the work item.
- pub fn try_spawn<T: 'static + Send + FnOnce()>(&self, func: T) -> Result<(), AllocError> {
+ pub fn try_spawn<T: 'static + Send + FnOnce()>(
+ &self,
+ flags: Flags,
+ func: T,
+ ) -> Result<(), AllocError> {
let init = pin_init!(ClosureWork {
work <- new_work!("Queue::try_spawn"),
func: Some(func),
});
- self.enqueue(Box::pin_init(init).map_err(|_| AllocError)?);
+ self.enqueue(KBox::pin_init(init, flags).map_err(|_| AllocError)?);
Ok(())
}
}
@@ -239,9 +242,9 @@ impl<T> ClosureWork<T> {
}
impl<T: FnOnce()> WorkItem for ClosureWork<T> {
- type Pointer = Pin<Box<Self>>;
+ type Pointer = Pin<KBox<Self>>;
- fn run(mut this: Pin<Box<Self>>) {
+ fn run(mut this: Pin<KBox<Self>>) {
if let Some(func) = this.as_mut().project().take() {
(func)()
}
@@ -297,7 +300,7 @@ pub unsafe trait RawWorkItem<const ID: u64> {
/// Defines the method that should be called directly when a work item is executed.
///
-/// This trait is implemented by `Pin<Box<T>>` and [`Arc<T>`], and is mainly intended to be
+/// This trait is implemented by `Pin<KBox<T>>` and [`Arc<T>`], and is mainly intended to be
/// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`]
/// instead. The [`run`] method on this trait will usually just perform the appropriate
/// `container_of` translation and then call into the [`run`][WorkItem::run] method from the
@@ -329,7 +332,7 @@ pub unsafe trait WorkItemPointer<const ID: u64>: RawWorkItem<ID> {
/// This trait is used when the `work_struct` field is defined using the [`Work`] helper.
pub trait WorkItem<const ID: u64 = 0> {
/// The pointer type that this struct is wrapped in. This will typically be `Arc<Self>` or
- /// `Pin<Box<Self>>`.
+ /// `Pin<KBox<Self>>`.
type Pointer: WorkItemPointer<ID>;
/// The method that should be called when this work item is executed.
@@ -346,8 +349,10 @@ pub trait WorkItem<const ID: u64 = 0> {
/// This is a helper type used to associate a `work_struct` with the [`WorkItem`] that uses it.
///
/// [`run`]: WorkItemPointer::run
+#[pin_data]
#[repr(transparent)]
pub struct Work<T: ?Sized, const ID: u64 = 0> {
+ #[pin]
work: Opaque<bindings::work_struct>,
_inner: PhantomData<T>,
}
@@ -364,26 +369,26 @@ unsafe impl<T: ?Sized, const ID: u64> Sync for Work<T, ID> {}
impl<T: ?Sized, const ID: u64> Work<T, ID> {
/// Creates a new instance of [`Work`].
#[inline]
- #[allow(clippy::new_ret_no_self)]
- pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self>
+ pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self>
where
T: WorkItem<ID>,
{
- // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as the work
- // item function.
- unsafe {
- kernel::init::pin_init_from_closure(move |slot| {
- let slot = Self::raw_get(slot);
- bindings::init_work_with_key(
- slot,
- Some(T::Pointer::run),
- false,
- name.as_char_ptr(),
- key.as_ptr(),
- );
- Ok(())
- })
- }
+ pin_init!(Self {
+ work <- Opaque::ffi_init(|slot| {
+ // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as
+ // the work item function.
+ unsafe {
+ bindings::init_work_with_key(
+ slot,
+ Some(T::Pointer::run),
+ false,
+ name.as_char_ptr(),
+ key.as_ptr(),
+ )
+ }
+ }),
+ _inner: PhantomData,
+ })
}
/// Get a pointer to the inner `work_struct`.
@@ -408,7 +413,6 @@ impl<T: ?Sized, const ID: u64> Work<T, ID> {
/// like this:
///
/// ```no_run
-/// use kernel::prelude::*;
/// use kernel::workqueue::{impl_has_work, Work};
///
/// struct MyWorkItem {
@@ -425,51 +429,28 @@ impl<T: ?Sized, const ID: u64> Work<T, ID> {
///
/// # Safety
///
-/// The [`OFFSET`] constant must be the offset of a field in `Self` of type [`Work<T, ID>`]. The
-/// methods on this trait must have exactly the behavior that the definitions given below have.
+/// The methods [`raw_get_work`] and [`work_container_of`] must return valid pointers and must be
+/// true inverses of each other; that is, they must satisfy the following invariants:
+/// - `work_container_of(raw_get_work(ptr)) == ptr` for any `ptr: *mut Self`.
+/// - `raw_get_work(work_container_of(ptr)) == ptr` for any `ptr: *mut Work<T, ID>`.
///
/// [`impl_has_work!`]: crate::impl_has_work
-/// [`OFFSET`]: HasWork::OFFSET
+/// [`raw_get_work`]: HasWork::raw_get_work
+/// [`work_container_of`]: HasWork::work_container_of
pub unsafe trait HasWork<T, const ID: u64 = 0> {
- /// The offset of the [`Work<T, ID>`] field.
- const OFFSET: usize;
-
- /// Returns the offset of the [`Work<T, ID>`] field.
- ///
- /// This method exists because the [`OFFSET`] constant cannot be accessed if the type is not
- /// [`Sized`].
- ///
- /// [`OFFSET`]: HasWork::OFFSET
- #[inline]
- fn get_work_offset(&self) -> usize {
- Self::OFFSET
- }
-
/// Returns a pointer to the [`Work<T, ID>`] field.
///
/// # Safety
///
/// The provided pointer must point at a valid struct of type `Self`.
- #[inline]
- unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID> {
- // SAFETY: The caller promises that the pointer is valid.
- unsafe { (ptr as *mut u8).add(Self::OFFSET) as *mut Work<T, ID> }
- }
+ unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID>;
/// Returns a pointer to the struct containing the [`Work<T, ID>`] field.
///
/// # Safety
///
/// The pointer must point at a [`Work<T, ID>`] field in a struct of type `Self`.
- #[inline]
- unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self
- where
- Self: Sized,
- {
- // SAFETY: The caller promises that the pointer points at a field of the right type in the
- // right kind of struct.
- unsafe { (ptr as *mut u8).sub(Self::OFFSET) as *mut Self }
- }
+ unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self;
}
/// Used to safely implement the [`HasWork<T, ID>`] trait.
@@ -480,26 +461,26 @@ pub unsafe trait HasWork<T, const ID: u64 = 0> {
/// use kernel::sync::Arc;
/// use kernel::workqueue::{self, impl_has_work, Work};
///
-/// struct MyStruct {
-/// work_field: Work<MyStruct, 17>,
+/// struct MyStruct<'a, T, const N: usize> {
+/// work_field: Work<MyStruct<'a, T, N>, 17>,
+/// f: fn(&'a [T; N]),
/// }
///
/// impl_has_work! {
-/// impl HasWork<MyStruct, 17> for MyStruct { self.work_field }
+/// impl{'a, T, const N: usize} HasWork<MyStruct<'a, T, N>, 17>
+/// for MyStruct<'a, T, N> { self.work_field }
/// }
/// ```
#[macro_export]
macro_rules! impl_has_work {
- ($(impl$(<$($implarg:ident),*>)?
+ ($(impl$({$($generics:tt)*})?
HasWork<$work_type:ty $(, $id:tt)?>
- for $self:ident $(<$($selfarg:ident),*>)?
+ for $self:ty
{ self.$field:ident }
)*) => {$(
// SAFETY: The implementation of `raw_get_work` only compiles if the field has the right
// type.
- unsafe impl$(<$($implarg),*>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self $(<$($selfarg),*>)? {
- const OFFSET: usize = ::core::mem::offset_of!(Self, $field) as usize;
-
+ unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self {
#[inline]
unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> {
// SAFETY: The caller promises that the pointer is not dangling.
@@ -507,22 +488,40 @@ macro_rules! impl_has_work {
::core::ptr::addr_of_mut!((*ptr).$field)
}
}
+
+ #[inline]
+ unsafe fn work_container_of(
+ ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>,
+ ) -> *mut Self {
+ // SAFETY: The caller promises that the pointer points at a field of the right type
+ // in the right kind of struct.
+ unsafe { $crate::container_of!(ptr, Self, $field) }
+ }
}
)*};
}
pub use impl_has_work;
impl_has_work! {
- impl<T> HasWork<Self> for ClosureWork<T> { self.work }
+ impl{T} HasWork<Self> for ClosureWork<T> { self.work }
}
+// SAFETY: The `__enqueue` implementation in RawWorkItem uses a `work_struct` initialized with the
+// `run` method of this trait as the function pointer because:
+// - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`.
+// - The only safe way to create a `Work` object is through `Work::new`.
+// - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`.
+// - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field
+// will be used because of the ID const generic bound. This makes sure that `T::raw_get_work`
+// uses the correct offset for the `Work` field, and `Work::new` picks the correct
+// implementation of `WorkItemPointer` for `Arc<T>`.
unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Arc<T>
where
T: WorkItem<ID, Pointer = Self>,
T: HasWork<T, ID>,
{
unsafe extern "C" fn run(ptr: *mut bindings::work_struct) {
- // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`.
+ // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`.
let ptr = ptr as *mut Work<T, ID>;
// SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`.
let ptr = unsafe { T::work_container_of(ptr) };
@@ -533,6 +532,13 @@ where
}
}
+// SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to
+// the closure because we get it from an `Arc`, which means that the ref count will be at least 1,
+// and we don't drop the `Arc` ourselves. If `queue_work_on` returns true, it is further guaranteed
+// to be valid until a call to the function pointer in `work_struct` because we leak the memory it
+// points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which
+// is what the function pointer in the `work_struct` must be pointing to, according to the safety
+// requirements of `WorkItemPointer`.
unsafe impl<T, const ID: u64> RawWorkItem<ID> for Arc<T>
where
T: WorkItem<ID, Pointer = Self>,
@@ -561,18 +567,19 @@ where
}
}
-unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<Box<T>>
+// SAFETY: TODO.
+unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<KBox<T>>
where
T: WorkItem<ID, Pointer = Self>,
T: HasWork<T, ID>,
{
unsafe extern "C" fn run(ptr: *mut bindings::work_struct) {
- // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`.
+ // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`.
let ptr = ptr as *mut Work<T, ID>;
// SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`.
let ptr = unsafe { T::work_container_of(ptr) };
// SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership.
- let boxed = unsafe { Box::from_raw(ptr) };
+ let boxed = unsafe { KBox::from_raw(ptr) };
// SAFETY: The box was already pinned when it was enqueued.
let pinned = unsafe { Pin::new_unchecked(boxed) };
@@ -580,7 +587,8 @@ where
}
}
-unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<Box<T>>
+// SAFETY: TODO.
+unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<KBox<T>>
where
T: WorkItem<ID, Pointer = Self>,
T: HasWork<T, ID>,
@@ -594,9 +602,9 @@ where
// SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily
// remove the `Pin` wrapper.
let boxed = unsafe { Pin::into_inner_unchecked(self) };
- let ptr = Box::into_raw(boxed);
+ let ptr = KBox::into_raw(boxed);
- // SAFETY: Pointers into a `Box` point at a valid value.
+ // SAFETY: Pointers into a `KBox` point at a valid value.
let work_ptr = unsafe { T::raw_get_work(ptr) };
// SAFETY: `raw_get_work` returns a pointer to a valid value.
let work_ptr = unsafe { Work::raw_get(work_ptr) };
@@ -679,3 +687,21 @@ pub fn system_freezable_power_efficient() -> &'static Queue {
// SAFETY: `system_freezable_power_efficient_wq` is a C global, always available.
unsafe { Queue::from_raw(bindings::system_freezable_power_efficient_wq) }
}
+
+/// Returns the system bottom halves work queue (`system_bh_wq`).
+///
+/// It is similar to the one returned by [`system`] but for work items which
+/// need to run from a softirq context.
+pub fn system_bh() -> &'static Queue {
+ // SAFETY: `system_bh_wq` is a C global, always available.
+ unsafe { Queue::from_raw(bindings::system_bh_wq) }
+}
+
+/// Returns the system bottom halves high-priority work queue (`system_bh_highpri_wq`).
+///
+/// It is similar to the one returned by [`system_bh`] but for work items which
+/// require higher scheduling priority.
+pub fn system_bh_highpri() -> &'static Queue {
+ // SAFETY: `system_bh_highpri_wq` is a C global, always available.
+ unsafe { Queue::from_raw(bindings::system_bh_highpri_wq) }
+}
generated by cgit (git 2.34.1) at 2025-07-01 08:38:43 +0000