diff options
Diffstat (limited to 'rust/kernel')
32 files changed, 2662 insertions, 431 deletions
diff --git a/rust/kernel/acpi.rs b/rust/kernel/acpi.rs new file mode 100644 index 000000000000..7ae317368b00 --- /dev/null +++ b/rust/kernel/acpi.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Advanced Configuration and Power Interface abstractions. + +use crate::{ + bindings, + device_id::{RawDeviceId, RawDeviceIdIndex}, + prelude::*, +}; + +/// IdTable type for ACPI drivers. +pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; + +/// An ACPI device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::acpi_device_id); + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `acpi_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::acpi_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::acpi_device_id, driver_data); + + fn index(&self) -> usize { + self.0.driver_data + } +} + +impl DeviceId { + const ACPI_ID_LEN: usize = 16; + + /// Create a new device id from an ACPI 'id' string. + #[inline(always)] + pub const fn new(id: &'static CStr) -> Self { + build_assert!( + id.len_with_nul() <= Self::ACPI_ID_LEN, + "ID exceeds 16 bytes" + ); + let src = id.as_bytes_with_nul(); + // Replace with `bindings::acpi_device_id::default()` once stabilized for `const`. + // SAFETY: FFI type is valid to be zero-initialized. + let mut acpi: bindings::acpi_device_id = unsafe { core::mem::zeroed() }; + let mut i = 0; + while i < src.len() { + acpi.id[i] = src[i]; + i += 1; + } + + Self(acpi) + } +} + +/// Create an ACPI `IdTable` with an "alias" for modpost. +#[macro_export] +macro_rules! acpi_device_table { + ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { + const $table_name: $crate::device_id::IdArray< + $crate::acpi::DeviceId, + $id_info_type, + { $table_data.len() }, + > = $crate::device_id::IdArray::new($table_data); + + $crate::module_device_table!("acpi", $module_table_name, $table_name); + }; +} diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index d2cfe1eeefb6..4749fb6bffef 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -6,12 +6,11 @@ use crate::{ bindings, container_of, device, - device_id::RawDeviceId, + device_id::{RawDeviceId, RawDeviceIdIndex}, driver, - error::{to_result, Result}, + error::{from_result, to_result, Result}, prelude::*, - str::CStr, - types::{ForeignOwnable, Opaque}, + types::Opaque, ThisModule, }; use core::{ @@ -61,37 +60,32 @@ impl<T: Driver + 'static> Adapter<T> { // `struct auxiliary_device`. // // INVARIANT: `adev` is valid for the duration of `probe_callback()`. - let adev = unsafe { &*adev.cast::<Device<device::Core>>() }; + let adev = unsafe { &*adev.cast::<Device<device::CoreInternal>>() }; // SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id` // and does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::<DeviceId>() }; let info = T::ID_TABLE.info(id.index()); - match T::probe(adev, info) { - Ok(data) => { - // Let the `struct auxiliary_device` own a reference of the driver's private data. - // SAFETY: By the type invariant `adev.as_raw` returns a valid pointer to a - // `struct auxiliary_device`. - unsafe { - bindings::auxiliary_set_drvdata(adev.as_raw(), data.into_foreign().cast()) - }; - } - Err(err) => return Error::to_errno(err), - } + from_result(|| { + let data = T::probe(adev, info)?; - 0 + adev.as_ref().set_drvdata(data); + Ok(0) + }) } extern "C" fn remove_callback(adev: *mut bindings::auxiliary_device) { - // SAFETY: The auxiliary bus only ever calls the remove callback with a valid pointer to a + // SAFETY: The auxiliary bus only ever calls the probe callback with a valid pointer to a // `struct auxiliary_device`. - let ptr = unsafe { bindings::auxiliary_get_drvdata(adev) }; + // + // INVARIANT: `adev` is valid for the duration of `probe_callback()`. + let adev = unsafe { &*adev.cast::<Device<device::CoreInternal>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to - // `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized - // `KBox<T>` pointer created through `KBox::into_foreign`. - drop(unsafe { KBox::<T>::from_foreign(ptr.cast()) }); + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<T>>`. + drop(unsafe { adev.as_ref().drvdata_obtain::<Pin<KBox<T>>>() }); } } @@ -140,13 +134,14 @@ impl DeviceId { } } -// SAFETY: -// * `DeviceId` is a `#[repr(transparent)`] wrapper of `auxiliary_device_id` and does not add -// additional invariants, so it's safe to transmute to `RawType`. -// * `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `auxiliary_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. unsafe impl RawDeviceId for DeviceId { type RawType = bindings::auxiliary_device_id; +} +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::auxiliary_device_id, driver_data); @@ -276,7 +271,7 @@ impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> { let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; // SAFETY: `dev` points to a valid `struct device`. - unsafe { device::Device::as_ref(dev) } + unsafe { device::Device::from_raw(dev) } } } diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs index bcf4214ad149..c3cf56d52bee 100644 --- a/rust/kernel/block/mq/tag_set.rs +++ b/rust/kernel/block/mq/tag_set.rs @@ -9,7 +9,7 @@ use core::pin::Pin; use crate::{ bindings, block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations}, - error, + error::{self, Result}, prelude::try_pin_init, types::Opaque, }; @@ -41,7 +41,7 @@ impl<T: Operations> TagSet<T> { // SAFETY: `blk_mq_tag_set` only contains integers and pointers, which // all are allowed to be 0. let tag_set: bindings::blk_mq_tag_set = unsafe { core::mem::zeroed() }; - let tag_set = core::mem::size_of::<RequestDataWrapper>() + let tag_set: Result<_> = core::mem::size_of::<RequestDataWrapper>() .try_into() .map(|cmd_size| { bindings::blk_mq_tag_set { @@ -56,12 +56,14 @@ impl<T: Operations> TagSet<T> { nr_maps: num_maps, ..tag_set } - }); + }) + .map(Opaque::new) + .map_err(|e| e.into()); try_pin_init!(TagSet { - inner <- PinInit::<_, error::Error>::pin_chain(Opaque::new(tag_set?), |tag_set| { + inner <- tag_set.pin_chain(|tag_set| { // SAFETY: we do not move out of `tag_set`. - let tag_set = unsafe { Pin::get_unchecked_mut(tag_set) }; + let tag_set: &mut Opaque<_> = unsafe { Pin::get_unchecked_mut(tag_set) }; // SAFETY: `tag_set` is a reference to an initialized `blk_mq_tag_set`. error::to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())}) }), diff --git a/rust/kernel/cpu.rs b/rust/kernel/cpu.rs index b75403b0eb56..5de730c8d817 100644 --- a/rust/kernel/cpu.rs +++ b/rust/kernel/cpu.rs @@ -147,5 +147,5 @@ pub unsafe fn from_cpu(cpu: CpuId) -> Result<&'static Device> { // SAFETY: The pointer returned by `get_cpu_device()`, if not `NULL`, is a valid pointer to // a `struct device` and is never freed by the C code. - Ok(unsafe { Device::as_ref(ptr) }) + Ok(unsafe { Device::from_raw(ptr) }) } diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 11b03e9d7e89..d0ea24236ae4 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -13,7 +13,7 @@ use crate::{ cpu::CpuId, cpumask, device::{Bound, Device}, - devres::Devres, + devres, error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR}, ffi::{c_char, c_ulong}, prelude::*, @@ -1046,10 +1046,13 @@ impl<T: Driver> Registration<T> { /// Same as [`Registration::new`], but does not return a [`Registration`] instance. /// - /// Instead the [`Registration`] is owned by [`Devres`] and will be revoked / dropped, once the + /// Instead the [`Registration`] is owned by [`devres::register`] and will be dropped, once the /// device is detached. - pub fn new_foreign_owned(dev: &Device<Bound>) -> Result { - Devres::new_foreign_owned(dev, Self::new()?, GFP_KERNEL) + pub fn new_foreign_owned(dev: &Device<Bound>) -> Result + where + T: 'static, + { + devres::register(dev, Self::new()?, GFP_KERNEL) } } @@ -1061,7 +1064,7 @@ impl<T: Driver> Registration<T> { /// /// - This function may only be called from the cpufreq C infrastructure. /// - The pointer arguments must be valid pointers. - unsafe extern "C" fn init_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + unsafe extern "C" fn init_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1094,7 +1097,7 @@ impl<T: Driver> Registration<T> { /// /// - This function may only be called from the cpufreq C infrastructure. /// - The pointer arguments must be valid pointers. - unsafe extern "C" fn online_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + unsafe extern "C" fn online_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1109,9 +1112,7 @@ impl<T: Driver> Registration<T> { /// /// - This function may only be called from the cpufreq C infrastructure. /// - The pointer arguments must be valid pointers. - unsafe extern "C" fn offline_callback( - ptr: *mut bindings::cpufreq_policy, - ) -> kernel::ffi::c_int { + unsafe extern "C" fn offline_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1126,9 +1127,7 @@ impl<T: Driver> Registration<T> { /// /// - This function may only be called from the cpufreq C infrastructure. /// - The pointer arguments must be valid pointers. - unsafe extern "C" fn suspend_callback( - ptr: *mut bindings::cpufreq_policy, - ) -> kernel::ffi::c_int { + unsafe extern "C" fn suspend_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1143,7 +1142,7 @@ impl<T: Driver> Registration<T> { /// /// - This function may only be called from the cpufreq C infrastructure. /// - The pointer arguments must be valid pointers. - unsafe extern "C" fn resume_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + unsafe extern "C" fn resume_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1171,9 +1170,7 @@ impl<T: Driver> Registration<T> { /// /// - This function may only be called from the cpufreq C infrastructure. /// - The pointer arguments must be valid pointers. - unsafe extern "C" fn verify_callback( - ptr: *mut bindings::cpufreq_policy_data, - ) -> kernel::ffi::c_int { + unsafe extern "C" fn verify_callback(ptr: *mut bindings::cpufreq_policy_data) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1188,9 +1185,7 @@ impl<T: Driver> Registration<T> { /// /// - This function may only be called from the cpufreq C infrastructure. /// - The pointer arguments must be valid pointers. - unsafe extern "C" fn setpolicy_callback( - ptr: *mut bindings::cpufreq_policy, - ) -> kernel::ffi::c_int { + unsafe extern "C" fn setpolicy_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1207,9 +1202,9 @@ impl<T: Driver> Registration<T> { /// - The pointer arguments must be valid pointers. unsafe extern "C" fn target_callback( ptr: *mut bindings::cpufreq_policy, - target_freq: u32, - relation: u32, - ) -> kernel::ffi::c_int { + target_freq: c_uint, + relation: c_uint, + ) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1226,8 +1221,8 @@ impl<T: Driver> Registration<T> { /// - The pointer arguments must be valid pointers. unsafe extern "C" fn target_index_callback( ptr: *mut bindings::cpufreq_policy, - index: u32, - ) -> kernel::ffi::c_int { + index: c_uint, + ) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1249,8 +1244,8 @@ impl<T: Driver> Registration<T> { /// - The pointer arguments must be valid pointers. unsafe extern "C" fn fast_switch_callback( ptr: *mut bindings::cpufreq_policy, - target_freq: u32, - ) -> kernel::ffi::c_uint { + target_freq: c_uint, + ) -> c_uint { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. let policy = unsafe { Policy::from_raw_mut(ptr) }; @@ -1263,10 +1258,10 @@ impl<T: Driver> Registration<T> { /// /// - This function may only be called from the cpufreq C infrastructure. unsafe extern "C" fn adjust_perf_callback( - cpu: u32, - min_perf: usize, - target_perf: usize, - capacity: usize, + cpu: c_uint, + min_perf: c_ulong, + target_perf: c_ulong, + capacity: c_ulong, ) { // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; @@ -1284,8 +1279,8 @@ impl<T: Driver> Registration<T> { /// - The pointer arguments must be valid pointers. unsafe extern "C" fn get_intermediate_callback( ptr: *mut bindings::cpufreq_policy, - index: u32, - ) -> kernel::ffi::c_uint { + index: c_uint, + ) -> c_uint { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. let policy = unsafe { Policy::from_raw_mut(ptr) }; @@ -1305,8 +1300,8 @@ impl<T: Driver> Registration<T> { /// - The pointer arguments must be valid pointers. unsafe extern "C" fn target_intermediate_callback( ptr: *mut bindings::cpufreq_policy, - index: u32, - ) -> kernel::ffi::c_int { + index: c_uint, + ) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1325,7 +1320,7 @@ impl<T: Driver> Registration<T> { /// # Safety /// /// - This function may only be called from the cpufreq C infrastructure. - unsafe extern "C" fn get_callback(cpu: u32) -> kernel::ffi::c_uint { + unsafe extern "C" fn get_callback(cpu: c_uint) -> c_uint { // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; @@ -1351,7 +1346,7 @@ impl<T: Driver> Registration<T> { /// /// - This function may only be called from the cpufreq C infrastructure. /// - The pointer arguments must be valid pointers. - unsafe extern "C" fn bios_limit_callback(cpu: i32, limit: *mut u32) -> kernel::ffi::c_int { + unsafe extern "C" fn bios_limit_callback(cpu: c_int, limit: *mut c_uint) -> c_int { // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. let cpu_id = unsafe { CpuId::from_i32_unchecked(cpu) }; @@ -1371,8 +1366,8 @@ impl<T: Driver> Registration<T> { /// - The pointer arguments must be valid pointers. unsafe extern "C" fn set_boost_callback( ptr: *mut bindings::cpufreq_policy, - state: i32, - ) -> kernel::ffi::c_int { + state: c_int, + ) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs index 19c607709b5f..e07f8ff5e3fd 100644 --- a/rust/kernel/cpumask.rs +++ b/rust/kernel/cpumask.rs @@ -14,9 +14,6 @@ use crate::{ #[cfg(CONFIG_CPUMASK_OFFSTACK)] use core::ptr::{self, NonNull}; -#[cfg(not(CONFIG_CPUMASK_OFFSTACK))] -use core::mem::MaybeUninit; - use core::ops::{Deref, DerefMut}; /// A CPU Mask. @@ -239,10 +236,7 @@ impl CpumaskVar { }, #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] - // SAFETY: FFI type is valid to be zero-initialized. - // - // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of scope. - mask: unsafe { core::mem::zeroed() }, + mask: Cpumask(Opaque::zeroed()), }) } @@ -266,10 +260,7 @@ impl CpumaskVar { NonNull::new(ptr.cast()).ok_or(AllocError)? }, #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] - // SAFETY: Guaranteed by the safety requirements of the function. - // - // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of scope. - mask: unsafe { MaybeUninit::uninit().assume_init() }, + mask: Cpumask(Opaque::uninit()), }) } diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index dea06b79ecb5..ca82926fd67f 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -6,14 +6,15 @@ use crate::{ bindings, - str::CStr, - types::{ARef, Opaque}, + types::{ARef, ForeignOwnable, Opaque}, }; use core::{fmt, marker::PhantomData, ptr}; #[cfg(CONFIG_PRINTK)] use crate::c_str; +pub mod property; + /// A reference-counted device. /// /// This structure represents the Rust abstraction for a C `struct device`. This implementation @@ -57,7 +58,73 @@ impl Device { /// While not officially documented, this should be the case for any `struct device`. pub unsafe fn get_device(ptr: *mut bindings::device) -> ARef<Self> { // SAFETY: By the safety requirements ptr is valid - unsafe { Self::as_ref(ptr) }.into() + unsafe { Self::from_raw(ptr) }.into() + } + + /// Convert a [`&Device`](Device) into a [`&Device<Bound>`](Device<Bound>). + /// + /// # Safety + /// + /// The caller is responsible to ensure that the returned [`&Device<Bound>`](Device<Bound>) + /// only lives as long as it can be guaranteed that the [`Device`] is actually bound. + pub unsafe fn as_bound(&self) -> &Device<Bound> { + let ptr = core::ptr::from_ref(self); + + // CAST: By the safety requirements the caller is responsible to guarantee that the + // returned reference only lives as long as the device is actually bound. + let ptr = ptr.cast(); + + // SAFETY: + // - `ptr` comes from `from_ref(self)` above, hence it's guaranteed to be valid. + // - Any valid `Device` pointer is also a valid pointer for `Device<Bound>`. + unsafe { &*ptr } + } +} + +impl Device<CoreInternal> { + /// Store a pointer to the bound driver's private data. + pub fn set_drvdata(&self, data: impl ForeignOwnable) { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + unsafe { bindings::dev_set_drvdata(self.as_raw(), data.into_foreign().cast()) } + } + + /// Take ownership of the private data stored in this [`Device`]. + /// + /// # Safety + /// + /// - Must only be called once after a preceding call to [`Device::set_drvdata`]. + /// - The type `T` must match the type of the `ForeignOwnable` previously stored by + /// [`Device::set_drvdata`]. + pub unsafe fn drvdata_obtain<T: ForeignOwnable>(&self) -> T { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; + + // SAFETY: + // - By the safety requirements of this function, `ptr` comes from a previous call to + // `into_foreign()`. + // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` + // in `into_foreign()`. + unsafe { T::from_foreign(ptr.cast()) } + } + + /// Borrow the driver's private data bound to this [`Device`]. + /// + /// # Safety + /// + /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before + /// [`Device::drvdata_obtain`]. + /// - The type `T` must match the type of the `ForeignOwnable` previously stored by + /// [`Device::set_drvdata`]. + pub unsafe fn drvdata_borrow<T: ForeignOwnable>(&self) -> T::Borrowed<'_> { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; + + // SAFETY: + // - By the safety requirements of this function, `ptr` comes from a previous call to + // `into_foreign()`. + // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` + // in `into_foreign()`. + unsafe { T::borrow(ptr.cast()) } } } @@ -82,7 +149,7 @@ impl<Ctx: DeviceContext> Device<Ctx> { // - Since `parent` is not NULL, it must be a valid pointer to a `struct device`. // - `parent` is valid for the lifetime of `self`, since a `struct device` holds a // reference count of its parent. - Some(unsafe { Self::as_ref(parent) }) + Some(unsafe { Self::from_raw(parent) }) } } @@ -94,7 +161,7 @@ impl<Ctx: DeviceContext> Device<Ctx> { /// i.e. it must be ensured that the reference count of the C `struct device` `ptr` points to /// can't drop to zero, for the duration of this function call and the entire duration when the /// returned reference exists. - pub unsafe fn as_ref<'a>(ptr: *mut bindings::device) -> &'a Self { + pub unsafe fn from_raw<'a>(ptr: *mut bindings::device) -> &'a Self { // SAFETY: Guaranteed by the safety requirements of the function. unsafe { &*ptr.cast() } } @@ -203,10 +270,19 @@ impl<Ctx: DeviceContext> Device<Ctx> { }; } - /// Checks if property is present or not. - pub fn property_present(&self, name: &CStr) -> bool { - // SAFETY: By the invariant of `CStr`, `name` is null-terminated. - unsafe { bindings::device_property_present(self.as_raw().cast_const(), name.as_char_ptr()) } + /// Obtain the [`FwNode`](property::FwNode) corresponding to this [`Device`]. + pub fn fwnode(&self) -> Option<&property::FwNode> { + // SAFETY: `self` is valid. + let fwnode_handle = unsafe { bindings::__dev_fwnode(self.as_raw()) }; + if fwnode_handle.is_null() { + return None; + } + // SAFETY: `fwnode_handle` is valid. Its lifetime is tied to `&self`. We + // return a reference instead of an `ARef<FwNode>` because `dev_fwnode()` + // doesn't increment the refcount. It is safe to cast from a + // `struct fwnode_handle*` to a `*const FwNode` because `FwNode` is + // defined as a `#[repr(transparent)]` wrapper around `fwnode_handle`. + Some(unsafe { &*fwnode_handle.cast() }) } } @@ -251,6 +327,10 @@ pub struct Normal; /// any of the bus callbacks, such as `probe()`. pub struct Core; +/// Semantically the same as [`Core`] but reserved for internal usage of the corresponding bus +/// abstraction. +pub struct CoreInternal; + /// The [`Bound`] context is the context of a bus specific device reference when it is guaranteed to /// be bound for the duration of its lifetime. pub struct Bound; @@ -260,11 +340,13 @@ mod private { impl Sealed for super::Bound {} impl Sealed for super::Core {} + impl Sealed for super::CoreInternal {} impl Sealed for super::Normal {} } impl DeviceContext for Bound {} impl DeviceContext for Core {} +impl DeviceContext for CoreInternal {} impl DeviceContext for Normal {} /// # Safety @@ -306,6 +388,13 @@ macro_rules! impl_device_context_deref { // `__impl_device_context_deref!`. ::kernel::__impl_device_context_deref!(unsafe { $device, + $crate::device::CoreInternal => $crate::device::Core + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, $crate::device::Core => $crate::device::Bound }); @@ -335,6 +424,7 @@ macro_rules! __impl_device_context_into_aref { #[macro_export] macro_rules! impl_device_context_into_aref { ($device:tt) => { + ::kernel::__impl_device_context_into_aref!($crate::device::CoreInternal, $device); ::kernel::__impl_device_context_into_aref!($crate::device::Core, $device); ::kernel::__impl_device_context_into_aref!($crate::device::Bound, $device); }; diff --git a/rust/kernel/device/property.rs b/rust/kernel/device/property.rs new file mode 100644 index 000000000000..49ee12a906db --- /dev/null +++ b/rust/kernel/device/property.rs @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Unified device property interface. +//! +//! C header: [`include/linux/property.h`](srctree/include/linux/property.h) + +use core::{mem::MaybeUninit, ptr}; + +use super::private::Sealed; +use crate::{ + alloc::KVec, + bindings, + error::{to_result, Result}, + prelude::*, + str::{CStr, CString}, + types::{ARef, Opaque}, +}; + +/// A reference-counted fwnode_handle. +/// +/// This structure represents the Rust abstraction for a +/// C `struct fwnode_handle`. This implementation abstracts the usage of an +/// already existing C `struct fwnode_handle` within Rust code that we get +/// passed from the C side. +/// +/// # Invariants +/// +/// A `FwNode` instance represents a valid `struct fwnode_handle` created by the +/// C portion of the kernel. +/// +/// Instances of this type are always reference-counted, that is, a call to +/// `fwnode_handle_get` ensures that the allocation remains valid at least until +/// the matching call to `fwnode_handle_put`. +#[repr(transparent)] +pub struct FwNode(Opaque<bindings::fwnode_handle>); + +impl FwNode { + /// # Safety + /// + /// Callers must ensure that: + /// - The reference count was incremented at least once. + /// - They relinquish that increment. That is, if there is only one + /// increment, callers must not use the underlying object anymore -- it is + /// only safe to do so via the newly created `ARef<FwNode>`. + unsafe fn from_raw(raw: *mut bindings::fwnode_handle) -> ARef<Self> { + // SAFETY: As per the safety requirements of this function: + // - `NonNull::new_unchecked`: + // - `raw` is not null. + // - `ARef::from_raw`: + // - `raw` has an incremented refcount. + // - that increment is relinquished, i.e. it won't be decremented + // elsewhere. + // CAST: It is safe to cast from a `*mut fwnode_handle` to + // `*mut FwNode`, because `FwNode` is defined as a + // `#[repr(transparent)]` wrapper around `fwnode_handle`. + unsafe { ARef::from_raw(ptr::NonNull::new_unchecked(raw.cast())) } + } + + /// Obtain the raw `struct fwnode_handle *`. + pub(crate) fn as_raw(&self) -> *mut bindings::fwnode_handle { + self.0.get() + } + + /// Returns `true` if `&self` is an OF node, `false` otherwise. + pub fn is_of_node(&self) -> bool { + // SAFETY: The type invariant of `Self` guarantees that `self.as_raw() is a pointer to a + // valid `struct fwnode_handle`. + unsafe { bindings::is_of_node(self.as_raw()) } + } + + /// Returns an object that implements [`Display`](core::fmt::Display) for + /// printing the name of a node. + /// + /// This is an alternative to the default `Display` implementation, which + /// prints the full path. + pub fn display_name(&self) -> impl core::fmt::Display + '_ { + struct FwNodeDisplayName<'a>(&'a FwNode); + + impl core::fmt::Display for FwNodeDisplayName<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // SAFETY: `self` is valid by its type invariant. + let name = unsafe { bindings::fwnode_get_name(self.0.as_raw()) }; + if name.is_null() { + return Ok(()); + } + // SAFETY: + // - `fwnode_get_name` returns null or a valid C string. + // - `name` was checked to be non-null. + let name = unsafe { CStr::from_char_ptr(name) }; + write!(f, "{name}") + } + } + + FwNodeDisplayName(self) + } + + /// Checks if property is present or not. + pub fn property_present(&self, name: &CStr) -> bool { + // SAFETY: By the invariant of `CStr`, `name` is null-terminated. + unsafe { bindings::fwnode_property_present(self.as_raw().cast_const(), name.as_char_ptr()) } + } + + /// Returns firmware property `name` boolean value. + pub fn property_read_bool(&self, name: &CStr) -> bool { + // SAFETY: + // - `name` is non-null and null-terminated. + // - `self.as_raw()` is valid because `self` is valid. + unsafe { bindings::fwnode_property_read_bool(self.as_raw(), name.as_char_ptr()) } + } + + /// Returns the index of matching string `match_str` for firmware string + /// property `name`. + pub fn property_match_string(&self, name: &CStr, match_str: &CStr) -> Result<usize> { + // SAFETY: + // - `name` and `match_str` are non-null and null-terminated. + // - `self.as_raw` is valid because `self` is valid. + let ret = unsafe { + bindings::fwnode_property_match_string( + self.as_raw(), + name.as_char_ptr(), + match_str.as_char_ptr(), + ) + }; + to_result(ret)?; + Ok(ret as usize) + } + + /// Returns firmware property `name` integer array values in a [`KVec`]. + pub fn property_read_array_vec<'fwnode, 'name, T: PropertyInt>( + &'fwnode self, + name: &'name CStr, + len: usize, + ) -> Result<PropertyGuard<'fwnode, 'name, KVec<T>>> { + let mut val: KVec<T> = KVec::with_capacity(len, GFP_KERNEL)?; + + let res = T::read_array_from_fwnode_property(self, name, val.spare_capacity_mut()); + let res = match res { + Ok(_) => { + // SAFETY: + // - `len` is equal to `val.capacity - val.len`, because + // `val.capacity` is `len` and `val.len` is zero. + // - All elements within the interval [`0`, `len`) were initialized + // by `read_array_from_fwnode_property`. + unsafe { val.inc_len(len) } + Ok(val) + } + Err(e) => Err(e), + }; + Ok(PropertyGuard { + inner: res, + fwnode: self, + name, + }) + } + + /// Returns integer array length for firmware property `name`. + pub fn property_count_elem<T: PropertyInt>(&self, name: &CStr) -> Result<usize> { + T::read_array_len_from_fwnode_property(self, name) + } + + /// Returns the value of firmware property `name`. + /// + /// This method is generic over the type of value to read. The types that + /// can be read are strings, integers and arrays of integers. + /// + /// Reading a [`KVec`] of integers is done with the separate + /// method [`Self::property_read_array_vec`], because it takes an + /// additional `len` argument. + /// + /// Reading a boolean is done with the separate method + /// [`Self::property_read_bool`], because this operation is infallible. + /// + /// For more precise documentation about what types can be read, see + /// the [implementors of Property][Property#implementors] and [its + /// implementations on foreign types][Property#foreign-impls]. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{c_str, device::{Device, property::FwNode}, str::CString}; + /// fn examples(dev: &Device) -> Result { + /// let fwnode = dev.fwnode().ok_or(ENOENT)?; + /// let b: u32 = fwnode.property_read(c_str!("some-number")).required_by(dev)?; + /// if let Some(s) = fwnode.property_read::<CString>(c_str!("some-str")).optional() { + /// // ... + /// } + /// Ok(()) + /// } + /// ``` + pub fn property_read<'fwnode, 'name, T: Property>( + &'fwnode self, + name: &'name CStr, + ) -> PropertyGuard<'fwnode, 'name, T> { + PropertyGuard { + inner: T::read_from_fwnode_property(self, name), + fwnode: self, + name, + } + } + + /// Returns first matching named child node handle. + pub fn get_child_by_name(&self, name: &CStr) -> Option<ARef<Self>> { + // SAFETY: `self` and `name` are valid by their type invariants. + let child = + unsafe { bindings::fwnode_get_named_child_node(self.as_raw(), name.as_char_ptr()) }; + if child.is_null() { + return None; + } + // SAFETY: + // - `fwnode_get_named_child_node` returns a pointer with its refcount + // incremented. + // - That increment is relinquished, i.e. the underlying object is not + // used anymore except via the newly created `ARef`. + Some(unsafe { Self::from_raw(child) }) + } + + /// Returns an iterator over a node's children. + pub fn children<'a>(&'a self) -> impl Iterator<Item = ARef<FwNode>> + 'a { + let mut prev: Option<ARef<FwNode>> = None; + + core::iter::from_fn(move || { + let prev_ptr = match prev.take() { + None => ptr::null_mut(), + Some(prev) => { + // We will pass `prev` to `fwnode_get_next_child_node`, + // which decrements its refcount, so we use + // `ARef::into_raw` to avoid decrementing the refcount + // twice. + let prev = ARef::into_raw(prev); + prev.as_ptr().cast() + } + }; + // SAFETY: + // - `self.as_raw()` is valid by its type invariant. + // - `prev_ptr` may be null, which is allowed and corresponds to + // getting the first child. Otherwise, `prev_ptr` is valid, as it + // is the stored return value from the previous invocation. + // - `prev_ptr` has its refount incremented. + // - The increment of `prev_ptr` is relinquished, i.e. the + // underlying object won't be used anymore. + let next = unsafe { bindings::fwnode_get_next_child_node(self.as_raw(), prev_ptr) }; + if next.is_null() { + return None; + } + // SAFETY: + // - `next` is valid because `fwnode_get_next_child_node` returns a + // pointer with its refcount incremented. + // - That increment is relinquished, i.e. the underlying object + // won't be used anymore, except via the newly created + // `ARef<Self>`. + let next = unsafe { FwNode::from_raw(next) }; + prev = Some(next.clone()); + Some(next) + }) + } + + /// Finds a reference with arguments. + pub fn property_get_reference_args( + &self, + prop: &CStr, + nargs: NArgs<'_>, + index: u32, + ) -> Result<FwNodeReferenceArgs> { + let mut out_args = FwNodeReferenceArgs::default(); + + let (nargs_prop, nargs) = match nargs { + NArgs::Prop(nargs_prop) => (nargs_prop.as_char_ptr(), 0), + NArgs::N(nargs) => (ptr::null(), nargs), + }; + + // SAFETY: + // - `self.0.get()` is valid. + // - `prop.as_char_ptr()` is valid and zero-terminated. + // - `nargs_prop` is valid and zero-terminated if `nargs` + // is zero, otherwise it is allowed to be a null-pointer. + // - The function upholds the type invariants of `out_args`, + // namely: + // - It may fill the field `fwnode` with a valid pointer, + // in which case its refcount is incremented. + // - It may modify the field `nargs`, in which case it + // initializes at least as many elements in `args`. + let ret = unsafe { + bindings::fwnode_property_get_reference_args( + self.0.get(), + prop.as_char_ptr(), + nargs_prop, + nargs, + index, + &mut out_args.0, + ) + }; + to_result(ret)?; + + Ok(out_args) + } +} + +/// The number of arguments to request [`FwNodeReferenceArgs`]. +pub enum NArgs<'a> { + /// The name of the property of the reference indicating the number of + /// arguments. + Prop(&'a CStr), + /// The known number of arguments. + N(u32), +} + +/// The return value of [`FwNode::property_get_reference_args`]. +/// +/// This structure represents the Rust abstraction for a C +/// `struct fwnode_reference_args` which was initialized by the C side. +/// +/// # Invariants +/// +/// If the field `fwnode` is valid, it owns an increment of its refcount. +/// +/// The field `args` contains at least as many initialized elements as indicated +/// by the field `nargs`. +#[repr(transparent)] +#[derive(Default)] +pub struct FwNodeReferenceArgs(bindings::fwnode_reference_args); + +impl Drop for FwNodeReferenceArgs { + fn drop(&mut self) { + if !self.0.fwnode.is_null() { + // SAFETY: + // - By the type invariants of `FwNodeReferenceArgs`, its field + // `fwnode` owns an increment of its refcount. + // - That increment is relinquished. The underlying object won't be + // used anymore because we are dropping it. + let _ = unsafe { FwNode::from_raw(self.0.fwnode) }; + } + } +} + +impl FwNodeReferenceArgs { + /// Returns the slice of reference arguments. + pub fn as_slice(&self) -> &[u64] { + // SAFETY: As per the safety invariant of `FwNodeReferenceArgs`, `nargs` + // is the minimum number of elements in `args` that is valid. + unsafe { core::slice::from_raw_parts(self.0.args.as_ptr(), self.0.nargs as usize) } + } + + /// Returns the number of reference arguments. + pub fn len(&self) -> usize { + self.0.nargs as usize + } + + /// Returns `true` if there are no reference arguments. + pub fn is_empty(&self) -> bool { + self.0.nargs == 0 + } +} + +impl core::fmt::Debug for FwNodeReferenceArgs { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{:?}", self.as_slice()) + } +} + +// SAFETY: Instances of `FwNode` are always reference-counted. +unsafe impl crate::types::AlwaysRefCounted for FwNode { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the + // refcount is non-zero. + unsafe { bindings::fwnode_handle_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: ptr::NonNull<Self>) { + // SAFETY: The safety requirements guarantee that the refcount is + // non-zero. + unsafe { bindings::fwnode_handle_put(obj.cast().as_ptr()) } + } +} + +enum Node<'a> { + Borrowed(&'a FwNode), + Owned(ARef<FwNode>), +} + +impl core::fmt::Display for FwNode { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // The logic here is the same as the one in lib/vsprintf.c + // (fwnode_full_name_string). + + // SAFETY: `self.as_raw()` is valid by its type invariant. + let num_parents = unsafe { bindings::fwnode_count_parents(self.as_raw()) }; + + for depth in (0..=num_parents).rev() { + let fwnode = if depth == 0 { + Node::Borrowed(self) + } else { + // SAFETY: `self.as_raw()` is valid. + let ptr = unsafe { bindings::fwnode_get_nth_parent(self.as_raw(), depth) }; + // SAFETY: + // - The depth passed to `fwnode_get_nth_parent` is + // within the valid range, so the returned pointer is + // not null. + // - The reference count was incremented by + // `fwnode_get_nth_parent`. + // - That increment is relinquished to + // `FwNode::from_raw`. + Node::Owned(unsafe { FwNode::from_raw(ptr) }) + }; + // Take a reference to the owned or borrowed `FwNode`. + let fwnode: &FwNode = match &fwnode { + Node::Borrowed(f) => f, + Node::Owned(f) => f, + }; + + // SAFETY: `fwnode` is valid by its type invariant. + let prefix = unsafe { bindings::fwnode_get_name_prefix(fwnode.as_raw()) }; + if !prefix.is_null() { + // SAFETY: `fwnode_get_name_prefix` returns null or a + // valid C string. + let prefix = unsafe { CStr::from_char_ptr(prefix) }; + write!(f, "{prefix}")?; + } + write!(f, "{}", fwnode.display_name())?; + } + + Ok(()) + } +} + +/// Implemented for types that can be read as properties. +/// +/// This is implemented for strings, integers and arrays of integers. It's used +/// to make [`FwNode::property_read`] generic over the type of property being +/// read. There are also two dedicated methods to read other types, because they +/// require more specialized function signatures: +/// - [`property_read_bool`](FwNode::property_read_bool) +/// - [`property_read_array_vec`](FwNode::property_read_array_vec) +/// +/// It must be public, because it appears in the signatures of other public +/// functions, but its methods shouldn't be used outside the kernel crate. +pub trait Property: Sized + Sealed { + /// Used to make [`FwNode::property_read`] generic. + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<Self>; +} + +impl Sealed for CString {} + +impl Property for CString { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<Self> { + let mut str: *mut u8 = ptr::null_mut(); + let pstr: *mut _ = &mut str; + + // SAFETY: + // - `name` is non-null and null-terminated. + // - `fwnode.as_raw` is valid because `fwnode` is valid. + let ret = unsafe { + bindings::fwnode_property_read_string(fwnode.as_raw(), name.as_char_ptr(), pstr.cast()) + }; + to_result(ret)?; + + // SAFETY: + // - `pstr` is a valid pointer to a NUL-terminated C string. + // - It is valid for at least as long as `fwnode`, but it's only used + // within the current function. + // - The memory it points to is not mutated during that time. + let str = unsafe { CStr::from_char_ptr(*pstr) }; + Ok(str.try_into()?) + } +} + +/// Implemented for all integers that can be read as properties. +/// +/// This helper trait is needed on top of the existing [`Property`] +/// trait to associate the integer types of various sizes with their +/// corresponding `fwnode_property_read_*_array` functions. +/// +/// It must be public, because it appears in the signatures of other public +/// functions, but its methods shouldn't be used outside the kernel crate. +pub trait PropertyInt: Copy + Sealed { + /// Reads a property array. + fn read_array_from_fwnode_property<'a>( + fwnode: &FwNode, + name: &CStr, + out: &'a mut [MaybeUninit<Self>], + ) -> Result<&'a mut [Self]>; + + /// Reads the length of a property array. + fn read_array_len_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<usize>; +} +// This macro generates implementations of the traits `Property` and +// `PropertyInt` for integers of various sizes. Its input is a list +// of pairs separated by commas. The first element of the pair is the +// type of the integer, the second one is the name of its corresponding +// `fwnode_property_read_*_array` function. +macro_rules! impl_property_for_int { + ($($int:ty: $f:ident),* $(,)?) => { $( + impl Sealed for $int {} + impl<const N: usize> Sealed for [$int; N] {} + + impl PropertyInt for $int { + fn read_array_from_fwnode_property<'a>( + fwnode: &FwNode, + name: &CStr, + out: &'a mut [MaybeUninit<Self>], + ) -> Result<&'a mut [Self]> { + // SAFETY: + // - `fwnode`, `name` and `out` are all valid by their type + // invariants. + // - `out.len()` is a valid bound for the memory pointed to by + // `out.as_mut_ptr()`. + // CAST: It's ok to cast from `*mut MaybeUninit<$int>` to a + // `*mut $int` because they have the same memory layout. + let ret = unsafe { + bindings::$f( + fwnode.as_raw(), + name.as_char_ptr(), + out.as_mut_ptr().cast(), + out.len(), + ) + }; + to_result(ret)?; + // SAFETY: Transmuting from `&'a mut [MaybeUninit<Self>]` to + // `&'a mut [Self]` is sound, because the previous call to a + // `fwnode_property_read_*_array` function (which didn't fail) + // fully initialized the slice. + Ok(unsafe { core::mem::transmute::<&mut [MaybeUninit<Self>], &mut [Self]>(out) }) + } + + fn read_array_len_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<usize> { + // SAFETY: + // - `fwnode` and `name` are valid by their type invariants. + // - It's ok to pass a null pointer to the + // `fwnode_property_read_*_array` functions if `nval` is zero. + // This will return the length of the array. + let ret = unsafe { + bindings::$f( + fwnode.as_raw(), + name.as_char_ptr(), + ptr::null_mut(), + 0, + ) + }; + to_result(ret)?; + Ok(ret as usize) + } + } + + impl Property for $int { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<Self> { + let val: [_; 1] = <[$int; 1]>::read_from_fwnode_property(fwnode, name)?; + Ok(val[0]) + } + } + + impl<const N: usize> Property for [$int; N] { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<Self> { + let mut val: [MaybeUninit<$int>; N] = [const { MaybeUninit::uninit() }; N]; + + <$int>::read_array_from_fwnode_property(fwnode, name, &mut val)?; + + // SAFETY: `val` is always initialized when + // `fwnode_property_read_*_array` is successful. + Ok(val.map(|v| unsafe { v.assume_init() })) + } + } + )* }; +} +impl_property_for_int! { + u8: fwnode_property_read_u8_array, + u16: fwnode_property_read_u16_array, + u32: fwnode_property_read_u32_array, + u64: fwnode_property_read_u64_array, + i8: fwnode_property_read_u8_array, + i16: fwnode_property_read_u16_array, + i32: fwnode_property_read_u32_array, + i64: fwnode_property_read_u64_array, +} + +/// A helper for reading device properties. +/// +/// Use [`Self::required_by`] if a missing property is considered a bug and +/// [`Self::optional`] otherwise. +/// +/// For convenience, [`Self::or`] and [`Self::or_default`] are provided. +pub struct PropertyGuard<'fwnode, 'name, T> { + /// The result of reading the property. + inner: Result<T>, + /// The fwnode of the property, used for logging in the "required" case. + fwnode: &'fwnode FwNode, + /// The name of the property, used for logging in the "required" case. + name: &'name CStr, +} + +impl<T> PropertyGuard<'_, '_, T> { + /// Access the property, indicating it is required. + /// + /// If the property is not present, the error is automatically logged. If a + /// missing property is not an error, use [`Self::optional`] instead. The + /// device is required to associate the log with it. + pub fn required_by(self, dev: &super::Device) -> Result<T> { + if self.inner.is_err() { + dev_err!( + dev, + "{}: property '{}' is missing\n", + self.fwnode, + self.name + ); + } + self.inner + } + + /// Access the property, indicating it is optional. + /// + /// In contrast to [`Self::required_by`], no error message is logged if + /// the property is not present. + pub fn optional(self) -> Option<T> { + self.inner.ok() + } + + /// Access the property or the specified default value. + /// + /// Do not pass a sentinel value as default to detect a missing property. + /// Use [`Self::required_by`] or [`Self::optional`] instead. + pub fn or(self, default: T) -> T { + self.inner.unwrap_or(default) + } +} + +impl<T: Default> PropertyGuard<'_, '_, T> { + /// Access the property or a default value. + /// + /// Use [`Self::or`] to specify a custom default value. + pub fn or_default(self) -> T { + self.inner.unwrap_or_default() + } +} diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs index 0a4eb56d98f2..8ed2c946144c 100644 --- a/rust/kernel/device_id.rs +++ b/rust/kernel/device_id.rs @@ -14,32 +14,41 @@ use core::mem::MaybeUninit; /// /// # Safety /// -/// Implementers must ensure that: -/// - `Self` is layout-compatible with [`RawDeviceId::RawType`]; i.e. it's safe to transmute to -/// `RawDeviceId`. +/// Implementers must ensure that `Self` is layout-compatible with [`RawDeviceId::RawType`]; +/// i.e. it's safe to transmute to `RawDeviceId`. /// -/// This requirement is needed so `IdArray::new` can convert `Self` to `RawType` when building -/// the ID table. +/// This requirement is needed so `IdArray::new` can convert `Self` to `RawType` when building +/// the ID table. /// -/// Ideally, this should be achieved using a const function that does conversion instead of -/// transmute; however, const trait functions relies on `const_trait_impl` unstable feature, -/// which is broken/gone in Rust 1.73. -/// -/// - `DRIVER_DATA_OFFSET` is the offset of context/data field of the device ID (usually named -/// `driver_data`) of the device ID, the field is suitable sized to write a `usize` value. -/// -/// Similar to the previous requirement, the data should ideally be added during `Self` to -/// `RawType` conversion, but there's currently no way to do it when using traits in const. +/// Ideally, this should be achieved using a const function that does conversion instead of +/// transmute; however, const trait functions relies on `const_trait_impl` unstable feature, +/// which is broken/gone in Rust 1.73. pub unsafe trait RawDeviceId { /// The raw type that holds the device id. /// /// Id tables created from [`Self`] are going to hold this type in its zero-terminated array. type RawType: Copy; +} - /// The offset to the context/data field. +/// Extension trait for [`RawDeviceId`] for devices that embed an index or context value. +/// +/// This is typically used when the device ID struct includes a field like `driver_data` +/// that is used to store a pointer-sized value (e.g., an index or context pointer). +/// +/// # Safety +/// +/// Implementers must ensure that `DRIVER_DATA_OFFSET` is the correct offset (in bytes) to +/// the context/data field (e.g., the `driver_data` field) within the raw device ID structure. +/// This field must be correctly sized to hold a `usize`. +/// +/// Ideally, the data should be added during `Self` to `RawType` conversion, +/// but there's currently no way to do it when using traits in const. +pub unsafe trait RawDeviceIdIndex: RawDeviceId { + /// The offset (in bytes) to the context/data field in the raw device ID. const DRIVER_DATA_OFFSET: usize; - /// The index stored at `DRIVER_DATA_OFFSET` of the implementor of the [`RawDeviceId`] trait. + /// The index stored at `DRIVER_DATA_OFFSET` of the implementor of the [`RawDeviceIdIndex`] + /// trait. fn index(&self) -> usize; } @@ -68,7 +77,15 @@ impl<T: RawDeviceId, U, const N: usize> IdArray<T, U, N> { /// Creates a new instance of the array. /// /// The contents are derived from the given identifiers and context information. - pub const fn new(ids: [(T, U); N]) -> Self { + /// + /// # Safety + /// + /// `data_offset` as `None` is always safe. + /// If `data_offset` is `Some(data_offset)`, then: + /// - `data_offset` must be the correct offset (in bytes) to the context/data field + /// (e.g., the `driver_data` field) within the raw device ID structure. + /// - The field at `data_offset` must be correctly sized to hold a `usize`. + const unsafe fn build(ids: [(T, U); N], data_offset: Option<usize>) -> Self { let mut raw_ids = [const { MaybeUninit::<T::RawType>::uninit() }; N]; let mut infos = [const { MaybeUninit::uninit() }; N]; @@ -77,14 +94,16 @@ impl<T: RawDeviceId, U, const N: usize> IdArray<T, U, N> { // SAFETY: by the safety requirement of `RawDeviceId`, we're guaranteed that `T` is // layout-wise compatible with `RawType`. raw_ids[i] = unsafe { core::mem::transmute_copy(&ids[i].0) }; - // SAFETY: by the safety requirement of `RawDeviceId`, this would be effectively - // `raw_ids[i].driver_data = i;`. - unsafe { - raw_ids[i] - .as_mut_ptr() - .byte_offset(T::DRIVER_DATA_OFFSET as _) - .cast::<usize>() - .write(i); + if let Some(data_offset) = data_offset { + // SAFETY: by the safety requirement of this function, this would be effectively + // `raw_ids[i].driver_data = i;`. + unsafe { + raw_ids[i] + .as_mut_ptr() + .byte_offset(data_offset as _) + .cast::<usize>() + .write(i); + } } // SAFETY: this is effectively a move: `infos[i] = ids[i].1`. We make a copy here but @@ -109,12 +128,34 @@ impl<T: RawDeviceId, U, const N: usize> IdArray<T, U, N> { } } + /// Creates a new instance of the array without writing index values. + /// + /// The contents are derived from the given identifiers and context information. + /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead. + pub const fn new_without_index(ids: [(T, U); N]) -> Self { + // SAFETY: Calling `Self::build` with `offset = None` is always safe, + // because no raw memory writes are performed in this case. + unsafe { Self::build(ids, None) } + } + /// Reference to the contained [`RawIdArray`]. pub const fn raw_ids(&self) -> &RawIdArray<T, N> { &self.raw_ids } } +impl<T: RawDeviceId + RawDeviceIdIndex, U, const N: usize> IdArray<T, U, N> { + /// Creates a new instance of the array. + /// + /// The contents are derived from the given identifiers and context information. + pub const fn new(ids: [(T, U); N]) -> Self { + // SAFETY: by the safety requirement of `RawDeviceIdIndex`, + // `T::DRIVER_DATA_OFFSET` is guaranteed to be the correct offset (in bytes) to + // a field within `T::RawType`. + unsafe { Self::build(ids, Some(T::DRIVER_DATA_OFFSET)) } + } +} + /// A device id table. /// /// This trait is only implemented by `IdArray`. diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 57502534d985..152a89b78943 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -9,20 +9,25 @@ use crate::{ alloc::Flags, bindings, device::{Bound, Device}, - error::{Error, Result}, + error::{to_result, Error, Result}, ffi::c_void, prelude::*, revocable::{Revocable, RevocableGuard}, - sync::{rcu, Arc, Completion}, - types::ARef, + sync::{rcu, Completion}, + types::{ARef, ForeignOwnable, Opaque, ScopeGuard}, }; +use pin_init::Wrapper; + +/// [`Devres`] inner data accessed from [`Devres::callback`]. #[pin_data] -struct DevresInner<T> { - dev: ARef<Device>, - callback: unsafe extern "C" fn(*mut c_void), +struct Inner<T: Send> { #[pin] data: Revocable<T>, + /// Tracks whether [`Devres::callback`] has been completed. + #[pin] + devm: Completion, + /// Tracks whether revoking [`Self::data`] has been completed. #[pin] revoke: Completion, } @@ -47,7 +52,7 @@ struct DevresInner<T> { /// # Example /// /// ```no_run -/// # use kernel::{bindings, c_str, device::{Bound, Device}, devres::Devres, io::{Io, IoRaw}}; +/// # use kernel::{bindings, device::{Bound, Device}, devres::Devres, io::{Io, IoRaw}}; /// # use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. @@ -88,108 +93,120 @@ struct DevresInner<T> { /// # fn no_run(dev: &Device<Bound>) -> Result<(), Error> { /// // SAFETY: Invalid usage for example purposes. /// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? }; -/// let devres = Devres::new(dev, iomem, GFP_KERNEL)?; +/// let devres = KBox::pin_init(Devres::new(dev, iomem), GFP_KERNEL)?; /// /// let res = devres.try_access().ok_or(ENXIO)?; /// res.write8(0x42, 0x0); /// # Ok(()) /// # } /// ``` -pub struct Devres<T>(Arc<DevresInner<T>>); - -impl<T> DevresInner<T> { - fn new(dev: &Device<Bound>, data: T, flags: Flags) -> Result<Arc<DevresInner<T>>> { - let inner = Arc::pin_init( - pin_init!( DevresInner { - dev: dev.into(), - callback: Self::devres_callback, - data <- Revocable::new(data), - revoke <- Completion::new(), - }), - flags, - )?; +/// +/// # Invariants +/// +/// [`Self::inner`] is guaranteed to be initialized and is always accessed read-only. +#[pin_data(PinnedDrop)] +pub struct Devres<T: Send> { + dev: ARef<Device>, + /// Pointer to [`Self::devres_callback`]. + /// + /// Has to be stored, since Rust does not guarantee to always return the same address for a + /// function. However, the C API uses the address as a key. + callback: unsafe extern "C" fn(*mut c_void), + /// Contains all the fields shared with [`Self::callback`]. + // TODO: Replace with `UnsafePinned`, once available. + // + // Subsequently, the `drop_in_place()` in `Devres::drop` and the explicit `Send` and `Sync' + // impls can be removed. + #[pin] + inner: Opaque<Inner<T>>, +} - // Convert `Arc<DevresInner>` into a raw pointer and make devres own this reference until - // `Self::devres_callback` is called. - let data = inner.clone().into_raw(); +impl<T: Send> Devres<T> { + /// Creates a new [`Devres`] instance of the given `data`. + /// + /// The `data` encapsulated within the returned `Devres` instance' `data` will be + /// (revoked)[`Revocable`] once the device is detached. + pub fn new<'a, E>( + dev: &'a Device<Bound>, + data: impl PinInit<T, E> + 'a, + ) -> impl PinInit<Self, Error> + 'a + where + T: 'a, + Error: From<E>, + { + let callback = Self::devres_callback; - // SAFETY: `devm_add_action` guarantees to call `Self::devres_callback` once `dev` is - // detached. - let ret = - unsafe { bindings::devm_add_action(dev.as_raw(), Some(inner.callback), data as _) }; + try_pin_init!(&this in Self { + dev: dev.into(), + callback, + // INVARIANT: `inner` is properly initialized. + inner <- { + // SAFETY: `this` is a valid pointer to uninitialized memory. + let inner = unsafe { &raw mut (*this.as_ptr()).inner }; - if ret != 0 { - // SAFETY: We just created another reference to `inner` in order to pass it to - // `bindings::devm_add_action`. If `bindings::devm_add_action` fails, we have to drop - // this reference accordingly. - let _ = unsafe { Arc::from_raw(data) }; - return Err(Error::from_errno(ret)); - } + // SAFETY: + // - `dev.as_raw()` is a pointer to a valid bound device. + // - `inner` is guaranteed to be a valid for the duration of the lifetime of `Self`. + // - `devm_add_action()` is guaranteed not to call `callback` until `this` has been + // properly initialized, because we require `dev` (i.e. the *bound* device) to + // live at least as long as the returned `impl PinInit<Self, Error>`. + to_result(unsafe { + bindings::devm_add_action(dev.as_raw(), Some(callback), inner.cast()) + })?; - Ok(inner) + Opaque::pin_init(try_pin_init!(Inner { + devm <- Completion::new(), + revoke <- Completion::new(), + data <- Revocable::new(data), + })) + }, + }) } - fn as_ptr(&self) -> *const Self { - self as _ + fn inner(&self) -> &Inner<T> { + // SAFETY: By the type invairants of `Self`, `inner` is properly initialized and always + // accessed read-only. + unsafe { &*self.inner.get() } } - fn remove_action(this: &Arc<Self>) -> bool { - // SAFETY: - // - `self.inner.dev` is a valid `Device`, - // - the `action` and `data` pointers are the exact same ones as given to devm_add_action() - // previously, - // - `self` is always valid, even if the action has been released already. - let success = unsafe { - bindings::devm_remove_action_nowarn( - this.dev.as_raw(), - Some(this.callback), - this.as_ptr() as _, - ) - } == 0; - - if success { - // SAFETY: We leaked an `Arc` reference to devm_add_action() in `DevresInner::new`; if - // devm_remove_action_nowarn() was successful we can (and have to) claim back ownership - // of this reference. - let _ = unsafe { Arc::from_raw(this.as_ptr()) }; - } - - success + fn data(&self) -> &Revocable<T> { + &self.inner().data } #[allow(clippy::missing_safety_doc)] unsafe extern "C" fn devres_callback(ptr: *mut kernel::ffi::c_void) { - let ptr = ptr as *mut DevresInner<T>; - // Devres owned this memory; now that we received the callback, drop the `Arc` and hence the - // reference. - // SAFETY: Safe, since we leaked an `Arc` reference to devm_add_action() in - // `DevresInner::new`. - let inner = unsafe { Arc::from_raw(ptr) }; + // SAFETY: In `Self::new` we've passed a valid pointer to `Inner` to `devm_add_action()`, + // hence `ptr` must be a valid pointer to `Inner`. + let inner = unsafe { &*ptr.cast::<Inner<T>>() }; + + // Ensure that `inner` can't be used anymore after we signal completion of this callback. + let inner = ScopeGuard::new_with_data(inner, |inner| inner.devm.complete_all()); if !inner.data.revoke() { // If `revoke()` returns false, it means that `Devres::drop` already started revoking - // `inner.data` for us. Hence we have to wait until `Devres::drop()` signals that it - // completed revoking `inner.data`. + // `data` for us. Hence we have to wait until `Devres::drop` signals that it + // completed revoking `data`. inner.revoke.wait_for_completion(); } } -} - -impl<T> Devres<T> { - /// Creates a new [`Devres`] instance of the given `data`. The `data` encapsulated within the - /// returned `Devres` instance' `data` will be revoked once the device is detached. - pub fn new(dev: &Device<Bound>, data: T, flags: Flags) -> Result<Self> { - let inner = DevresInner::new(dev, data, flags)?; - Ok(Devres(inner)) + fn remove_action(&self) -> bool { + // SAFETY: + // - `self.dev` is a valid `Device`, + // - the `action` and `data` pointers are the exact same ones as given to + // `devm_add_action()` previously, + (unsafe { + bindings::devm_remove_action_nowarn( + self.dev.as_raw(), + Some(self.callback), + core::ptr::from_ref(self.inner()).cast_mut().cast(), + ) + } == 0) } - /// Same as [`Devres::new`], but does not return a `Devres` instance. Instead the given `data` - /// is owned by devres and will be revoked / dropped, once the device is detached. - pub fn new_foreign_owned(dev: &Device<Bound>, data: T, flags: Flags) -> Result { - let _ = DevresInner::new(dev, data, flags)?; - - Ok(()) + /// Return a reference of the [`Device`] this [`Devres`] instance has been created with. + pub fn device(&self) -> &Device { + &self.dev } /// Obtain `&'a T`, bypassing the [`Revocable`]. @@ -221,43 +238,123 @@ impl<T> Devres<T> { /// } /// ``` pub fn access<'a>(&'a self, dev: &'a Device<Bound>) -> Result<&'a T> { - if self.0.dev.as_raw() != dev.as_raw() { + if self.dev.as_raw() != dev.as_raw() { return Err(EINVAL); } // SAFETY: `dev` being the same device as the device this `Devres` has been created for - // proves that `self.0.data` hasn't been revoked and is guaranteed to not be revoked as - // long as `dev` lives; `dev` lives at least as long as `self`. - Ok(unsafe { self.0.data.access() }) + // proves that `self.data` hasn't been revoked and is guaranteed to not be revoked as long + // as `dev` lives; `dev` lives at least as long as `self`. + Ok(unsafe { self.data().access() }) } /// [`Devres`] accessor for [`Revocable::try_access`]. pub fn try_access(&self) -> Option<RevocableGuard<'_, T>> { - self.0.data.try_access() + self.data().try_access() } /// [`Devres`] accessor for [`Revocable::try_access_with`]. pub fn try_access_with<R, F: FnOnce(&T) -> R>(&self, f: F) -> Option<R> { - self.0.data.try_access_with(f) + self.data().try_access_with(f) } /// [`Devres`] accessor for [`Revocable::try_access_with_guard`]. pub fn try_access_with_guard<'a>(&'a self, guard: &'a rcu::Guard) -> Option<&'a T> { - self.0.data.try_access_with_guard(guard) + self.data().try_access_with_guard(guard) } } -impl<T> Drop for Devres<T> { - fn drop(&mut self) { +// SAFETY: `Devres` can be send to any task, if `T: Send`. +unsafe impl<T: Send> Send for Devres<T> {} + +// SAFETY: `Devres` can be shared with any task, if `T: Sync`. +unsafe impl<T: Send + Sync> Sync for Devres<T> {} + +#[pinned_drop] +impl<T: Send> PinnedDrop for Devres<T> { + fn drop(self: Pin<&mut Self>) { // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data // anymore, hence it is safe not to wait for the grace period to finish. - if unsafe { self.0.data.revoke_nosync() } { - // We revoked `self.0.data` before the devres action did, hence try to remove it. - if !DevresInner::remove_action(&self.0) { + if unsafe { self.data().revoke_nosync() } { + // We revoked `self.data` before the devres action did, hence try to remove it. + if !self.remove_action() { // We could not remove the devres action, which means that it now runs concurrently, - // hence signal that `self.0.data` has been revoked successfully. - self.0.revoke.complete_all(); + // hence signal that `self.data` has been revoked by us successfully. + self.inner().revoke.complete_all(); + + // Wait for `Self::devres_callback` to be done using this object. + self.inner().devm.wait_for_completion(); } + } else { + // `Self::devres_callback` revokes `self.data` for us, hence wait for it to be done + // using this object. + self.inner().devm.wait_for_completion(); } + + // INVARIANT: At this point it is guaranteed that `inner` can't be accessed any more. + // + // SAFETY: `inner` is valid for dropping. + unsafe { core::ptr::drop_in_place(self.inner.get()) }; + } +} + +/// Consume `data` and [`Drop::drop`] `data` once `dev` is unbound. +fn register_foreign<P>(dev: &Device<Bound>, data: P) -> Result +where + P: ForeignOwnable + Send + 'static, +{ + let ptr = data.into_foreign(); + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn callback<P: ForeignOwnable>(ptr: *mut kernel::ffi::c_void) { + // SAFETY: `ptr` is the pointer to the `ForeignOwnable` leaked above and hence valid. + drop(unsafe { P::from_foreign(ptr.cast()) }); } + + // SAFETY: + // - `dev.as_raw()` is a pointer to a valid and bound device. + // - `ptr` is a valid pointer the `ForeignOwnable` devres takes ownership of. + to_result(unsafe { + // `devm_add_action_or_reset()` also calls `callback` on failure, such that the + // `ForeignOwnable` is released eventually. + bindings::devm_add_action_or_reset(dev.as_raw(), Some(callback::<P>), ptr.cast()) + }) +} + +/// Encapsulate `data` in a [`KBox`] and [`Drop::drop`] `data` once `dev` is unbound. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::{device::{Bound, Device}, devres}; +/// +/// /// Registration of e.g. a class device, IRQ, etc. +/// struct Registration; +/// +/// impl Registration { +/// fn new() -> Self { +/// // register +/// +/// Self +/// } +/// } +/// +/// impl Drop for Registration { +/// fn drop(&mut self) { +/// // unregister +/// } +/// } +/// +/// fn from_bound_context(dev: &Device<Bound>) -> Result { +/// devres::register(dev, Registration::new(), GFP_KERNEL) +/// } +/// ``` +pub fn register<T, E>(dev: &Device<Bound>, data: impl PinInit<T, E>, flags: Flags) -> Result +where + T: Send + 'static, + Error: From<E>, +{ + let data = KBox::pin_init(data, flags)?; + + register_foreign(dev, data) } diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index a33261c62e0c..99dcf79f0897 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -5,14 +5,156 @@ //! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) use crate::{ - bindings, build_assert, - device::{Bound, Device}, - error::code::*, - error::Result, + bindings, build_assert, device, + device::{Bound, Core}, + error::{to_result, Result}, + prelude::*, transmute::{AsBytes, FromBytes}, types::ARef, }; +/// Trait to be implemented by DMA capable bus devices. +/// +/// The [`dma::Device`](Device) trait should be implemented by bus specific device representations, +/// where the underlying bus is DMA capable, such as [`pci::Device`](::kernel::pci::Device) or +/// [`platform::Device`](::kernel::platform::Device). +pub trait Device: AsRef<device::Device<Core>> { + /// Set up the device's DMA streaming addressing capabilities. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`CoherentAllocation::alloc_attrs`]. + unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) }) + } + + /// Set up the device's DMA coherent addressing capabilities. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`CoherentAllocation::alloc_attrs`]. + unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) }) + } + + /// Set up the device's DMA addressing capabilities. + /// + /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`]. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`CoherentAllocation::alloc_attrs`]. + unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { + bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value()) + }) + } +} + +/// A DMA mask that holds a bitmask with the lowest `n` bits set. +/// +/// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values +/// are guaranteed to never exceed the bit width of `u64`. +/// +/// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DmaMask(u64); + +impl DmaMask { + /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. + /// + /// For `n <= 64`, sets exactly the lowest `n` bits. + /// For `n > 64`, results in a build error. + /// + /// # Examples + /// + /// ``` + /// use kernel::dma::DmaMask; + /// + /// let mask0 = DmaMask::new::<0>(); + /// assert_eq!(mask0.value(), 0); + /// + /// let mask1 = DmaMask::new::<1>(); + /// assert_eq!(mask1.value(), 0b1); + /// + /// let mask64 = DmaMask::new::<64>(); + /// assert_eq!(mask64.value(), u64::MAX); + /// + /// // Build failure. + /// // let mask_overflow = DmaMask::new::<100>(); + /// ``` + #[inline] + pub const fn new<const N: u32>() -> Self { + let Ok(mask) = Self::try_new(N) else { + build_error!("Invalid DMA Mask."); + }; + + mask + } + + /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. + /// + /// For `n <= 64`, sets exactly the lowest `n` bits. + /// For `n > 64`, returns [`EINVAL`]. + /// + /// # Examples + /// + /// ``` + /// use kernel::dma::DmaMask; + /// + /// let mask0 = DmaMask::try_new(0)?; + /// assert_eq!(mask0.value(), 0); + /// + /// let mask1 = DmaMask::try_new(1)?; + /// assert_eq!(mask1.value(), 0b1); + /// + /// let mask64 = DmaMask::try_new(64)?; + /// assert_eq!(mask64.value(), u64::MAX); + /// + /// let mask_overflow = DmaMask::try_new(100); + /// assert!(mask_overflow.is_err()); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub const fn try_new(n: u32) -> Result<Self> { + Ok(Self(match n { + 0 => 0, + 1..=64 => u64::MAX >> (64 - n), + _ => return Err(EINVAL), + })) + } + + /// Returns the underlying `u64` bitmask value. + #[inline] + pub const fn value(&self) -> u64 { + self.0 + } +} + /// Possible attributes associated with a DMA mapping. /// /// They can be combined with the operators `|`, `&`, and `!`. @@ -130,7 +272,7 @@ pub mod attrs { // Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the // entire `CoherentAllocation` including the allocated memory itself. pub struct CoherentAllocation<T: AsBytes + FromBytes> { - dev: ARef<Device>, + dev: ARef<device::Device>, dma_handle: bindings::dma_addr_t, count: usize, cpu_addr: *mut T, @@ -152,7 +294,7 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { /// # Ok::<(), Error>(()) } /// ``` pub fn alloc_attrs( - dev: &Device<Bound>, + dev: &device::Device<Bound>, count: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -194,7 +336,7 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the /// `dma_attrs` is 0 by default. pub fn alloc_coherent( - dev: &Device<Bound>, + dev: &device::Device<Bound>, count: usize, gfp_flags: kernel::alloc::Flags, ) -> Result<CoherentAllocation<T>> { diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index ec9166cedfa7..a8f2675ba7a7 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -6,7 +6,7 @@ //! register using the [`Registration`] class. use crate::error::{Error, Result}; -use crate::{device, of, str::CStr, try_pin_init, types::Opaque, ThisModule}; +use crate::{acpi, device, of, str::CStr, try_pin_init, types::Opaque, ThisModule}; use core::pin::Pin; use pin_init::{pin_data, pinned_drop, PinInit}; @@ -141,36 +141,76 @@ pub trait Adapter { /// The type holding driver private data about each device id supported by the driver. type IdInfo: 'static; + /// The [`acpi::IdTable`] of the corresponding driver + fn acpi_id_table() -> Option<acpi::IdTable<Self::IdInfo>>; + + /// Returns the driver's private data from the matching entry in the [`acpi::IdTable`], if any. + /// + /// If this returns `None`, it means there is no match with an entry in the [`acpi::IdTable`]. + fn acpi_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + #[cfg(not(CONFIG_ACPI))] + { + let _ = dev; + None + } + + #[cfg(CONFIG_ACPI)] + { + let table = Self::acpi_id_table()?; + + // SAFETY: + // - `table` has static lifetime, hence it's valid for read, + // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let raw_id = unsafe { bindings::acpi_match_device(table.as_ptr(), dev.as_raw()) }; + + if raw_id.is_null() { + None + } else { + // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct acpi_device_id` + // and does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*raw_id.cast::<acpi::DeviceId>() }; + + Some(table.info(<acpi::DeviceId as crate::device_id::RawDeviceIdIndex>::index(id))) + } + } + } + /// The [`of::IdTable`] of the corresponding driver. fn of_id_table() -> Option<of::IdTable<Self::IdInfo>>; /// Returns the driver's private data from the matching entry in the [`of::IdTable`], if any. /// /// If this returns `None`, it means there is no match with an entry in the [`of::IdTable`]. - #[cfg(CONFIG_OF)] fn of_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - let table = Self::of_id_table()?; - - // SAFETY: - // - `table` has static lifetime, hence it's valid for read, - // - `dev` is guaranteed to be valid while it's alive, and so is `pdev.as_ref().as_raw()`. - let raw_id = unsafe { bindings::of_match_device(table.as_ptr(), dev.as_raw()) }; - - if raw_id.is_null() { + #[cfg(not(CONFIG_OF))] + { + let _ = dev; None - } else { - // SAFETY: `DeviceId` is a `#[repr(transparent)` wrapper of `struct of_device_id` and - // does not add additional invariants, so it's safe to transmute. - let id = unsafe { &*raw_id.cast::<of::DeviceId>() }; - - Some(table.info(<of::DeviceId as crate::device_id::RawDeviceId>::index(id))) } - } - #[cfg(not(CONFIG_OF))] - #[allow(missing_docs)] - fn of_id_info(_dev: &device::Device) -> Option<&'static Self::IdInfo> { - None + #[cfg(CONFIG_OF)] + { + let table = Self::of_id_table()?; + + // SAFETY: + // - `table` has static lifetime, hence it's valid for read, + // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let raw_id = unsafe { bindings::of_match_device(table.as_ptr(), dev.as_raw()) }; + + if raw_id.is_null() { + None + } else { + // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` + // and does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*raw_id.cast::<of::DeviceId>() }; + + Some( + table.info(<of::DeviceId as crate::device_id::RawDeviceIdIndex>::index( + id, + )), + ) + } + } } /// Returns the driver's private data from the matching entry of any of the ID tables, if any. @@ -178,6 +218,11 @@ pub trait Adapter { /// If this returns `None`, it means that there is no match in any of the ID tables directly /// associated with a [`device::Device`]. fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + let id = Self::acpi_id_info(dev); + if id.is_some() { + return id; + } + let id = Self::of_id_info(dev); if id.is_some() { return id; diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 14c1aa402951..16935f42fe2e 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -200,7 +200,7 @@ impl<T: drm::Driver> AsRef<device::Device> for Device<T> { fn as_ref(&self) -> &device::Device { // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid, // which is guaranteed by the type invariant. - unsafe { device::Device::as_ref((*self.as_raw()).dev) } + unsafe { device::Device::from_raw((*self.as_raw()).dev) } } } diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index af93d46d03d3..fe7e8d06961a 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -5,9 +5,7 @@ //! C header: [`include/linux/drm/drm_drv.h`](srctree/include/linux/drm/drm_drv.h) use crate::{ - bindings, device, - devres::Devres, - drm, + bindings, device, devres, drm, error::{to_result, Result}, prelude::*, types::ARef, @@ -129,18 +127,22 @@ impl<T: Driver> Registration<T> { } /// Same as [`Registration::new`}, but transfers ownership of the [`Registration`] to - /// [`Devres`]. + /// [`devres::register`]. pub fn new_foreign_owned( drm: &drm::Device<T>, dev: &device::Device<device::Bound>, flags: usize, - ) -> Result { + ) -> Result + where + T: 'static, + { if drm.as_ref().as_raw() != dev.as_raw() { return Err(EINVAL); } let reg = Registration::<T>::new(drm, flags)?; - Devres::new_foreign_owned(dev, reg, GFP_KERNEL) + + devres::register(dev, reg, GFP_KERNEL) } /// Returns a reference to the `Device` instance for this registration. diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs index 8a50fcd4c9bb..7a906099993f 100644 --- a/rust/kernel/faux.rs +++ b/rust/kernel/faux.rs @@ -54,7 +54,7 @@ impl AsRef<device::Device> for Registration { fn as_ref(&self) -> &device::Device { // SAFETY: The underlying `device` in `faux_device` is guaranteed by the C API to be // a valid initialized `device`. - unsafe { device::Device::as_ref(addr_of_mut!((*self.as_raw()).dev)) } + unsafe { device::Device::from_raw(addr_of_mut!((*self.as_raw()).dev)) } } } diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 72d80a6f131e..b7fc759f8b5d 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -7,6 +7,11 @@ use crate::error::{code::EINVAL, Result}; use crate::{bindings, build_assert}; +pub mod mem; +pub mod resource; + +pub use resource::Resource; + /// Raw representation of an MMIO region. /// /// By itself, the existence of an instance of this structure does not provide any guarantees that diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs new file mode 100644 index 000000000000..6f99510bfc3a --- /dev/null +++ b/rust/kernel/io/mem.rs @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic memory-mapped IO. + +use core::ops::Deref; + +use crate::c_str; +use crate::device::Bound; +use crate::device::Device; +use crate::devres::Devres; +use crate::io; +use crate::io::resource::Region; +use crate::io::resource::Resource; +use crate::io::Io; +use crate::io::IoRaw; +use crate::prelude::*; + +/// An IO request for a specific device and resource. +pub struct IoRequest<'a> { + device: &'a Device<Bound>, + resource: &'a Resource, +} + +impl<'a> IoRequest<'a> { + /// Creates a new [`IoRequest`] instance. + /// + /// # Safety + /// + /// Callers must ensure that `resource` is valid for `device` during the + /// lifetime `'a`. + pub(crate) unsafe fn new(device: &'a Device<Bound>, resource: &'a Resource) -> Self { + IoRequest { device, resource } + } + + /// Maps an [`IoRequest`] where the size is known at compile time. + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + /// + /// # Examples + /// + /// The following example uses a [`kernel::platform::Device`] for + /// illustration purposes. + /// + /// ```no_run + /// use kernel::{bindings, c_str, platform, of, device::Core}; + /// struct SampleDriver; + /// + /// impl platform::Driver for SampleDriver { + /// # type IdInfo = (); + /// + /// fn probe( + /// pdev: &platform::Device<Core>, + /// info: Option<&Self::IdInfo>, + /// ) -> Result<Pin<KBox<Self>>> { + /// let offset = 0; // Some offset. + /// + /// // If the size is known at compile time, use [`Self::iomap_sized`]. + /// // + /// // No runtime checks will apply when reading and writing. + /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; + /// let iomem = request.iomap_sized::<42>(); + /// let iomem = KBox::pin_init(iomem, GFP_KERNEL)?; + /// + /// let io = iomem.access(pdev.as_ref())?; + /// + /// // Read and write a 32-bit value at `offset`. + /// let data = io.read32_relaxed(offset); + /// + /// io.write32_relaxed(data, offset); + /// + /// # Ok(KBox::new(SampleDriver, GFP_KERNEL)?.into()) + /// } + /// } + /// ``` + pub fn iomap_sized<const SIZE: usize>(self) -> impl PinInit<Devres<IoMem<SIZE>>, Error> + 'a { + IoMem::new(self) + } + + /// Same as [`Self::iomap_sized`] but with exclusive access to the + /// underlying region. + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + pub fn iomap_exclusive_sized<const SIZE: usize>( + self, + ) -> impl PinInit<Devres<ExclusiveIoMem<SIZE>>, Error> + 'a { + ExclusiveIoMem::new(self) + } + + /// Maps an [`IoRequest`] where the size is not known at compile time, + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + /// + /// # Examples + /// + /// The following example uses a [`kernel::platform::Device`] for + /// illustration purposes. + /// + /// ```no_run + /// use kernel::{bindings, c_str, platform, of, device::Core}; + /// struct SampleDriver; + /// + /// impl platform::Driver for SampleDriver { + /// # type IdInfo = (); + /// + /// fn probe( + /// pdev: &platform::Device<Core>, + /// info: Option<&Self::IdInfo>, + /// ) -> Result<Pin<KBox<Self>>> { + /// let offset = 0; // Some offset. + /// + /// // Unlike [`Self::iomap_sized`], here the size of the memory region + /// // is not known at compile time, so only the `try_read*` and `try_write*` + /// // family of functions should be used, leading to runtime checks on every + /// // access. + /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; + /// let iomem = request.iomap(); + /// let iomem = KBox::pin_init(iomem, GFP_KERNEL)?; + /// + /// let io = iomem.access(pdev.as_ref())?; + /// + /// let data = io.try_read32_relaxed(offset)?; + /// + /// io.try_write32_relaxed(data, offset)?; + /// + /// # Ok(KBox::new(SampleDriver, GFP_KERNEL)?.into()) + /// } + /// } + /// ``` + pub fn iomap(self) -> impl PinInit<Devres<IoMem<0>>, Error> + 'a { + Self::iomap_sized::<0>(self) + } + + /// Same as [`Self::iomap`] but with exclusive access to the underlying + /// region. + pub fn iomap_exclusive(self) -> impl PinInit<Devres<ExclusiveIoMem<0>>, Error> + 'a { + Self::iomap_exclusive_sized::<0>(self) + } +} + +/// An exclusive memory-mapped IO region. +/// +/// # Invariants +/// +/// - [`ExclusiveIoMem`] has exclusive access to the underlying [`IoMem`]. +pub struct ExclusiveIoMem<const SIZE: usize> { + /// The underlying `IoMem` instance. + iomem: IoMem<SIZE>, + + /// The region abstraction. This represents exclusive access to the + /// range represented by the underlying `iomem`. + /// + /// This field is needed for ownership of the region. + _region: Region, +} + +impl<const SIZE: usize> ExclusiveIoMem<SIZE> { + /// Creates a new `ExclusiveIoMem` instance. + fn ioremap(resource: &Resource) -> Result<Self> { + let start = resource.start(); + let size = resource.size(); + let name = resource.name().unwrap_or(c_str!("")); + + let region = resource + .request_region( + start, + size, + name.to_cstring()?, + io::resource::Flags::IORESOURCE_MEM, + ) + .ok_or(EBUSY)?; + + let iomem = IoMem::ioremap(resource)?; + + let iomem = ExclusiveIoMem { + iomem, + _region: region, + }; + + Ok(iomem) + } + + /// Creates a new `ExclusiveIoMem` instance from a previously acquired [`IoRequest`]. + pub fn new<'a>(io_request: IoRequest<'a>) -> impl PinInit<Devres<Self>, Error> + 'a { + let dev = io_request.device; + let res = io_request.resource; + + Devres::new(dev, Self::ioremap(res)) + } +} + +impl<const SIZE: usize> Deref for ExclusiveIoMem<SIZE> { + type Target = Io<SIZE>; + + fn deref(&self) -> &Self::Target { + &self.iomem + } +} + +/// A generic memory-mapped IO region. +/// +/// Accesses to the underlying region is checked either at compile time, if the +/// region's size is known at that point, or at runtime otherwise. +/// +/// # Invariants +/// +/// [`IoMem`] always holds an [`IoRaw`] instance that holds a valid pointer to the +/// start of the I/O memory mapped region. +pub struct IoMem<const SIZE: usize = 0> { + io: IoRaw<SIZE>, +} + +impl<const SIZE: usize> IoMem<SIZE> { + fn ioremap(resource: &Resource) -> Result<Self> { + // Note: Some ioremap() implementations use types that depend on the CPU + // word width rather than the bus address width. + // + // TODO: Properly address this in the C code to avoid this `try_into`. + let size = resource.size().try_into()?; + if size == 0 { + return Err(EINVAL); + } + + let res_start = resource.start(); + + let addr = if resource + .flags() + .contains(io::resource::Flags::IORESOURCE_MEM_NONPOSTED) + { + // SAFETY: + // - `res_start` and `size` are read from a presumably valid `struct resource`. + // - `size` is known not to be zero at this point. + unsafe { bindings::ioremap_np(res_start, size) } + } else { + // SAFETY: + // - `res_start` and `size` are read from a presumably valid `struct resource`. + // - `size` is known not to be zero at this point. + unsafe { bindings::ioremap(res_start, size) } + }; + + if addr.is_null() { + return Err(ENOMEM); + } + + let io = IoRaw::new(addr as usize, size)?; + let io = IoMem { io }; + + Ok(io) + } + + /// Creates a new `IoMem` instance from a previously acquired [`IoRequest`]. + pub fn new<'a>(io_request: IoRequest<'a>) -> impl PinInit<Devres<Self>, Error> + 'a { + let dev = io_request.device; + let res = io_request.resource; + + Devres::new(dev, Self::ioremap(res)) + } +} + +impl<const SIZE: usize> Drop for IoMem<SIZE> { + fn drop(&mut self) { + // SAFETY: Safe as by the invariant of `Io`. + unsafe { bindings::iounmap(self.io.addr() as *mut c_void) } + } +} + +impl<const SIZE: usize> Deref for IoMem<SIZE> { + type Target = Io<SIZE>; + + fn deref(&self) -> &Self::Target { + // SAFETY: Safe as by the invariant of `IoMem`. + unsafe { Io::from_raw(&self.io) } + } +} diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs new file mode 100644 index 000000000000..bea3ee0ed87b --- /dev/null +++ b/rust/kernel/io/resource.rs @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for [system +//! resources](https://docs.kernel.org/core-api/kernel-api.html#resources-management). +//! +//! C header: [`include/linux/ioport.h`](srctree/include/linux/ioport.h) + +use core::ops::Deref; +use core::ptr::NonNull; + +use crate::prelude::*; +use crate::str::{CStr, CString}; +use crate::types::Opaque; + +/// Resource Size type. +/// +/// This is a type alias to either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. +pub type ResourceSize = bindings::phys_addr_t; + +/// A region allocated from a parent [`Resource`]. +/// +/// # Invariants +/// +/// - `self.0` points to a valid `bindings::resource` that was obtained through +/// `bindings::__request_region`. +pub struct Region { + /// The resource returned when the region was requested. + resource: NonNull<bindings::resource>, + /// The name that was passed in when the region was requested. We need to + /// store it for ownership reasons. + _name: CString, +} + +impl Deref for Region { + type Target = Resource; + + fn deref(&self) -> &Self::Target { + // SAFETY: Safe as per the invariant of `Region`. + unsafe { Resource::from_raw(self.resource.as_ptr()) } + } +} + +impl Drop for Region { + fn drop(&mut self) { + let (flags, start, size) = { + let res = &**self; + (res.flags(), res.start(), res.size()) + }; + + let release_fn = if flags.contains(Flags::IORESOURCE_MEM) { + bindings::release_mem_region + } else { + bindings::release_region + }; + + // SAFETY: Safe as per the invariant of `Region`. + unsafe { release_fn(start, size) }; + } +} + +// SAFETY: `Region` only holds a pointer to a C `struct resource`, which is safe to be used from +// any thread. +unsafe impl Send for Region {} + +// SAFETY: `Region` only holds a pointer to a C `struct resource`, references to which are +// safe to be used from any thread. +unsafe impl Sync for Region {} + +/// A resource abstraction. +/// +/// # Invariants +/// +/// [`Resource`] is a transparent wrapper around a valid `bindings::resource`. +#[repr(transparent)] +pub struct Resource(Opaque<bindings::resource>); + +impl Resource { + /// Creates a reference to a [`Resource`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that for the duration of 'a, the pointer will + /// point at a valid `bindings::resource`. + /// + /// The caller must also ensure that the [`Resource`] is only accessed via the + /// returned reference for the duration of 'a. + pub(crate) const unsafe fn from_raw<'a>(ptr: *mut bindings::resource) -> &'a Self { + // SAFETY: Self is a transparent wrapper around `Opaque<bindings::resource>`. + unsafe { &*ptr.cast() } + } + + /// Requests a resource region. + /// + /// Exclusive access will be given and the region will be marked as busy. + /// Further calls to [`Self::request_region`] will return [`None`] if + /// the region, or a part of it, is already in use. + pub fn request_region( + &self, + start: ResourceSize, + size: ResourceSize, + name: CString, + flags: Flags, + ) -> Option<Region> { + // SAFETY: + // - Safe as per the invariant of `Resource`. + // - `__request_region` will store a reference to the name, but that is + // safe as we own it and it will not be dropped until the `Region` is + // dropped. + let region = unsafe { + bindings::__request_region( + self.0.get(), + start, + size, + name.as_char_ptr(), + flags.0 as c_int, + ) + }; + + Some(Region { + resource: NonNull::new(region)?, + _name: name, + }) + } + + /// Returns the size of the resource. + pub fn size(&self) -> ResourceSize { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + unsafe { bindings::resource_size(inner) } + } + + /// Returns the start address of the resource. + pub fn start(&self) -> ResourceSize { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + unsafe { (*inner).start } + } + + /// Returns the name of the resource. + pub fn name(&self) -> Option<&CStr> { + let inner = self.0.get(); + + // SAFETY: Safe as per the invariants of `Resource`. + let name = unsafe { (*inner).name }; + + if name.is_null() { + return None; + } + + // SAFETY: In the C code, `resource::name` either contains a null + // pointer or points to a valid NUL-terminated C string, and at this + // point we know it is not null, so we can safely convert it to a + // `CStr`. + Some(unsafe { CStr::from_char_ptr(name) }) + } + + /// Returns the flags associated with the resource. + pub fn flags(&self) -> Flags { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + let flags = unsafe { (*inner).flags }; + + Flags(flags) + } +} + +// SAFETY: `Resource` only holds a pointer to a C `struct resource`, which is +// safe to be used from any thread. +unsafe impl Send for Resource {} + +// SAFETY: `Resource` only holds a pointer to a C `struct resource`, references +// to which are safe to be used from any thread. +unsafe impl Sync for Resource {} + +/// Resource flags as stored in the C `struct resource::flags` field. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the associated constants such as +/// [`Flags::IORESOURCE_IO`]. +#[derive(Clone, Copy, PartialEq)] +pub struct Flags(c_ulong); + +impl Flags { + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Flags) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for Flags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Flags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Flags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +impl Flags { + /// PCI/ISA I/O ports. + pub const IORESOURCE_IO: Flags = Flags::new(bindings::IORESOURCE_IO); + + /// Resource is software muxed. + pub const IORESOURCE_MUXED: Flags = Flags::new(bindings::IORESOURCE_MUXED); + + /// Resource represents a memory region. + pub const IORESOURCE_MEM: Flags = Flags::new(bindings::IORESOURCE_MEM); + + /// Resource represents a memory region that must be ioremaped using `ioremap_np`. + pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED); + + const fn new(value: u32) -> Self { + crate::build_assert!(value as u64 <= c_ulong::MAX as u64); + Flags(value as c_ulong) + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index e13d6ed88fa6..c2d1b9375205 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -43,6 +43,10 @@ #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(coerce_unsized))] #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(dispatch_from_dyn))] #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(unsize))] +// +// `feature(file_with_nul)` is expected to become stable. Before Rust 1.89.0, it did not exist, so +// enable it conditionally. +#![cfg_attr(CONFIG_RUSTC_HAS_FILE_WITH_NUL, feature(file_with_nul))] // Ensure conditional compilation based on the kernel configuration works; // otherwise we may silently break things like initcall handling. @@ -54,6 +58,7 @@ extern crate self as kernel; pub use ffi; +pub mod acpi; pub mod alloc; #[cfg(CONFIG_AUXILIARY_BUS)] pub mod auxiliary; @@ -103,6 +108,7 @@ pub mod platform; pub mod prelude; pub mod print; pub mod rbtree; +pub mod regulator; pub mod revocable; pub mod security; pub mod seq_file; @@ -277,3 +283,47 @@ macro_rules! asm { ::core::arch::asm!( $($asm)*, $($rest)* ) }; } + +/// Gets the C string file name of a [`Location`]. +/// +/// If `file_with_nul()` is not available, returns a string that warns about it. +/// +/// [`Location`]: core::panic::Location +/// +/// # Examples +/// +/// ``` +/// # use kernel::file_from_location; +/// +/// #[track_caller] +/// fn foo() { +/// let caller = core::panic::Location::caller(); +/// +/// // Output: +/// // - A path like "rust/kernel/example.rs" if file_with_nul() is available. +/// // - "<Location::file_with_nul() not supported>" otherwise. +/// let caller_file = file_from_location(caller); +/// +/// // Prints out the message with caller's file name. +/// pr_info!("foo() called in file {caller_file:?}\n"); +/// +/// # if cfg!(CONFIG_RUSTC_HAS_FILE_WITH_NUL) { +/// # assert_eq!(Ok(caller.file()), caller_file.to_str()); +/// # } +/// } +/// +/// # foo(); +/// ``` +#[inline] +pub fn file_from_location<'a>(loc: &'a core::panic::Location<'a>) -> &'a core::ffi::CStr { + #[cfg(CONFIG_RUSTC_HAS_FILE_WITH_NUL)] + { + loc.file_with_nul() + } + + #[cfg(not(CONFIG_RUSTC_HAS_FILE_WITH_NUL))] + { + let _ = loc; + c"<Location::file_with_nul() not supported>" + } +} diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs index 939278bc7b03..a1eb5737e3cb 100644 --- a/rust/kernel/miscdevice.rs +++ b/rust/kernel/miscdevice.rs @@ -17,7 +17,6 @@ use crate::{ mm::virt::VmaNew, prelude::*, seq_file::SeqFile, - str::CStr, types::{ForeignOwnable, Opaque}, }; use core::{marker::PhantomData, mem::MaybeUninit, pin::Pin}; @@ -45,7 +44,13 @@ impl MiscDeviceOptions { /// /// # Invariants /// -/// `inner` is a registered misc device. +/// - `inner` contains a `struct miscdevice` that is registered using +/// `misc_register()`. +/// - This registration remains valid for the entire lifetime of the +/// [`MiscDeviceRegistration`] instance. +/// - Deregistration occurs exactly once in [`Drop`] via `misc_deregister()`. +/// - `inner` wraps a valid, pinned `miscdevice` created using +/// [`MiscDeviceOptions::into_raw`]. #[repr(transparent)] #[pin_data(PinnedDrop)] pub struct MiscDeviceRegistration<T> { @@ -92,7 +97,7 @@ impl<T: MiscDevice> MiscDeviceRegistration<T> { // function tells the borrow-checker that the `&Device` reference must not outlive the // `&MiscDeviceRegistration<T>` used to obtain it, so the last use of the reference must be // before the underlying `struct miscdevice` is destroyed. - unsafe { Device::as_ref((*self.as_raw()).this_device) } + unsafe { Device::from_raw((*self.as_raw()).this_device) } } } diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 32ea43ece646..95bd17d3e927 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -6,7 +6,7 @@ //! //! C headers: [`include/linux/phy.h`](srctree/include/linux/phy.h). -use crate::{error::*, prelude::*, types::Opaque}; +use crate::{device_id::RawDeviceId, error::*, prelude::*, types::Opaque}; use core::{marker::PhantomData, ptr::addr_of_mut}; pub mod reg; @@ -285,7 +285,7 @@ impl AsRef<kernel::device::Device> for Device { fn as_ref(&self) -> &kernel::device::Device { let phydev = self.0.get(); // SAFETY: The struct invariant ensures that `mdio.dev` is valid. - unsafe { kernel::device::Device::as_ref(addr_of_mut!((*phydev).mdio.dev)) } + unsafe { kernel::device::Device::from_raw(addr_of_mut!((*phydev).mdio.dev)) } } } @@ -507,7 +507,7 @@ pub const fn create_phy_driver<T: Driver>() -> DriverVTable { DriverVTable(Opaque::new(bindings::phy_driver { name: T::NAME.as_char_ptr().cast_mut(), flags: T::FLAGS, - phy_id: T::PHY_DEVICE_ID.id, + phy_id: T::PHY_DEVICE_ID.id(), phy_id_mask: T::PHY_DEVICE_ID.mask_as_int(), soft_reset: if T::HAS_SOFT_RESET { Some(Adapter::<T>::soft_reset_callback) @@ -691,42 +691,41 @@ impl Drop for Registration { /// /// Represents the kernel's `struct mdio_device_id`. This is used to find an appropriate /// PHY driver. -pub struct DeviceId { - id: u32, - mask: DeviceMask, -} +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::mdio_device_id); impl DeviceId { /// Creates a new instance with the exact match mask. pub const fn new_with_exact_mask(id: u32) -> Self { - DeviceId { - id, - mask: DeviceMask::Exact, - } + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Exact.as_int(), + }) } /// Creates a new instance with the model match mask. pub const fn new_with_model_mask(id: u32) -> Self { - DeviceId { - id, - mask: DeviceMask::Model, - } + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Model.as_int(), + }) } /// Creates a new instance with the vendor match mask. pub const fn new_with_vendor_mask(id: u32) -> Self { - DeviceId { - id, - mask: DeviceMask::Vendor, - } + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Vendor.as_int(), + }) } /// Creates a new instance with a custom match mask. pub const fn new_with_custom_mask(id: u32, mask: u32) -> Self { - DeviceId { - id, - mask: DeviceMask::Custom(mask), - } + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Custom(mask).as_int(), + }) } /// Creates a new instance from [`Driver`]. @@ -734,21 +733,29 @@ impl DeviceId { T::PHY_DEVICE_ID } - /// Get a `mask` as u32. + /// Get the MDIO device's PHY ID. + pub const fn id(&self) -> u32 { + self.0.phy_id + } + + /// Get the MDIO device's match mask. pub const fn mask_as_int(&self) -> u32 { - self.mask.as_int() + self.0.phy_id_mask } // macro use only #[doc(hidden)] pub const fn mdio_device_id(&self) -> bindings::mdio_device_id { - bindings::mdio_device_id { - phy_id: self.id, - phy_id_mask: self.mask.as_int(), - } + self.0 } } +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct mdio_device_id` +// and does not add additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::mdio_device_id; +} + enum DeviceMask { Exact, Model, @@ -849,19 +856,18 @@ impl DeviceMask { /// } /// }; /// -/// const _DEVICE_TABLE: [::kernel::bindings::mdio_device_id; 2] = [ -/// ::kernel::bindings::mdio_device_id { -/// phy_id: 0x00000001, -/// phy_id_mask: 0xffffffff, -/// }, -/// ::kernel::bindings::mdio_device_id { -/// phy_id: 0, -/// phy_id_mask: 0, -/// }, -/// ]; -/// #[cfg(MODULE)] -/// #[no_mangle] -/// static __mod_device_table__mdio__phydev: [::kernel::bindings::mdio_device_id; 2] = _DEVICE_TABLE; +/// const N: usize = 1; +/// +/// const TABLE: ::kernel::device_id::IdArray<::kernel::net::phy::DeviceId, (), N> = +/// ::kernel::device_id::IdArray::new_without_index([ +/// ::kernel::net::phy::DeviceId( +/// ::kernel::bindings::mdio_device_id { +/// phy_id: 0x00000001, +/// phy_id_mask: 0xffffffff, +/// }), +/// ]); +/// +/// ::kernel::module_device_table!("mdio", phydev, TABLE); /// ``` #[macro_export] macro_rules! module_phy_driver { @@ -872,20 +878,12 @@ macro_rules! module_phy_driver { }; (@device_table [$($dev:expr),+]) => { - // SAFETY: C will not read off the end of this constant since the last element is zero. - const _DEVICE_TABLE: [$crate::bindings::mdio_device_id; - $crate::module_phy_driver!(@count_devices $($dev),+) + 1] = [ - $($dev.mdio_device_id()),+, - $crate::bindings::mdio_device_id { - phy_id: 0, - phy_id_mask: 0 - } - ]; + const N: usize = $crate::module_phy_driver!(@count_devices $($dev),+); + + const TABLE: $crate::device_id::IdArray<$crate::net::phy::DeviceId, (), N> = + $crate::device_id::IdArray::new_without_index([ $(($dev,())),+, ]); - #[cfg(MODULE)] - #[no_mangle] - static __mod_device_table__mdio__phydev: [$crate::bindings::mdio_device_id; - $crate::module_phy_driver!(@count_devices $($dev),+) + 1] = _DEVICE_TABLE; + $crate::module_device_table!("mdio", phydev, TABLE); }; (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => { diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index 04f2d8ef29cb..0888469bddb7 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -2,7 +2,11 @@ //! Device Tree / Open Firmware abstractions. -use crate::{bindings, device_id::RawDeviceId, prelude::*}; +use crate::{ + bindings, + device_id::{RawDeviceId, RawDeviceIdIndex}, + prelude::*, +}; /// IdTable type for OF drivers. pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; @@ -12,13 +16,14 @@ pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; #[derive(Clone, Copy)] pub struct DeviceId(bindings::of_device_id); -// SAFETY: -// * `DeviceId` is a `#[repr(transparent)` wrapper of `struct of_device_id` and does not add -// additional invariants, so it's safe to transmute to `RawType`. -// * `DRIVER_DATA_OFFSET` is the offset to the `data` field. +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` and +// does not add additional invariants, so it's safe to transmute to `RawType`. unsafe impl RawDeviceId for DeviceId { type RawType = bindings::of_device_id; +} +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `data` field. +unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::of_device_id, data); fn index(&self) -> usize { diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs index a566fc3e7dcb..846583da9a2f 100644 --- a/rust/kernel/opp.rs +++ b/rust/kernel/opp.rs @@ -514,9 +514,9 @@ impl<T: ConfigOps + Default> Config<T> { dev: *mut bindings::device, opp_table: *mut bindings::opp_table, opp: *mut bindings::dev_pm_opp, - _data: *mut kernel::ffi::c_void, + _data: *mut c_void, scaling_down: bool, - ) -> kernel::ffi::c_int { + ) -> c_int { from_result(|| { // SAFETY: 'dev' is guaranteed by the C code to be valid. let dev = unsafe { Device::get_device(dev) }; @@ -540,8 +540,8 @@ impl<T: ConfigOps + Default> Config<T> { old_opp: *mut bindings::dev_pm_opp, new_opp: *mut bindings::dev_pm_opp, regulators: *mut *mut bindings::regulator, - count: kernel::ffi::c_uint, - ) -> kernel::ffi::c_int { + count: c_uint, + ) -> c_int { from_result(|| { // SAFETY: 'dev' is guaranteed by the C code to be valid. let dev = unsafe { Device::get_device(dev) }; diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 8435f8132e38..44a2f3d2884a 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -5,16 +5,15 @@ //! C header: [`include/linux/pci.h`](srctree/include/linux/pci.h) use crate::{ - alloc::flags::*, bindings, container_of, device, - device_id::RawDeviceId, + device_id::{RawDeviceId, RawDeviceIdIndex}, devres::Devres, driver, - error::{to_result, Result}, + error::{from_result, to_result, Result}, io::Io, io::IoRaw, str::CStr, - types::{ARef, ForeignOwnable, Opaque}, + types::{ARef, Opaque}, ThisModule, }; use core::{ @@ -66,35 +65,34 @@ impl<T: Driver + 'static> Adapter<T> { // `struct pci_dev`. // // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. - let pdev = unsafe { &*pdev.cast::<Device<device::Core>>() }; + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; - // SAFETY: `DeviceId` is a `#[repr(transparent)` wrapper of `struct pci_device_id` and + // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct pci_device_id` and // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::<DeviceId>() }; let info = T::ID_TABLE.info(id.index()); - match T::probe(pdev, info) { - Ok(data) => { - // Let the `struct pci_dev` own a reference of the driver's private data. - // SAFETY: By the type invariant `pdev.as_raw` returns a valid pointer to a - // `struct pci_dev`. - unsafe { bindings::pci_set_drvdata(pdev.as_raw(), data.into_foreign() as _) }; - } - Err(err) => return Error::to_errno(err), - } + from_result(|| { + let data = T::probe(pdev, info)?; - 0 + pdev.as_ref().set_drvdata(data); + Ok(0) + }) } extern "C" fn remove_callback(pdev: *mut bindings::pci_dev) { // SAFETY: The PCI bus only ever calls the remove callback with a valid pointer to a // `struct pci_dev`. - let ptr = unsafe { bindings::pci_get_drvdata(pdev) }.cast(); + // + // INVARIANT: `pdev` is valid for the duration of `remove_callback()`. + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to - // `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized - // `KBox<T>` pointer created through `KBox::into_foreign`. - let _ = unsafe { KBox::<T>::from_foreign(ptr) }; + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<T>>`. + let data = unsafe { pdev.as_ref().drvdata_obtain::<Pin<KBox<T>>>() }; + + T::unbind(pdev, data.as_ref()); } } @@ -161,13 +159,14 @@ impl DeviceId { } } -// SAFETY: -// * `DeviceId` is a `#[repr(transparent)` wrapper of `pci_device_id` and does not add -// additional invariants, so it's safe to transmute to `RawType`. -// * `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `pci_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. unsafe impl RawDeviceId for DeviceId { type RawType = bindings::pci_device_id; +} +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::pci_device_id, driver_data); fn index(&self) -> usize { @@ -241,6 +240,20 @@ pub trait Driver: Send { /// Called when a new platform device is added or discovered. /// Implementers should attempt to initialize the device here. fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>; + + /// Platform driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Self::drop`. + fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) { + let _ = (dev, this); + } } /// The PCI device representation. @@ -251,7 +264,8 @@ pub trait Driver: Send { /// /// # Invariants /// -/// A [`Device`] instance represents a valid `struct device` created by the C portion of the kernel. +/// A [`Device`] instance represents a valid `struct pci_dev` created by the C portion of the +/// kernel. #[repr(transparent)] pub struct Device<Ctx: device::DeviceContext = device::Normal>( Opaque<bindings::pci_dev>, @@ -398,19 +412,20 @@ impl Device { impl Device<device::Bound> { /// Mapps an entire PCI-BAR after performing a region-request on it. I/O operation bound checks /// can be performed on compile time for offsets (plus the requested type size) < SIZE. - pub fn iomap_region_sized<const SIZE: usize>( - &self, + pub fn iomap_region_sized<'a, const SIZE: usize>( + &'a self, bar: u32, - name: &CStr, - ) -> Result<Devres<Bar<SIZE>>> { - let bar = Bar::<SIZE>::new(self, bar, name)?; - let devres = Devres::new(self.as_ref(), bar, GFP_KERNEL)?; - - Ok(devres) + name: &'a CStr, + ) -> impl PinInit<Devres<Bar<SIZE>>, Error> + 'a { + Devres::new(self.as_ref(), Bar::<SIZE>::new(self, bar, name)) } /// Mapps an entire PCI-BAR after performing a region-request on it. - pub fn iomap_region(&self, bar: u32, name: &CStr) -> Result<Devres<Bar>> { + pub fn iomap_region<'a>( + &'a self, + bar: u32, + name: &'a CStr, + ) -> impl PinInit<Devres<Bar>, Error> + 'a { self.iomap_region_sized::<0>(bar, name) } } @@ -434,6 +449,8 @@ impl Device<device::Core> { kernel::impl_device_context_deref!(unsafe { Device }); kernel::impl_device_context_into_aref!(Device); +impl crate::dma::Device for Device<device::Core> {} + // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::types::AlwaysRefCounted for Device { fn inc_ref(&self) { @@ -454,7 +471,7 @@ impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> { let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; // SAFETY: `dev` points to a valid `struct device`. - unsafe { device::Device::as_ref(dev) } + unsafe { device::Device::from_raw(dev) } } } diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 5b21fa517e55..b4d3087aff52 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -5,12 +5,14 @@ //! C header: [`include/linux/platform_device.h`](srctree/include/linux/platform_device.h) use crate::{ - bindings, container_of, device, driver, - error::{to_result, Result}, + acpi, bindings, container_of, + device::{self, Bound}, + driver, + error::{from_result, to_result, Result}, + io::{mem::IoRequest, Resource}, of, prelude::*, - str::CStr, - types::{ForeignOwnable, Opaque}, + types::Opaque, ThisModule, }; @@ -37,12 +39,18 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { None => core::ptr::null(), }; + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + // SAFETY: It's safe to set the fields of `struct platform_driver` on initialization. unsafe { (*pdrv.get()).driver.name = name.as_char_ptr(); (*pdrv.get()).probe = Some(Self::probe_callback); (*pdrv.get()).remove = Some(Self::remove_callback); (*pdrv.get()).driver.of_match_table = of_table; + (*pdrv.get()).driver.acpi_match_table = acpi_table; } // SAFETY: `pdrv` is guaranteed to be a valid `RegType`. @@ -61,30 +69,30 @@ impl<T: Driver + 'static> Adapter<T> { // `struct platform_device`. // // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. - let pdev = unsafe { &*pdev.cast::<Device<device::Core>>() }; - + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; let info = <Self as driver::Adapter>::id_info(pdev.as_ref()); - match T::probe(pdev, info) { - Ok(data) => { - // Let the `struct platform_device` own a reference of the driver's private data. - // SAFETY: By the type invariant `pdev.as_raw` returns a valid pointer to a - // `struct platform_device`. - unsafe { bindings::platform_set_drvdata(pdev.as_raw(), data.into_foreign() as _) }; - } - Err(err) => return Error::to_errno(err), - } - 0 + from_result(|| { + let data = T::probe(pdev, info)?; + + pdev.as_ref().set_drvdata(data); + Ok(0) + }) } extern "C" fn remove_callback(pdev: *mut bindings::platform_device) { - // SAFETY: `pdev` is a valid pointer to a `struct platform_device`. - let ptr = unsafe { bindings::platform_get_drvdata(pdev) }.cast(); + // SAFETY: The platform bus only ever calls the remove callback with a valid pointer to a + // `struct platform_device`. + // + // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to - // `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized - // `KBox<T>` pointer created through `KBox::into_foreign`. - let _ = unsafe { KBox::<T>::from_foreign(ptr) }; + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<T>>`. + let data = unsafe { pdev.as_ref().drvdata_obtain::<Pin<KBox<T>>>() }; + + T::unbind(pdev, data.as_ref()); } } @@ -94,6 +102,10 @@ impl<T: Driver + 'static> driver::Adapter for Adapter<T> { fn of_id_table() -> Option<of::IdTable<Self::IdInfo>> { T::OF_ID_TABLE } + + fn acpi_id_table() -> Option<acpi::IdTable<Self::IdInfo>> { + T::ACPI_ID_TABLE + } } /// Declares a kernel module that exposes a single platform driver. @@ -123,7 +135,7 @@ macro_rules! module_platform_driver { /// # Example /// ///``` -/// # use kernel::{bindings, c_str, device::Core, of, platform}; +/// # use kernel::{acpi, bindings, c_str, device::Core, of, platform}; /// /// struct MyDriver; /// @@ -136,9 +148,19 @@ macro_rules! module_platform_driver { /// ] /// ); /// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// MODULE_ACPI_TABLE, +/// <MyDriver as platform::Driver>::IdInfo, +/// [ +/// (acpi::DeviceId::new(c_str!("LNUXBEEF")), ()) +/// ] +/// ); +/// /// impl platform::Driver for MyDriver { /// type IdInfo = (); /// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = Some(&ACPI_TABLE); /// /// fn probe( /// _pdev: &platform::Device<Core>, @@ -158,7 +180,10 @@ pub trait Driver: Send { type IdInfo: 'static; /// The table of OF device ids supported by the driver. - const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>>; + const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = None; /// Platform driver probe. /// @@ -166,6 +191,20 @@ pub trait Driver: Send { /// Implementers should attempt to initialize the device here. fn probe(dev: &Device<device::Core>, id_info: Option<&Self::IdInfo>) -> Result<Pin<KBox<Self>>>; + + /// Platform driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Self::drop`. + fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) { + let _ = (dev, this); + } } /// The platform device representation. @@ -188,6 +227,61 @@ impl<Ctx: device::DeviceContext> Device<Ctx> { fn as_raw(&self) -> *mut bindings::platform_device { self.0.get() } + + /// Returns the resource at `index`, if any. + pub fn resource_by_index(&self, index: u32) -> Option<&Resource> { + // SAFETY: `self.as_raw()` returns a valid pointer to a `struct platform_device`. + let resource = unsafe { + bindings::platform_get_resource(self.as_raw(), bindings::IORESOURCE_MEM, index) + }; + + if resource.is_null() { + return None; + } + + // SAFETY: `resource` is a valid pointer to a `struct resource` as + // returned by `platform_get_resource`. + Some(unsafe { Resource::from_raw(resource) }) + } + + /// Returns the resource with a given `name`, if any. + pub fn resource_by_name(&self, name: &CStr) -> Option<&Resource> { + // SAFETY: `self.as_raw()` returns a valid pointer to a `struct + // platform_device` and `name` points to a valid C string. + let resource = unsafe { + bindings::platform_get_resource_byname( + self.as_raw(), + bindings::IORESOURCE_MEM, + name.as_char_ptr(), + ) + }; + + if resource.is_null() { + return None; + } + + // SAFETY: `resource` is a valid pointer to a `struct resource` as + // returned by `platform_get_resource`. + Some(unsafe { Resource::from_raw(resource) }) + } +} + +impl Device<Bound> { + /// Returns an `IoRequest` for the resource at `index`, if any. + pub fn io_request_by_index(&self, index: u32) -> Option<IoRequest<'_>> { + self.resource_by_index(index) + // SAFETY: `resource` is a valid resource for `&self` during the + // lifetime of the `IoRequest`. + .map(|resource| unsafe { IoRequest::new(self.as_ref(), resource) }) + } + + /// Returns an `IoRequest` for the resource with a given `name`, if any. + pub fn io_request_by_name(&self, name: &CStr) -> Option<IoRequest<'_>> { + self.resource_by_name(name) + // SAFETY: `resource` is a valid resource for `&self` during the + // lifetime of the `IoRequest`. + .map(|resource| unsafe { IoRequest::new(self.as_ref(), resource) }) + } } // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic @@ -195,6 +289,8 @@ impl<Ctx: device::DeviceContext> Device<Ctx> { kernel::impl_device_context_deref!(unsafe { Device }); kernel::impl_device_context_into_aref!(Device); +impl crate::dma::Device for Device<device::Core> {} + // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::types::AlwaysRefCounted for Device { fn inc_ref(&self) { @@ -215,7 +311,7 @@ impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> { let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; // SAFETY: `dev` points to a valid `struct device`. - unsafe { device::Device::as_ref(dev) } + unsafe { device::Device::from_raw(dev) } } } diff --git a/rust/kernel/regulator.rs b/rust/kernel/regulator.rs new file mode 100644 index 000000000000..65f3a125348f --- /dev/null +++ b/rust/kernel/regulator.rs @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Regulator abstractions, providing a standard kernel interface to control +//! voltage and current regulators. +//! +//! The intention is to allow systems to dynamically control regulator power +//! output in order to save power and prolong battery life. This applies to both +//! voltage regulators (where voltage output is controllable) and current sinks +//! (where current limit is controllable). +//! +//! C header: [`include/linux/regulator/consumer.h`](srctree/include/linux/regulator/consumer.h) +//! +//! Regulators are modeled in Rust with a collection of states. Each state may +//! enforce a given invariant, and they may convert between each other where applicable. +//! +//! See [Voltage and current regulator API](https://docs.kernel.org/driver-api/regulator.html) +//! for more information. + +use crate::{ + bindings, + device::Device, + error::{from_err_ptr, to_result, Result}, + prelude::*, +}; + +use core::{marker::PhantomData, mem::ManuallyDrop, ptr::NonNull}; + +mod private { + pub trait Sealed {} + + impl Sealed for super::Enabled {} + impl Sealed for super::Disabled {} + impl Sealed for super::Dynamic {} +} + +/// A trait representing the different states a [`Regulator`] can be in. +pub trait RegulatorState: private::Sealed + 'static { + /// Whether the regulator should be disabled when dropped. + const DISABLE_ON_DROP: bool; +} + +/// A state where the [`Regulator`] is known to be enabled. +/// +/// The `enable` reference count held by this state is decremented when it is +/// dropped. +pub struct Enabled; + +/// A state where this [`Regulator`] handle has not specifically asked for the +/// underlying regulator to be enabled. This means that this reference does not +/// own an `enable` reference count, but the regulator may still be on. +pub struct Disabled; + +/// A state that models the C API. The [`Regulator`] can be either enabled or +/// disabled, and the user is in control of the reference count. This is also +/// the default state. +/// +/// Use [`Regulator::is_enabled`] to check the regulator's current state. +pub struct Dynamic; + +impl RegulatorState for Enabled { + const DISABLE_ON_DROP: bool = true; +} + +impl RegulatorState for Disabled { + const DISABLE_ON_DROP: bool = false; +} + +impl RegulatorState for Dynamic { + const DISABLE_ON_DROP: bool = false; +} + +/// A trait that abstracts the ability to check if a [`Regulator`] is enabled. +pub trait IsEnabled: RegulatorState {} +impl IsEnabled for Disabled {} +impl IsEnabled for Dynamic {} + +/// An error that can occur when trying to convert a [`Regulator`] between states. +pub struct Error<State: RegulatorState> { + /// The error that occurred. + pub error: kernel::error::Error, + + /// The regulator that caused the error, so that the operation may be retried. + pub regulator: Regulator<State>, +} + +/// A `struct regulator` abstraction. +/// +/// # Examples +/// +/// ## Enabling a regulator +/// +/// This example uses [`Regulator<Enabled>`], which is suitable for drivers that +/// enable a regulator at probe time and leave them on until the device is +/// removed or otherwise shutdown. +/// +/// These users can store [`Regulator<Enabled>`] directly in their driver's +/// private data struct. +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::c_str; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Voltage, Regulator, Disabled, Enabled}; +/// fn enable(dev: &Device, min_voltage: Voltage, max_voltage: Voltage) -> Result { +/// // Obtain a reference to a (fictitious) regulator. +/// let regulator: Regulator<Disabled> = Regulator::<Disabled>::get(dev, c_str!("vcc"))?; +/// +/// // The voltage can be set before enabling the regulator if needed, e.g.: +/// regulator.set_voltage(min_voltage, max_voltage)?; +/// +/// // The same applies for `get_voltage()`, i.e.: +/// let voltage: Voltage = regulator.get_voltage()?; +/// +/// // Enables the regulator, consuming the previous value. +/// // +/// // From now on, the regulator is known to be enabled because of the type +/// // `Enabled`. +/// // +/// // If this operation fails, the `Error` will contain the regulator +/// // reference, so that the operation may be retried. +/// let regulator: Regulator<Enabled> = +/// regulator.try_into_enabled().map_err(|error| error.error)?; +/// +/// // The voltage can also be set after enabling the regulator, e.g.: +/// regulator.set_voltage(min_voltage, max_voltage)?; +/// +/// // The same applies for `get_voltage()`, i.e.: +/// let voltage: Voltage = regulator.get_voltage()?; +/// +/// // Dropping an enabled regulator will disable it. The refcount will be +/// // decremented. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// A more concise shortcut is available for enabling a regulator. This is +/// equivalent to `regulator_get_enable()`: +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::c_str; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Voltage, Regulator, Enabled}; +/// fn enable(dev: &Device) -> Result { +/// // Obtain a reference to a (fictitious) regulator and enable it. +/// let regulator: Regulator<Enabled> = Regulator::<Enabled>::get(dev, c_str!("vcc"))?; +/// +/// // Dropping an enabled regulator will disable it. The refcount will be +/// // decremented. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// ## Disabling a regulator +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Regulator, Enabled, Disabled}; +/// fn disable(dev: &Device, regulator: Regulator<Enabled>) -> Result { +/// // We can also disable an enabled regulator without reliquinshing our +/// // refcount: +/// // +/// // If this operation fails, the `Error` will contain the regulator +/// // reference, so that the operation may be retried. +/// let regulator: Regulator<Disabled> = +/// regulator.try_into_disabled().map_err(|error| error.error)?; +/// +/// // The refcount will be decremented when `regulator` is dropped. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// ## Using [`Regulator<Dynamic>`] +/// +/// This example mimics the behavior of the C API, where the user is in +/// control of the enabled reference count. This is useful for drivers that +/// might call enable and disable to manage the `enable` reference count at +/// runtime, perhaps as a result of `open()` and `close()` calls or whatever +/// other driver-specific or subsystem-specific hooks. +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::c_str; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Regulator, Dynamic}; +/// struct PrivateData { +/// regulator: Regulator<Dynamic>, +/// } +/// +/// // A fictictious probe function that obtains a regulator and sets it up. +/// fn probe(dev: &Device) -> Result<PrivateData> { +/// // Obtain a reference to a (fictitious) regulator. +/// let mut regulator = Regulator::<Dynamic>::get(dev, c_str!("vcc"))?; +/// +/// Ok(PrivateData { regulator }) +/// } +/// +/// // A fictictious function that indicates that the device is going to be used. +/// fn open(dev: &Device, data: &mut PrivateData) -> Result { +/// // Increase the `enabled` reference count. +/// data.regulator.enable()?; +/// +/// Ok(()) +/// } +/// +/// fn close(dev: &Device, data: &mut PrivateData) -> Result { +/// // Decrease the `enabled` reference count. +/// data.regulator.disable()?; +/// +/// Ok(()) +/// } +/// +/// fn remove(dev: &Device, data: PrivateData) -> Result { +/// // `PrivateData` is dropped here, which will drop the +/// // `Regulator<Dynamic>` in turn. +/// // +/// // The reference that was obtained by `regulator_get()` will be +/// // released, but it is up to the user to make sure that the number of calls +/// // to `enable()` and `disabled()` are balanced before this point. +/// Ok(()) +/// } +/// ``` +/// +/// # Invariants +/// +/// - `inner` is a non-null wrapper over a pointer to a `struct +/// regulator` obtained from [`regulator_get()`]. +/// +/// [`regulator_get()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_get +pub struct Regulator<State = Dynamic> +where + State: RegulatorState, +{ + inner: NonNull<bindings::regulator>, + _phantom: PhantomData<State>, +} + +impl<T: RegulatorState> Regulator<T> { + /// Sets the voltage for the regulator. + /// + /// This can be used to ensure that the device powers up cleanly. + pub fn set_voltage(&self, min_voltage: Voltage, max_voltage: Voltage) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { + bindings::regulator_set_voltage( + self.inner.as_ptr(), + min_voltage.as_microvolts(), + max_voltage.as_microvolts(), + ) + }) + } + + /// Gets the current voltage of the regulator. + pub fn get_voltage(&self) -> Result<Voltage> { + // SAFETY: Safe as per the type invariants of `Regulator`. + let voltage = unsafe { bindings::regulator_get_voltage(self.inner.as_ptr()) }; + if voltage < 0 { + Err(kernel::error::Error::from_errno(voltage)) + } else { + Ok(Voltage::from_microvolts(voltage)) + } + } + + fn get_internal(dev: &Device, name: &CStr) -> Result<Regulator<T>> { + // SAFETY: It is safe to call `regulator_get()`, on a device pointer + // received from the C code. + let inner = from_err_ptr(unsafe { bindings::regulator_get(dev.as_raw(), name.as_ptr()) })?; + + // SAFETY: We can safely trust `inner` to be a pointer to a valid + // regulator if `ERR_PTR` was not returned. + let inner = unsafe { NonNull::new_unchecked(inner) }; + + Ok(Self { + inner, + _phantom: PhantomData, + }) + } + + fn enable_internal(&mut self) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { bindings::regulator_enable(self.inner.as_ptr()) }) + } + + fn disable_internal(&mut self) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { bindings::regulator_disable(self.inner.as_ptr()) }) + } +} + +impl Regulator<Disabled> { + /// Obtains a [`Regulator`] instance from the system. + pub fn get(dev: &Device, name: &CStr) -> Result<Self> { + Regulator::get_internal(dev, name) + } + + /// Attempts to convert the regulator to an enabled state. + pub fn try_into_enabled(self) -> Result<Regulator<Enabled>, Error<Disabled>> { + // We will be transferring the ownership of our `regulator_get()` count to + // `Regulator<Enabled>`. + let mut regulator = ManuallyDrop::new(self); + + regulator + .enable_internal() + .map(|()| Regulator { + inner: regulator.inner, + _phantom: PhantomData, + }) + .map_err(|error| Error { + error, + regulator: ManuallyDrop::into_inner(regulator), + }) + } +} + +impl Regulator<Enabled> { + /// Obtains a [`Regulator`] instance from the system and enables it. + /// + /// This is equivalent to calling `regulator_get_enable()` in the C API. + pub fn get(dev: &Device, name: &CStr) -> Result<Self> { + Regulator::<Disabled>::get_internal(dev, name)? + .try_into_enabled() + .map_err(|error| error.error) + } + + /// Attempts to convert the regulator to a disabled state. + pub fn try_into_disabled(self) -> Result<Regulator<Disabled>, Error<Enabled>> { + // We will be transferring the ownership of our `regulator_get()` count + // to `Regulator<Disabled>`. + let mut regulator = ManuallyDrop::new(self); + + regulator + .disable_internal() + .map(|()| Regulator { + inner: regulator.inner, + _phantom: PhantomData, + }) + .map_err(|error| Error { + error, + regulator: ManuallyDrop::into_inner(regulator), + }) + } +} + +impl Regulator<Dynamic> { + /// Obtains a [`Regulator`] instance from the system. The current state of + /// the regulator is unknown and it is up to the user to manage the enabled + /// reference count. + /// + /// This closely mimics the behavior of the C API and can be used to + /// dynamically manage the enabled reference count at runtime. + pub fn get(dev: &Device, name: &CStr) -> Result<Self> { + Regulator::get_internal(dev, name) + } + + /// Increases the `enabled` reference count. + pub fn enable(&mut self) -> Result { + self.enable_internal() + } + + /// Decreases the `enabled` reference count. + pub fn disable(&mut self) -> Result { + self.disable_internal() + } +} + +impl<T: IsEnabled> Regulator<T> { + /// Checks if the regulator is enabled. + pub fn is_enabled(&self) -> bool { + // SAFETY: Safe as per the type invariants of `Regulator`. + unsafe { bindings::regulator_is_enabled(self.inner.as_ptr()) != 0 } + } +} + +impl<T: RegulatorState> Drop for Regulator<T> { + fn drop(&mut self) { + if T::DISABLE_ON_DROP { + // SAFETY: By the type invariants, we know that `self` owns a + // reference on the enabled refcount, so it is safe to relinquish it + // now. + unsafe { bindings::regulator_disable(self.inner.as_ptr()) }; + } + // SAFETY: By the type invariants, we know that `self` owns a reference, + // so it is safe to relinquish it now. + unsafe { bindings::regulator_put(self.inner.as_ptr()) }; + } +} + +/// A voltage. +/// +/// This type represents a voltage value in microvolts. +#[repr(transparent)] +#[derive(Copy, Clone, PartialEq, Eq)] +pub struct Voltage(i32); + +impl Voltage { + /// Creates a new `Voltage` from a value in microvolts. + pub fn from_microvolts(uv: i32) -> Self { + Self(uv) + } + + /// Returns the value of the voltage in microvolts as an [`i32`]. + pub fn as_microvolts(self) -> i32 { + self.0 + } +} diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index 06a3cdfce344..46768b374656 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -5,6 +5,8 @@ //! The [`Revocable`] type wraps other types and allows access to them to be revoked. The existence //! of a [`RevocableGuard`] ensures that objects remain valid. +use pin_init::Wrapper; + use crate::{bindings, prelude::*, sync::rcu, types::Opaque}; use core::{ marker::PhantomData, @@ -80,11 +82,11 @@ unsafe impl<T: Sync + Send> Sync for Revocable<T> {} impl<T> Revocable<T> { /// Creates a new revocable instance of the given data. - pub fn new(data: impl PinInit<T>) -> impl PinInit<Self> { - pin_init!(Self { + pub fn new<E>(data: impl PinInit<T, E>) -> impl PinInit<Self, E> { + try_pin_init!(Self { is_available: AtomicBool::new(true), data <- Opaque::pin_init(data), - }) + }? E) } /// Tries to access the revocable wrapped object. diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index caebf03f553b..c6ec64295c9f 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -216,6 +216,7 @@ impl CondVar { /// This method behaves like `notify_one`, except that it hints to the scheduler that the /// current thread is about to go to sleep, so it should schedule the target thread on the same /// CPU. + #[inline] pub fn notify_sync(&self) { // SAFETY: `wait_queue_head` points to valid memory. unsafe { bindings::__wake_up_sync(self.wait_queue_head.get(), TASK_NORMAL) }; @@ -225,6 +226,7 @@ impl CondVar { /// /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost /// completely (as opposed to automatically waking up the next waiter). + #[inline] pub fn notify_one(&self) { self.notify(1); } @@ -233,6 +235,7 @@ impl CondVar { /// /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost /// completely (as opposed to automatically waking up the next waiter). + #[inline] pub fn notify_all(&self) { self.notify(0); } diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index e82fa5be289c..27202beef90c 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -175,6 +175,8 @@ impl<T: ?Sized, B: Backend> Lock<T, B> { /// Tries to acquire the lock. /// /// Returns a guard that can be used to access the data protected by the lock if successful. + // `Option<T>` is not `#[must_use]` even if `T` is, thus the attribute is needed here. + #[must_use = "if unused, the lock will be immediately unlocked"] pub fn try_lock(&self) -> Option<Guard<'_, T, B>> { // SAFETY: The constructor of the type calls `init`, so the existence of the object proves // that `init` was called. diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index d7e6e59e124b..0ec985d560c8 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -9,9 +9,8 @@ use crate::{ fs::File, prelude::*, sync::{CondVar, LockClassKey}, - types::Opaque, }; -use core::ops::Deref; +use core::{marker::PhantomData, ops::Deref}; /// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -23,58 +22,43 @@ macro_rules! new_poll_condvar { }; } -/// Wraps the kernel's `struct poll_table`. +/// Wraps the kernel's `poll_table`. /// /// # Invariants /// -/// This struct contains a valid `struct poll_table`. -/// -/// For a `struct poll_table` to be valid, its `_qproc` function must follow the safety -/// requirements of `_qproc` functions: -/// -/// * The `_qproc` function is given permission to enqueue a waiter to the provided `poll_table` -/// during the call. Once the waiter is removed and an rcu grace period has passed, it must no -/// longer access the `wait_queue_head`. +/// The pointer must be null or reference a valid `poll_table`. #[repr(transparent)] -pub struct PollTable(Opaque<bindings::poll_table>); +pub struct PollTable<'a> { + table: *mut bindings::poll_table, + _lifetime: PhantomData<&'a bindings::poll_table>, +} -impl PollTable { - /// Creates a reference to a [`PollTable`] from a valid pointer. +impl<'a> PollTable<'a> { + /// Creates a [`PollTable`] from a valid pointer. /// /// # Safety /// - /// The caller must ensure that for the duration of `'a`, the pointer will point at a valid poll - /// table (as defined in the type invariants). - /// - /// The caller must also ensure that the `poll_table` is only accessed via the returned - /// reference for the duration of `'a`. - pub unsafe fn from_ptr<'a>(ptr: *mut bindings::poll_table) -> &'a mut PollTable { - // SAFETY: The safety requirements guarantee the validity of the dereference, while the - // `PollTable` type being transparent makes the cast ok. - unsafe { &mut *ptr.cast() } - } - - fn get_qproc(&self) -> bindings::poll_queue_proc { - let ptr = self.0.get(); - // SAFETY: The `ptr` is valid because it originates from a reference, and the `_qproc` - // field is not modified concurrently with this call since we have an immutable reference. - unsafe { (*ptr)._qproc } + /// The pointer must be null or reference a valid `poll_table` for the duration of `'a`. + pub unsafe fn from_raw(table: *mut bindings::poll_table) -> Self { + // INVARIANTS: The safety requirements are the same as the struct invariants. + PollTable { + table, + _lifetime: PhantomData, + } } /// Register this [`PollTable`] with the provided [`PollCondVar`], so that it can be notified /// using the condition variable. - pub fn register_wait(&mut self, file: &File, cv: &PollCondVar) { - if let Some(qproc) = self.get_qproc() { - // SAFETY: The pointers to `file` and `self` need to be valid for the duration of this - // call to `qproc`, which they are because they are references. - // - // The `cv.wait_queue_head` pointer must be valid until an rcu grace period after the - // waiter is removed. The `PollCondVar` is pinned, so before `cv.wait_queue_head` can - // be destroyed, the destructor must run. That destructor first removes all waiters, - // and then waits for an rcu grace period. Therefore, `cv.wait_queue_head` is valid for - // long enough. - unsafe { qproc(file.as_ptr() as _, cv.wait_queue_head.get(), self.0.get()) }; - } + pub fn register_wait(&self, file: &File, cv: &PollCondVar) { + // SAFETY: + // * `file.as_ptr()` references a valid file for the duration of this call. + // * `self.table` is null or references a valid poll_table for the duration of this call. + // * Since `PollCondVar` is pinned, its destructor is guaranteed to run before the memory + // containing `cv.wait_queue_head` is invalidated. Since the destructor clears all + // waiters and then waits for an rcu grace period, it's guaranteed that + // `cv.wait_queue_head` remains valid for at least an rcu grace period after the removal + // of the last waiter. + unsafe { bindings::poll_wait(file.as_ptr(), cv.wait_queue_head.get(), self.table) } } } @@ -107,6 +91,7 @@ impl Deref for PollCondVar { #[pinned_drop] impl PinnedDrop for PollCondVar { + #[inline] fn drop(self: Pin<&mut Self>) { // Clear anything registered using `register_wait`. // diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 927413d85484..7d0935bc325c 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -173,6 +173,7 @@ impl Task { /// Callers must ensure that the returned object is only used to access a [`CurrentTask`] /// within the task context that was active when this function was called. For more details, /// see the invariants section for [`CurrentTask`]. + #[inline] pub unsafe fn current() -> impl Deref<Target = CurrentTask> { struct TaskRef { task: *const CurrentTask, @@ -222,24 +223,28 @@ impl Task { } /// Returns the UID of the given task. + #[inline] pub fn uid(&self) -> Kuid { // SAFETY: It's always safe to call `task_uid` on a valid task. Kuid::from_raw(unsafe { bindings::task_uid(self.as_ptr()) }) } /// Returns the effective UID of the given task. + #[inline] pub fn euid(&self) -> Kuid { // SAFETY: It's always safe to call `task_euid` on a valid task. Kuid::from_raw(unsafe { bindings::task_euid(self.as_ptr()) }) } /// Determines whether the given task has pending signals. + #[inline] pub fn signal_pending(&self) -> bool { // SAFETY: It's always safe to call `signal_pending` on a valid task. unsafe { bindings::signal_pending(self.as_ptr()) != 0 } } /// Returns task's pid namespace with elevated reference count + #[inline] pub fn get_pid_ns(&self) -> Option<ARef<PidNamespace>> { // SAFETY: By the type invariant, we know that `self.0` is valid. let ptr = unsafe { bindings::task_get_pid_ns(self.as_ptr()) }; @@ -255,6 +260,7 @@ impl Task { /// Returns the given task's pid in the provided pid namespace. #[doc(alias = "task_tgid_nr_ns")] + #[inline] pub fn tgid_nr_ns(&self, pidns: Option<&PidNamespace>) -> Pid { let pidns = match pidns { Some(pidns) => pidns.as_ptr(), @@ -268,6 +274,7 @@ impl Task { } /// Wakes up the task. + #[inline] pub fn wake_up(&self) { // SAFETY: It's always safe to call `wake_up_process` on a valid task, even if the task // running. @@ -341,11 +348,13 @@ impl CurrentTask { // SAFETY: The type invariants guarantee that `Task` is always refcounted. unsafe impl crate::types::AlwaysRefCounted for Task { + #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference means that the refcount is nonzero. unsafe { bindings::get_task_struct(self.as_ptr()) }; } + #[inline] unsafe fn dec_ref(obj: ptr::NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is nonzero. unsafe { bindings::put_task_struct(obj.cast().as_ptr()) } @@ -391,3 +400,27 @@ impl PartialEq for Kuid { } impl Eq for Kuid {} + +/// Annotation for functions that can sleep. +/// +/// Equivalent to the C side [`might_sleep()`], this function serves as +/// a debugging aid and a potential scheduling point. +/// +/// This function can only be used in a nonatomic context. +/// +/// [`might_sleep()`]: https://docs.kernel.org/driver-api/basics.html#c.might_sleep +#[track_caller] +#[inline] +pub fn might_sleep() { + #[cfg(CONFIG_DEBUG_ATOMIC_SLEEP)] + { + let loc = core::panic::Location::caller(); + let file = kernel::file_from_location(loc); + + // SAFETY: `file.as_ptr()` is valid for reading and guaranteed to be nul-terminated. + unsafe { crate::bindings::__might_sleep(file.as_ptr().cast(), loc.line() as i32) } + } + + // SAFETY: Always safe to call. + unsafe { crate::bindings::might_resched() } +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 22985b6f6982..3958a5f44d56 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -9,7 +9,7 @@ use core::{ ops::{Deref, DerefMut}, ptr::NonNull, }; -use pin_init::{PinInit, Zeroable}; +use pin_init::{PinInit, Wrapper, Zeroable}; /// Used to transfer ownership to and from foreign (non-Rust) languages. /// @@ -353,17 +353,6 @@ impl<T> Opaque<T> { } } - /// Create an opaque pin-initializer from the given pin-initializer. - pub fn pin_init(slot: impl PinInit<T>) -> impl PinInit<Self> { - Self::ffi_init(|ptr: *mut T| { - // SAFETY: - // - `ptr` is a valid pointer to uninitialized memory, - // - `slot` is not accessed on error; the call is infallible, - // - `slot` is pinned in memory. - let _ = unsafe { PinInit::<T>::__pinned_init(slot, ptr) }; - }) - } - /// Creates a pin-initializer from the given initializer closure. /// /// The returned initializer calls the given closure with the pointer to the inner `T` of this @@ -415,6 +404,19 @@ impl<T> Opaque<T> { } } +impl<T> Wrapper<T> for Opaque<T> { + /// Create an opaque pin-initializer from the given pin-initializer. + fn pin_init<E>(slot: impl PinInit<T, E>) -> impl PinInit<Self, E> { + Self::try_ffi_init(|ptr: *mut T| { + // SAFETY: + // - `ptr` is a valid pointer to uninitialized memory, + // - `slot` is not accessed on error, + // - `slot` is pinned in memory. + unsafe { PinInit::<T, E>::__pinned_init(slot, ptr) } + }) + } +} + /// Types that are _always_ reference counted. /// /// It allows such types to define their own custom ref increment and decrement functions. |