diff options
Diffstat (limited to 'rust/kernel')
34 files changed, 2678 insertions, 392 deletions
diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index 7a3b0b9c418e..56f3c180e8f6 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -7,6 +7,7 @@ use crate::{ bindings, container_of, device, device_id::{RawDeviceId, RawDeviceIdIndex}, + devres::Devres, driver, error::{from_result, to_result, Result}, prelude::*, @@ -15,6 +16,7 @@ use crate::{ }; use core::{ marker::PhantomData, + mem::offset_of, ptr::{addr_of_mut, NonNull}, }; @@ -68,9 +70,9 @@ impl<T: Driver + 'static> Adapter<T> { let info = T::ID_TABLE.info(id.index()); from_result(|| { - let data = T::probe(adev, info)?; + let data = T::probe(adev, info); - adev.as_ref().set_drvdata(data); + adev.as_ref().set_drvdata(data)?; Ok(0) }) } @@ -85,7 +87,7 @@ impl<T: Driver + 'static> Adapter<T> { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called // and stored a `Pin<KBox<T>>`. - drop(unsafe { adev.as_ref().drvdata_obtain::<Pin<KBox<T>>>() }); + drop(unsafe { adev.as_ref().drvdata_obtain::<T>() }); } } @@ -184,7 +186,7 @@ pub trait Driver { /// Auxiliary driver probe. /// /// Called when an auxiliary device is matches a corresponding driver. - fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>; + fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> impl PinInit<Self, Error>; } /// The auxiliary device representation. @@ -214,14 +216,25 @@ impl<Ctx: device::DeviceContext> Device<Ctx> { // `struct auxiliary_device`. unsafe { (*self.as_raw()).id } } +} + +impl Device<device::Bound> { + /// Returns a bound reference to the parent [`device::Device`]. + pub fn parent(&self) -> &device::Device<device::Bound> { + let parent = (**self).parent(); - /// Returns a reference to the parent [`device::Device`], if any. - pub fn parent(&self) -> Option<&device::Device> { - self.as_ref().parent() + // SAFETY: A bound auxiliary device always has a bound parent device. + unsafe { parent.as_bound() } } } impl Device { + /// Returns a reference to the parent [`device::Device`]. + pub fn parent(&self) -> &device::Device { + // SAFETY: A `struct auxiliary_device` always has a parent. + unsafe { self.as_ref().parent().unwrap_unchecked() } + } + extern "C" fn release(dev: *mut bindings::device) { // SAFETY: By the type invariant `self.0.as_raw` is a pointer to the `struct device` // embedded in `struct auxiliary_device`. @@ -233,6 +246,12 @@ impl Device { } } +// SAFETY: `auxiliary::Device` is a transparent wrapper of `struct auxiliary_device`. +// The offset is guaranteed to point to a valid device field inside `auxiliary::Device`. +unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Device<Ctx> { + const OFFSET: usize = offset_of!(bindings::auxiliary_device, dev); +} + // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic // argument. kernel::impl_device_context_deref!(unsafe { Device }); @@ -278,8 +297,8 @@ unsafe impl Sync for Device {} /// The registration of an auxiliary device. /// -/// This type represents the registration of a [`struct auxiliary_device`]. When an instance of this -/// type is dropped, its respective auxiliary device will be unregistered from the system. +/// This type represents the registration of a [`struct auxiliary_device`]. When its parent device +/// is unbound, the corresponding auxiliary device will be unregistered from the system. /// /// # Invariants /// @@ -289,44 +308,55 @@ pub struct Registration(NonNull<bindings::auxiliary_device>); impl Registration { /// Create and register a new auxiliary device. - pub fn new(parent: &device::Device, name: &CStr, id: u32, modname: &CStr) -> Result<Self> { - let boxed = KBox::new(Opaque::<bindings::auxiliary_device>::zeroed(), GFP_KERNEL)?; - let adev = boxed.get(); - - // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization. - unsafe { - (*adev).dev.parent = parent.as_raw(); - (*adev).dev.release = Some(Device::release); - (*adev).name = name.as_char_ptr(); - (*adev).id = id; - } - - // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, - // which has not been initialized yet. - unsafe { bindings::auxiliary_device_init(adev) }; + pub fn new<'a>( + parent: &'a device::Device<device::Bound>, + name: &'a CStr, + id: u32, + modname: &'a CStr, + ) -> impl PinInit<Devres<Self>, Error> + 'a { + pin_init::pin_init_scope(move || { + let boxed = KBox::new(Opaque::<bindings::auxiliary_device>::zeroed(), GFP_KERNEL)?; + let adev = boxed.get(); + + // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization. + unsafe { + (*adev).dev.parent = parent.as_raw(); + (*adev).dev.release = Some(Device::release); + (*adev).name = name.as_char_ptr(); + (*adev).id = id; + } - // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be freed - // by `Device::release` when the last reference to the `struct auxiliary_device` is dropped. - let _ = KBox::into_raw(boxed); - - // SAFETY: - // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which has - // been initialialized, - // - `modname.as_char_ptr()` is a NULL terminated string. - let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) }; - if ret != 0 { // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, - // which has been initialialized. - unsafe { bindings::auxiliary_device_uninit(adev) }; - - return Err(Error::from_errno(ret)); - } - - // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated successfully. - // - // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is called, - // which happens in `Self::drop()`. - Ok(Self(unsafe { NonNull::new_unchecked(adev) })) + // which has not been initialized yet. + unsafe { bindings::auxiliary_device_init(adev) }; + + // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be + // freed by `Device::release` when the last reference to the `struct auxiliary_device` + // is dropped. + let _ = KBox::into_raw(boxed); + + // SAFETY: + // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which + // has been initialized, + // - `modname.as_char_ptr()` is a NULL terminated string. + let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) }; + if ret != 0 { + // SAFETY: `adev` is guaranteed to be a valid pointer to a + // `struct auxiliary_device`, which has been initialized. + unsafe { bindings::auxiliary_device_uninit(adev) }; + + return Err(Error::from_errno(ret)); + } + + // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is + // called, which happens in `Self::drop()`. + Ok(Devres::new( + parent, + // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated + // successfully. + Self(unsafe { NonNull::new_unchecked(adev) }), + )) + }) } } diff --git a/rust/kernel/bitmap.rs b/rust/kernel/bitmap.rs index aa8fc7bf06fc..83d7dea99137 100644 --- a/rust/kernel/bitmap.rs +++ b/rust/kernel/bitmap.rs @@ -12,8 +12,6 @@ use crate::bindings; use crate::pr_err; use core::ptr::NonNull; -const BITS_PER_LONG: usize = bindings::BITS_PER_LONG as usize; - /// Represents a C bitmap. Wraps underlying C bitmap API. /// /// # Invariants @@ -149,14 +147,14 @@ macro_rules! bitmap_assert_return { /// /// # Invariants /// -/// * `nbits` is `<= i32::MAX` and never changes. -/// * if `nbits <= bindings::BITS_PER_LONG`, then `repr` is a `usize`. +/// * `nbits` is `<= MAX_LEN`. +/// * if `nbits <= MAX_INLINE_LEN`, then `repr` is a `usize`. /// * otherwise, `repr` holds a non-null pointer to an initialized /// array of `unsigned long` that is large enough to hold `nbits` bits. pub struct BitmapVec { /// Representation of bitmap. repr: BitmapRepr, - /// Length of this bitmap. Must be `<= i32::MAX`. + /// Length of this bitmap. Must be `<= MAX_LEN`. nbits: usize, } @@ -164,7 +162,7 @@ impl core::ops::Deref for BitmapVec { type Target = Bitmap; fn deref(&self) -> &Bitmap { - let ptr = if self.nbits <= BITS_PER_LONG { + let ptr = if self.nbits <= BitmapVec::MAX_INLINE_LEN { // SAFETY: Bitmap is represented inline. #[allow(unused_unsafe, reason = "Safe since Rust 1.92.0")] unsafe { @@ -183,7 +181,7 @@ impl core::ops::Deref for BitmapVec { impl core::ops::DerefMut for BitmapVec { fn deref_mut(&mut self) -> &mut Bitmap { - let ptr = if self.nbits <= BITS_PER_LONG { + let ptr = if self.nbits <= BitmapVec::MAX_INLINE_LEN { // SAFETY: Bitmap is represented inline. #[allow(unused_unsafe, reason = "Safe since Rust 1.92.0")] unsafe { @@ -213,7 +211,7 @@ unsafe impl Sync for BitmapVec {} impl Drop for BitmapVec { fn drop(&mut self) { - if self.nbits <= BITS_PER_LONG { + if self.nbits <= BitmapVec::MAX_INLINE_LEN { return; } // SAFETY: `self.ptr` was returned by the C `bitmap_zalloc`. @@ -226,23 +224,39 @@ impl Drop for BitmapVec { } impl BitmapVec { + /// The maximum possible length of a `BitmapVec`. + pub const MAX_LEN: usize = i32::MAX as usize; + + /// The maximum length that uses the inline representation. + pub const MAX_INLINE_LEN: usize = usize::BITS as usize; + + /// Construct a longest possible inline [`BitmapVec`]. + #[inline] + pub fn new_inline() -> Self { + // INVARIANT: `nbits <= MAX_INLINE_LEN`, so an inline bitmap is the right repr. + BitmapVec { + repr: BitmapRepr { bitmap: 0 }, + nbits: BitmapVec::MAX_INLINE_LEN, + } + } + /// Constructs a new [`BitmapVec`]. /// /// Fails with [`AllocError`] when the [`BitmapVec`] could not be allocated. This - /// includes the case when `nbits` is greater than `i32::MAX`. + /// includes the case when `nbits` is greater than `MAX_LEN`. #[inline] pub fn new(nbits: usize, flags: Flags) -> Result<Self, AllocError> { - if nbits <= BITS_PER_LONG { + if nbits <= BitmapVec::MAX_INLINE_LEN { return Ok(BitmapVec { repr: BitmapRepr { bitmap: 0 }, nbits, }); } - if nbits > i32::MAX.try_into().unwrap() { + if nbits > Self::MAX_LEN { return Err(AllocError); } let nbits_u32 = u32::try_from(nbits).unwrap(); - // SAFETY: `BITS_PER_LONG < nbits` and `nbits <= i32::MAX`. + // SAFETY: `MAX_INLINE_LEN < nbits` and `nbits <= MAX_LEN`. let ptr = unsafe { bindings::bitmap_zalloc(nbits_u32, flags.as_raw()) }; let ptr = NonNull::new(ptr).ok_or(AllocError)?; // INVARIANT: `ptr` returned by C `bitmap_zalloc` and `nbits` checked. @@ -495,9 +509,10 @@ mod tests { #[test] fn bitmap_borrow() { let fake_bitmap: [usize; 2] = [0, 0]; + let fake_bitmap_len = 2 * usize::BITS as usize; // SAFETY: `fake_c_bitmap` is an array of expected length. - let b = unsafe { Bitmap::from_raw(fake_bitmap.as_ptr(), 2 * BITS_PER_LONG) }; - assert_eq!(2 * BITS_PER_LONG, b.len()); + let b = unsafe { Bitmap::from_raw(fake_bitmap.as_ptr(), fake_bitmap_len) }; + assert_eq!(fake_bitmap_len, b.len()); assert_eq!(None, b.next_bit(0)); } diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 1a555fcb120a..f968fbd22890 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -893,9 +893,9 @@ pub trait Driver { /// fn probe( /// pdev: &platform::Device<Core>, /// _id_info: Option<&Self::IdInfo>, -/// ) -> Result<Pin<KBox<Self>>> { +/// ) -> impl PinInit<Self, Error> { /// cpufreq::Registration::<SampleDriver>::new_foreign_owned(pdev.as_ref())?; -/// Ok(KBox::new(Self {}, GFP_KERNEL)?.into()) +/// Ok(Self {}) /// } /// } /// ``` diff --git a/rust/kernel/debugfs.rs b/rust/kernel/debugfs.rs index 8c35d032acfe..facad81e8290 100644 --- a/rust/kernel/debugfs.rs +++ b/rust/kernel/debugfs.rs @@ -21,12 +21,15 @@ use core::mem::ManuallyDrop; use core::ops::Deref; mod traits; -pub use traits::{Reader, Writer}; +pub use traits::{BinaryReader, BinaryReaderMut, BinaryWriter, Reader, Writer}; mod callback_adapters; use callback_adapters::{FormatAdapter, NoWriter, WritableAdapter}; mod file_ops; -use file_ops::{FileOps, ReadFile, ReadWriteFile, WriteFile}; +use file_ops::{ + BinaryReadFile, BinaryReadWriteFile, BinaryWriteFile, FileOps, ReadFile, ReadWriteFile, + WriteFile, +}; #[cfg(CONFIG_DEBUG_FS)] mod entry; #[cfg(CONFIG_DEBUG_FS)] @@ -150,6 +153,32 @@ impl Dir { self.create_file(name, data, file_ops) } + /// Creates a read-only binary file in this directory. + /// + /// The file's contents are produced by invoking [`BinaryWriter::write_to_slice`] on the value + /// initialized by `data`. + /// + /// # Examples + /// + /// ``` + /// # use kernel::c_str; + /// # use kernel::debugfs::Dir; + /// # use kernel::prelude::*; + /// # let dir = Dir::new(c_str!("my_debugfs_dir")); + /// let file = KBox::pin_init(dir.read_binary_file(c_str!("foo"), [0x1, 0x2]), GFP_KERNEL)?; + /// # Ok::<(), Error>(()) + /// ``` + pub fn read_binary_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit<T, E> + 'a, + ) -> impl PinInit<File<T>, E> + 'a + where + T: BinaryWriter + Send + Sync + 'static, + { + self.create_file(name, data, &T::FILE_OPS) + } + /// Creates a read-only file in this directory, with contents from a callback. /// /// `f` must be a function item or a non-capturing closure. @@ -206,6 +235,22 @@ impl Dir { self.create_file(name, data, file_ops) } + /// Creates a read-write binary file in this directory. + /// + /// Reading the file uses the [`BinaryWriter`] implementation. + /// Writing to the file uses the [`BinaryReader`] implementation. + pub fn read_write_binary_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit<T, E> + 'a, + ) -> impl PinInit<File<T>, E> + 'a + where + T: BinaryWriter + BinaryReader + Send + Sync + 'static, + { + let file_ops = &<T as BinaryReadWriteFile<_>>::FILE_OPS; + self.create_file(name, data, file_ops) + } + /// Creates a read-write file in this directory, with logic from callbacks. /// /// Reading from the file is handled by `f`. Writing to the file is handled by `w`. @@ -248,6 +293,23 @@ impl Dir { self.create_file(name, data, &T::FILE_OPS) } + /// Creates a write-only binary file in this directory. + /// + /// The file owns its backing data. Writing to the file uses the [`BinaryReader`] + /// implementation. + /// + /// The file is removed when the returned [`File`] is dropped. + pub fn write_binary_file<'a, T, E: 'a>( + &'a self, + name: &'a CStr, + data: impl PinInit<T, E> + 'a, + ) -> impl PinInit<File<T>, E> + 'a + where + T: BinaryReader + Send + Sync + 'static, + { + self.create_file(name, data, &T::FILE_OPS) + } + /// Creates a write-only file in this directory, with write logic from a callback. /// /// `w` must be a function item or a non-capturing closure. @@ -468,6 +530,20 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { self.create_file(name, data, &T::FILE_OPS) } + /// Creates a read-only binary file in this directory. + /// + /// The file's contents are produced by invoking [`BinaryWriter::write_to_slice`]. + /// + /// This function does not produce an owning handle to the file. The created file is removed + /// when the [`Scope`] that this directory belongs to is dropped. + pub fn read_binary_file<T: BinaryWriter + Send + Sync + 'static>( + &self, + name: &CStr, + data: &'data T, + ) { + self.create_file(name, data, &T::FILE_OPS) + } + /// Creates a read-only file in this directory, with contents from a callback. /// /// The file contents are generated by calling `f` with `data`. @@ -505,6 +581,22 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { self.create_file(name, data, vtable) } + /// Creates a read-write binary file in this directory. + /// + /// Reading the file uses the [`BinaryWriter`] implementation on `data`. Writing to the file + /// uses the [`BinaryReader`] implementation on `data`. + /// + /// This function does not produce an owning handle to the file. The created file is removed + /// when the [`Scope`] that this directory belongs to is dropped. + pub fn read_write_binary_file<T: BinaryWriter + BinaryReader + Send + Sync + 'static>( + &self, + name: &CStr, + data: &'data T, + ) { + let vtable = &<T as BinaryReadWriteFile<_>>::FILE_OPS; + self.create_file(name, data, vtable) + } + /// Creates a read-write file in this directory, with logic from callbacks. /// /// Reading from the file is handled by `f`. Writing to the file is handled by `w`. @@ -544,6 +636,20 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { self.create_file(name, data, vtable) } + /// Creates a write-only binary file in this directory. + /// + /// Writing to the file uses the [`BinaryReader`] implementation on `data`. + /// + /// This function does not produce an owning handle to the file. The created file is removed + /// when the [`Scope`] that this directory belongs to is dropped. + pub fn write_binary_file<T: BinaryReader + Send + Sync + 'static>( + &self, + name: &CStr, + data: &'data T, + ) { + self.create_file(name, data, &T::FILE_OPS) + } + /// Creates a write-only file in this directory, with write logic from a callback. /// /// Writing to the file is handled by `w`. diff --git a/rust/kernel/debugfs/file_ops.rs b/rust/kernel/debugfs/file_ops.rs index 9ad5e3fa6f69..8a0442d6dd7a 100644 --- a/rust/kernel/debugfs/file_ops.rs +++ b/rust/kernel/debugfs/file_ops.rs @@ -1,9 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2025 Google LLC. -use super::{Reader, Writer}; +use super::{BinaryReader, BinaryWriter, Reader, Writer}; use crate::debugfs::callback_adapters::Adapter; use crate::fmt; +use crate::fs::file; use crate::prelude::*; use crate::seq_file::SeqFile; use crate::seq_print; @@ -245,3 +246,140 @@ impl<T: Reader + Sync> WriteFile<T> for T { unsafe { FileOps::new(operations, 0o200) } }; } + +extern "C" fn blob_read<T: BinaryWriter>( + file: *mut bindings::file, + buf: *mut c_char, + count: usize, + ppos: *mut bindings::loff_t, +) -> isize { + // SAFETY: + // - `file` is a valid pointer to a `struct file`. + // - The type invariant of `FileOps` guarantees that `private_data` points to a valid `T`. + let this = unsafe { &*((*file).private_data.cast::<T>()) }; + + // SAFETY: + // - `ppos` is a valid `file::Offset` pointer. + // - We have exclusive access to `ppos`. + let pos: &mut file::Offset = unsafe { &mut *ppos }; + + let mut writer = UserSlice::new(UserPtr::from_ptr(buf.cast()), count).writer(); + + let ret = || -> Result<isize> { + let written = this.write_to_slice(&mut writer, pos)?; + + Ok(written.try_into()?) + }(); + + match ret { + Ok(n) => n, + Err(e) => e.to_errno() as isize, + } +} + +/// Representation of [`FileOps`] for read only binary files. +pub(crate) trait BinaryReadFile<T> { + const FILE_OPS: FileOps<T>; +} + +impl<T: BinaryWriter + Sync> BinaryReadFile<T> for T { + const FILE_OPS: FileOps<T> = { + let operations = bindings::file_operations { + read: Some(blob_read::<T>), + llseek: Some(bindings::default_llseek), + open: Some(bindings::simple_open), + // SAFETY: `file_operations` supports zeroes in all fields. + ..unsafe { core::mem::zeroed() } + }; + + // SAFETY: + // - The private data of `struct inode` does always contain a pointer to a valid `T`. + // - `simple_open()` stores the `struct inode`'s private data in the private data of the + // corresponding `struct file`. + // - `blob_read()` re-creates a reference to `T` from the `struct file`'s private data. + // - `default_llseek()` does not access the `struct file`'s private data. + unsafe { FileOps::new(operations, 0o400) } + }; +} + +extern "C" fn blob_write<T: BinaryReader>( + file: *mut bindings::file, + buf: *const c_char, + count: usize, + ppos: *mut bindings::loff_t, +) -> isize { + // SAFETY: + // - `file` is a valid pointer to a `struct file`. + // - The type invariant of `FileOps` guarantees that `private_data` points to a valid `T`. + let this = unsafe { &*((*file).private_data.cast::<T>()) }; + + // SAFETY: + // - `ppos` is a valid `file::Offset` pointer. + // - We have exclusive access to `ppos`. + let pos: &mut file::Offset = unsafe { &mut *ppos }; + + let mut reader = UserSlice::new(UserPtr::from_ptr(buf.cast_mut().cast()), count).reader(); + + let ret = || -> Result<isize> { + let read = this.read_from_slice(&mut reader, pos)?; + + Ok(read.try_into()?) + }(); + + match ret { + Ok(n) => n, + Err(e) => e.to_errno() as isize, + } +} + +/// Representation of [`FileOps`] for write only binary files. +pub(crate) trait BinaryWriteFile<T> { + const FILE_OPS: FileOps<T>; +} + +impl<T: BinaryReader + Sync> BinaryWriteFile<T> for T { + const FILE_OPS: FileOps<T> = { + let operations = bindings::file_operations { + write: Some(blob_write::<T>), + llseek: Some(bindings::default_llseek), + open: Some(bindings::simple_open), + // SAFETY: `file_operations` supports zeroes in all fields. + ..unsafe { core::mem::zeroed() } + }; + + // SAFETY: + // - The private data of `struct inode` does always contain a pointer to a valid `T`. + // - `simple_open()` stores the `struct inode`'s private data in the private data of the + // corresponding `struct file`. + // - `blob_write()` re-creates a reference to `T` from the `struct file`'s private data. + // - `default_llseek()` does not access the `struct file`'s private data. + unsafe { FileOps::new(operations, 0o200) } + }; +} + +/// Representation of [`FileOps`] for read/write binary files. +pub(crate) trait BinaryReadWriteFile<T> { + const FILE_OPS: FileOps<T>; +} + +impl<T: BinaryWriter + BinaryReader + Sync> BinaryReadWriteFile<T> for T { + const FILE_OPS: FileOps<T> = { + let operations = bindings::file_operations { + read: Some(blob_read::<T>), + write: Some(blob_write::<T>), + llseek: Some(bindings::default_llseek), + open: Some(bindings::simple_open), + // SAFETY: `file_operations` supports zeroes in all fields. + ..unsafe { core::mem::zeroed() } + }; + + // SAFETY: + // - The private data of `struct inode` does always contain a pointer to a valid `T`. + // - `simple_open()` stores the `struct inode`'s private data in the private data of the + // corresponding `struct file`. + // - `blob_read()` re-creates a reference to `T` from the `struct file`'s private data. + // - `blob_write()` re-creates a reference to `T` from the `struct file`'s private data. + // - `default_llseek()` does not access the `struct file`'s private data. + unsafe { FileOps::new(operations, 0o600) } + }; +} diff --git a/rust/kernel/debugfs/traits.rs b/rust/kernel/debugfs/traits.rs index e8a8a98f18dc..3eee60463fd5 100644 --- a/rust/kernel/debugfs/traits.rs +++ b/rust/kernel/debugfs/traits.rs @@ -3,11 +3,16 @@ //! Traits for rendering or updating values exported to DebugFS. +use crate::alloc::Allocator; use crate::fmt; +use crate::fs::file; use crate::prelude::*; use crate::sync::atomic::{Atomic, AtomicBasicOps, AtomicType, Relaxed}; +use crate::sync::Arc; use crate::sync::Mutex; -use crate::uaccess::UserSliceReader; +use crate::transmute::{AsBytes, FromBytes}; +use crate::uaccess::{UserSliceReader, UserSliceWriter}; +use core::ops::{Deref, DerefMut}; use core::str::FromStr; /// A trait for types that can be written into a string. @@ -36,6 +41,110 @@ impl<T: fmt::Debug> Writer for T { } } +/// Trait for types that can be written out as binary. +pub trait BinaryWriter { + /// Writes the binary form of `self` into `writer`. + /// + /// `offset` is the requested offset into the binary representation of `self`. + /// + /// On success, returns the number of bytes written in to `writer`. + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result<usize>; +} + +// Base implementation for any `T: AsBytes`. +impl<T: AsBytes> BinaryWriter for T { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result<usize> { + writer.write_slice_file(self.as_bytes(), offset) + } +} + +// Delegate for `Mutex<T>`: Support a `T` with an outer mutex. +impl<T: BinaryWriter> BinaryWriter for Mutex<T> { + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result<usize> { + let guard = self.lock(); + + guard.write_to_slice(writer, offset) + } +} + +// Delegate for `Box<T, A>`: Support a `Box<T, A>` with no lock or an inner lock. +impl<T, A> BinaryWriter for Box<T, A> +where + T: BinaryWriter, + A: Allocator, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result<usize> { + self.deref().write_to_slice(writer, offset) + } +} + +// Delegate for `Pin<Box<T, A>>`: Support a `Pin<Box<T, A>>` with no lock or an inner lock. +impl<T, A> BinaryWriter for Pin<Box<T, A>> +where + T: BinaryWriter, + A: Allocator, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result<usize> { + self.deref().write_to_slice(writer, offset) + } +} + +// Delegate for `Arc<T>`: Support a `Arc<T>` with no lock or an inner lock. +impl<T> BinaryWriter for Arc<T> +where + T: BinaryWriter, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result<usize> { + self.deref().write_to_slice(writer, offset) + } +} + +// Delegate for `Vec<T, A>`. +impl<T, A> BinaryWriter for Vec<T, A> +where + T: AsBytes, + A: Allocator, +{ + fn write_to_slice( + &self, + writer: &mut UserSliceWriter, + offset: &mut file::Offset, + ) -> Result<usize> { + let slice = self.as_slice(); + + // SAFETY: `T: AsBytes` allows us to treat `&[T]` as `&[u8]`. + let buffer = unsafe { + core::slice::from_raw_parts(slice.as_ptr().cast(), core::mem::size_of_val(slice)) + }; + + writer.write_slice_file(buffer, offset) + } +} + /// A trait for types that can be updated from a user slice. /// /// This works similarly to `FromStr`, but operates on a `UserSliceReader` rather than a &str. @@ -81,3 +190,130 @@ where Ok(()) } } + +/// Trait for types that can be constructed from a binary representation. +/// +/// See also [`BinaryReader`] for interior mutability. +pub trait BinaryReaderMut { + /// Reads the binary form of `self` from `reader`. + /// + /// Same as [`BinaryReader::read_from_slice`], but takes a mutable reference. + /// + /// `offset` is the requested offset into the binary representation of `self`. + /// + /// On success, returns the number of bytes read from `reader`. + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result<usize>; +} + +// Base implementation for any `T: AsBytes + FromBytes`. +impl<T: AsBytes + FromBytes> BinaryReaderMut for T { + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result<usize> { + reader.read_slice_file(self.as_bytes_mut(), offset) + } +} + +// Delegate for `Box<T, A>`: Support a `Box<T, A>` with an outer lock. +impl<T: ?Sized + BinaryReaderMut, A: Allocator> BinaryReaderMut for Box<T, A> { + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result<usize> { + self.deref_mut().read_from_slice_mut(reader, offset) + } +} + +// Delegate for `Vec<T, A>`: Support a `Vec<T, A>` with an outer lock. +impl<T, A> BinaryReaderMut for Vec<T, A> +where + T: AsBytes + FromBytes, + A: Allocator, +{ + fn read_from_slice_mut( + &mut self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result<usize> { + let slice = self.as_mut_slice(); + + // SAFETY: `T: AsBytes + FromBytes` allows us to treat `&mut [T]` as `&mut [u8]`. + let buffer = unsafe { + core::slice::from_raw_parts_mut( + slice.as_mut_ptr().cast(), + core::mem::size_of_val(slice), + ) + }; + + reader.read_slice_file(buffer, offset) + } +} + +/// Trait for types that can be constructed from a binary representation. +/// +/// See also [`BinaryReaderMut`] for the mutable version. +pub trait BinaryReader { + /// Reads the binary form of `self` from `reader`. + /// + /// `offset` is the requested offset into the binary representation of `self`. + /// + /// On success, returns the number of bytes read from `reader`. + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result<usize>; +} + +// Delegate for `Mutex<T>`: Support a `T` with an outer `Mutex`. +impl<T: BinaryReaderMut + Unpin> BinaryReader for Mutex<T> { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result<usize> { + let mut this = self.lock(); + + this.read_from_slice_mut(reader, offset) + } +} + +// Delegate for `Box<T, A>`: Support a `Box<T, A>` with an inner lock. +impl<T: ?Sized + BinaryReader, A: Allocator> BinaryReader for Box<T, A> { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result<usize> { + self.deref().read_from_slice(reader, offset) + } +} + +// Delegate for `Pin<Box<T, A>>`: Support a `Pin<Box<T, A>>` with an inner lock. +impl<T: ?Sized + BinaryReader, A: Allocator> BinaryReader for Pin<Box<T, A>> { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result<usize> { + self.deref().read_from_slice(reader, offset) + } +} + +// Delegate for `Arc<T>`: Support an `Arc<T>` with an inner lock. +impl<T: ?Sized + BinaryReader> BinaryReader for Arc<T> { + fn read_from_slice( + &self, + reader: &mut UserSliceReader, + offset: &mut file::Offset, + ) -> Result<usize> { + self.deref().read_from_slice(reader, offset) + } +} diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 7116dd7539a6..c79be2e2bfe3 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -6,10 +6,11 @@ use crate::{ bindings, fmt, + prelude::*, sync::aref::ARef, types::{ForeignOwnable, Opaque}, }; -use core::{marker::PhantomData, ptr}; +use core::{any::TypeId, marker::PhantomData, ptr}; #[cfg(CONFIG_PRINTK)] use crate::c_str; @@ -17,6 +18,9 @@ use crate::str::CStrExt as _; pub mod property; +// Assert that we can `read()` / `write()` a `TypeId` instance from / into `struct driver_type`. +static_assert!(core::mem::size_of::<bindings::driver_type>() >= core::mem::size_of::<TypeId>()); + /// The core representation of a device in the kernel's driver model. /// /// This structure represents the Rust abstraction for a C `struct device`. A [`Device`] can either @@ -198,10 +202,31 @@ impl Device { } impl Device<CoreInternal> { + fn set_type_id<T: 'static>(&self) { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + let private = unsafe { (*self.as_raw()).p }; + + // SAFETY: For a bound device (implied by the `CoreInternal` device context), `private` is + // guaranteed to be a valid pointer to a `struct device_private`. + let driver_type = unsafe { &raw mut (*private).driver_type }; + + // SAFETY: `driver_type` is valid for (unaligned) writes of a `TypeId`. + unsafe { + driver_type + .cast::<TypeId>() + .write_unaligned(TypeId::of::<T>()) + }; + } + /// Store a pointer to the bound driver's private data. - pub fn set_drvdata(&self, data: impl ForeignOwnable) { + pub fn set_drvdata<T: 'static>(&self, data: impl PinInit<T, Error>) -> Result { + let data = KBox::pin_init(data, GFP_KERNEL)?; + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. - unsafe { bindings::dev_set_drvdata(self.as_raw(), data.into_foreign().cast()) } + unsafe { bindings::dev_set_drvdata(self.as_raw(), data.into_foreign().cast()) }; + self.set_type_id::<T>(); + + Ok(()) } /// Take ownership of the private data stored in this [`Device`]. @@ -211,16 +236,19 @@ impl Device<CoreInternal> { /// - Must only be called once after a preceding call to [`Device::set_drvdata`]. /// - The type `T` must match the type of the `ForeignOwnable` previously stored by /// [`Device::set_drvdata`]. - pub unsafe fn drvdata_obtain<T: ForeignOwnable>(&self) -> T { + pub unsafe fn drvdata_obtain<T: 'static>(&self) -> Pin<KBox<T>> { // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + unsafe { bindings::dev_set_drvdata(self.as_raw(), core::ptr::null_mut()) }; + // SAFETY: // - By the safety requirements of this function, `ptr` comes from a previous call to // `into_foreign()`. // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` // in `into_foreign()`. - unsafe { T::from_foreign(ptr.cast()) } + unsafe { Pin::<KBox<T>>::from_foreign(ptr.cast()) } } /// Borrow the driver's private data bound to this [`Device`]. @@ -231,7 +259,23 @@ impl Device<CoreInternal> { /// [`Device::drvdata_obtain`]. /// - The type `T` must match the type of the `ForeignOwnable` previously stored by /// [`Device::set_drvdata`]. - pub unsafe fn drvdata_borrow<T: ForeignOwnable>(&self) -> T::Borrowed<'_> { + pub unsafe fn drvdata_borrow<T: 'static>(&self) -> Pin<&T> { + // SAFETY: `drvdata_unchecked()` has the exact same safety requirements as the ones + // required by this method. + unsafe { self.drvdata_unchecked() } + } +} + +impl Device<Bound> { + /// Borrow the driver's private data bound to this [`Device`]. + /// + /// # Safety + /// + /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before + /// [`Device::drvdata_obtain`]. + /// - The type `T` must match the type of the `ForeignOwnable` previously stored by + /// [`Device::set_drvdata`]. + unsafe fn drvdata_unchecked<T: 'static>(&self) -> Pin<&T> { // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; @@ -240,7 +284,46 @@ impl Device<CoreInternal> { // `into_foreign()`. // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` // in `into_foreign()`. - unsafe { T::borrow(ptr.cast()) } + unsafe { Pin::<KBox<T>>::borrow(ptr.cast()) } + } + + fn match_type_id<T: 'static>(&self) -> Result { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + let private = unsafe { (*self.as_raw()).p }; + + // SAFETY: For a bound device, `private` is guaranteed to be a valid pointer to a + // `struct device_private`. + let driver_type = unsafe { &raw mut (*private).driver_type }; + + // SAFETY: + // - `driver_type` is valid for (unaligned) reads of a `TypeId`. + // - A bound device guarantees that `driver_type` contains a valid `TypeId` value. + let type_id = unsafe { driver_type.cast::<TypeId>().read_unaligned() }; + + if type_id != TypeId::of::<T>() { + return Err(EINVAL); + } + + Ok(()) + } + + /// Access a driver's private data. + /// + /// Returns a pinned reference to the driver's private data or [`EINVAL`] if it doesn't match + /// the asserted type `T`. + pub fn drvdata<T: 'static>(&self) -> Result<Pin<&T>> { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + if unsafe { bindings::dev_get_drvdata(self.as_raw()) }.is_null() { + return Err(ENOENT); + } + + self.match_type_id::<T>()?; + + // SAFETY: + // - The above check of `dev_get_drvdata()` guarantees that we are called after + // `set_drvdata()` and before `drvdata_obtain()`. + // - We've just checked that the type of the driver's private data is in fact `T`. + Ok(unsafe { self.drvdata_unchecked() }) } } @@ -512,6 +595,39 @@ impl DeviceContext for Core {} impl DeviceContext for CoreInternal {} impl DeviceContext for Normal {} +/// Convert device references to bus device references. +/// +/// Bus devices can implement this trait to allow abstractions to provide the bus device in +/// class device callbacks. +/// +/// This must not be used by drivers and is intended for bus and class device abstractions only. +/// +/// # Safety +/// +/// `AsBusDevice::OFFSET` must be the offset of the embedded base `struct device` field within a +/// bus device structure. +pub unsafe trait AsBusDevice<Ctx: DeviceContext>: AsRef<Device<Ctx>> { + /// The relative offset to the device field. + /// + /// Use `offset_of!(bindings, field)` macro to avoid breakage. + const OFFSET: usize; + + /// Convert a reference to [`Device`] into `Self`. + /// + /// # Safety + /// + /// `dev` must be contained in `Self`. + unsafe fn from_device(dev: &Device<Ctx>) -> &Self + where + Self: Sized, + { + let raw = dev.as_raw(); + // SAFETY: `raw - Self::OFFSET` is guaranteed by the safety requirements + // to be a valid pointer to `Self`. + unsafe { &*raw.byte_sub(Self::OFFSET).cast::<Self>() } + } +} + /// # Safety /// /// The type given as `$device` must be a transparent wrapper of a type that doesn't depend on the diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 2392c281459e..835d9c11948e 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -52,8 +52,20 @@ struct Inner<T: Send> { /// # Examples /// /// ```no_run -/// # use kernel::{bindings, device::{Bound, Device}, devres::Devres, io::{Io, IoRaw}}; -/// # use core::ops::Deref; +/// use kernel::{ +/// bindings, +/// device::{ +/// Bound, +/// Device, +/// }, +/// devres::Devres, +/// io::{ +/// Io, +/// IoRaw, +/// PhysAddr, +/// }, +/// }; +/// use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. /// struct IoMem<const SIZE: usize>(IoRaw<SIZE>); @@ -66,7 +78,7 @@ struct Inner<T: Send> { /// unsafe fn new(paddr: usize) -> Result<Self>{ /// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is /// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as bindings::phys_addr_t, SIZE) }; +/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; /// if addr.is_null() { /// return Err(ENOMEM); /// } diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 4e0af3e1a3b9..84d3c67269e8 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -12,6 +12,7 @@ use crate::{ sync::aref::ARef, transmute::{AsBytes, FromBytes}, }; +use core::ptr::NonNull; /// DMA address type. /// @@ -358,7 +359,7 @@ pub struct CoherentAllocation<T: AsBytes + FromBytes> { dev: ARef<device::Device>, dma_handle: DmaAddress, count: usize, - cpu_addr: *mut T, + cpu_addr: NonNull<T>, dma_attrs: Attrs, } @@ -392,7 +393,7 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { .ok_or(EOVERFLOW)?; let mut dma_handle = 0; // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. - let ret = unsafe { + let addr = unsafe { bindings::dma_alloc_attrs( dev.as_raw(), size, @@ -401,9 +402,7 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { dma_attrs.as_raw(), ) }; - if ret.is_null() { - return Err(ENOMEM); - } + let addr = NonNull::new(addr).ok_or(ENOMEM)?; // INVARIANT: // - We just successfully allocated a coherent region which is accessible for // `count` elements, hence the cpu address is valid. We also hold a refcounted reference @@ -414,7 +413,7 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { dev: dev.into(), dma_handle, count, - cpu_addr: ret.cast::<T>(), + cpu_addr: addr.cast(), dma_attrs, }) } @@ -446,13 +445,13 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { /// Returns the base address to the allocated region in the CPU's virtual address space. pub fn start_ptr(&self) -> *const T { - self.cpu_addr + self.cpu_addr.as_ptr() } /// Returns the base address to the allocated region in the CPU's virtual address space as /// a mutable pointer. pub fn start_ptr_mut(&mut self) -> *mut T { - self.cpu_addr + self.cpu_addr.as_ptr() } /// Returns a DMA handle which may be given to the device as the DMA address base of @@ -505,7 +504,7 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { // data is also guaranteed by the safety requirements of the function. // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked // that `self.count` won't overflow early in the constructor. - Ok(unsafe { core::slice::from_raw_parts(self.cpu_addr.add(offset), count) }) + Ok(unsafe { core::slice::from_raw_parts(self.start_ptr().add(offset), count) }) } /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable @@ -525,7 +524,7 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { // data is also guaranteed by the safety requirements of the function. // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked // that `self.count` won't overflow early in the constructor. - Ok(unsafe { core::slice::from_raw_parts_mut(self.cpu_addr.add(offset), count) }) + Ok(unsafe { core::slice::from_raw_parts_mut(self.start_ptr_mut().add(offset), count) }) } /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the @@ -557,7 +556,11 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked // that `self.count` won't overflow early in the constructor. unsafe { - core::ptr::copy_nonoverlapping(src.as_ptr(), self.cpu_addr.add(offset), src.len()) + core::ptr::copy_nonoverlapping( + src.as_ptr(), + self.start_ptr_mut().add(offset), + src.len(), + ) }; Ok(()) } @@ -576,7 +579,7 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { // and we've just checked that the range and index is within bounds. // - `offset` can't overflow since it is smaller than `self.count` and we've checked // that `self.count` won't overflow early in the constructor. - Ok(unsafe { self.cpu_addr.add(offset) }) + Ok(unsafe { self.cpu_addr.as_ptr().add(offset) }) } /// Reads the value of `field` and ensures that its type is [`FromBytes`]. @@ -637,7 +640,7 @@ impl<T: AsBytes + FromBytes> Drop for CoherentAllocation<T> { bindings::dma_free_attrs( self.dev.as_raw(), size, - self.cpu_addr.cast(), + self.start_ptr_mut().cast(), self.dma_handle, self.dma_attrs.as_raw(), ) diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index 279e3af20682..9beae2e3d57e 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -24,7 +24,7 @@ //! const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = None; //! //! /// Driver probe. -//! fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>; +//! fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> impl PinInit<Self, Error>; //! //! /// Driver unbind (optional). //! fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) { @@ -35,7 +35,7 @@ //! //! For specific examples see [`auxiliary::Driver`], [`pci::Driver`] and [`platform::Driver`]. //! -//! The `probe()` callback should return a `Result<Pin<KBox<Self>>>`, i.e. the driver's private +//! The `probe()` callback should return a `impl PinInit<Self, Error>`, i.e. the driver's private //! data. The bus abstraction should store the pointer in the corresponding bus device. The generic //! [`Device`] infrastructure provides common helpers for this purpose on its //! [`Device<CoreInternal>`] implementation. diff --git a/rust/kernel/fs/file.rs b/rust/kernel/fs/file.rs index cd6987850332..23ee689bd240 100644 --- a/rust/kernel/fs/file.rs +++ b/rust/kernel/fs/file.rs @@ -17,6 +17,11 @@ use crate::{ }; use core::ptr; +/// Primitive type representing the offset within a [`File`]. +/// +/// Type alias for `bindings::loff_t`. +pub type Offset = bindings::loff_t; + /// Flags associated with a [`File`]. pub mod flags { /// File is opened in append mode. diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs new file mode 100644 index 000000000000..491e6cc25cf4 --- /dev/null +++ b/rust/kernel/i2c.rs @@ -0,0 +1,586 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! I2C Driver subsystem + +// I2C Driver abstractions. +use crate::{ + acpi, + container_of, + device, + device_id::{ + RawDeviceId, + RawDeviceIdIndex, // + }, + devres::Devres, + driver, + error::*, + of, + prelude::*, + types::{ + AlwaysRefCounted, + Opaque, // + }, // +}; + +use core::{ + marker::PhantomData, + mem::offset_of, + ptr::{ + from_ref, + NonNull, // + }, // +}; + +use kernel::types::ARef; + +/// An I2C device id table. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::i2c_device_id); + +impl DeviceId { + const I2C_NAME_SIZE: usize = 20; + + /// Create a new device id from an I2C 'id' string. + #[inline(always)] + pub const fn new(id: &'static CStr) -> Self { + let src = id.to_bytes_with_nul(); + build_assert!(src.len() <= Self::I2C_NAME_SIZE, "ID exceeds 20 bytes"); + let mut i2c: bindings::i2c_device_id = pin_init::zeroed(); + let mut i = 0; + while i < src.len() { + i2c.name[i] = src[i]; + i += 1; + } + + Self(i2c) + } +} + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `i2c_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::i2c_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::i2c_device_id, driver_data); + + fn index(&self) -> usize { + self.0.driver_data + } +} + +/// IdTable type for I2C +pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; + +/// Create a I2C `IdTable` with its alias for modpost. +#[macro_export] +macro_rules! i2c_device_table { + ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { + const $table_name: $crate::device_id::IdArray< + $crate::i2c::DeviceId, + $id_info_type, + { $table_data.len() }, + > = $crate::device_id::IdArray::new($table_data); + + $crate::module_device_table!("i2c", $module_table_name, $table_name); + }; +} + +/// An adapter for the registration of I2C drivers. +pub struct Adapter<T: Driver>(T); + +// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { + type RegType = bindings::i2c_driver; + + unsafe fn register( + idrv: &Opaque<Self::RegType>, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + build_assert!( + T::ACPI_ID_TABLE.is_some() || T::OF_ID_TABLE.is_some() || T::I2C_ID_TABLE.is_some(), + "At least one of ACPI/OF/Legacy tables must be present when registering an i2c driver" + ); + + let i2c_table = match T::I2C_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct i2c_client` on initialization. + unsafe { + (*idrv.get()).driver.name = name.as_char_ptr(); + (*idrv.get()).probe = Some(Self::probe_callback); + (*idrv.get()).remove = Some(Self::remove_callback); + (*idrv.get()).shutdown = Some(Self::shutdown_callback); + (*idrv.get()).id_table = i2c_table; + (*idrv.get()).driver.of_match_table = of_table; + (*idrv.get()).driver.acpi_match_table = acpi_table; + } + + // SAFETY: `idrv` is guaranteed to be a valid `RegType`. + to_result(unsafe { bindings::i2c_register_driver(module.0, idrv.get()) }) + } + + unsafe fn unregister(idrv: &Opaque<Self::RegType>) { + // SAFETY: `idrv` is guaranteed to be a valid `RegType`. + unsafe { bindings::i2c_del_driver(idrv.get()) } + } +} + +impl<T: Driver + 'static> Adapter<T> { + extern "C" fn probe_callback(idev: *mut bindings::i2c_client) -> kernel::ffi::c_int { + // SAFETY: The I2C bus only ever calls the probe callback with a valid pointer to a + // `struct i2c_client`. + // + // INVARIANT: `idev` is valid for the duration of `probe_callback()`. + let idev = unsafe { &*idev.cast::<I2cClient<device::CoreInternal>>() }; + + let info = + Self::i2c_id_info(idev).or_else(|| <Self as driver::Adapter>::id_info(idev.as_ref())); + + from_result(|| { + let data = T::probe(idev, info); + + idev.as_ref().set_drvdata(data)?; + Ok(0) + }) + } + + extern "C" fn remove_callback(idev: *mut bindings::i2c_client) { + // SAFETY: `idev` is a valid pointer to a `struct i2c_client`. + let idev = unsafe { &*idev.cast::<I2cClient<device::CoreInternal>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called + // and stored a `Pin<KBox<T>>`. + let data = unsafe { idev.as_ref().drvdata_obtain::<T>() }; + + T::unbind(idev, data.as_ref()); + } + + extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) { + // SAFETY: `shutdown_callback` is only ever called for a valid `idev` + let idev = unsafe { &*idev.cast::<I2cClient<device::CoreInternal>>() }; + + // SAFETY: `shutdown_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<T>>`. + let data = unsafe { idev.as_ref().drvdata_obtain::<T>() }; + + T::shutdown(idev, data.as_ref()); + } + + /// The [`i2c::IdTable`] of the corresponding driver. + fn i2c_id_table() -> Option<IdTable<<Self as driver::Adapter>::IdInfo>> { + T::I2C_ID_TABLE + } + + /// Returns the driver's private data from the matching entry in the [`i2c::IdTable`], if any. + /// + /// If this returns `None`, it means there is no match with an entry in the [`i2c::IdTable`]. + fn i2c_id_info(dev: &I2cClient) -> Option<&'static <Self as driver::Adapter>::IdInfo> { + let table = Self::i2c_id_table()?; + + // SAFETY: + // - `table` has static lifetime, hence it's valid for reads + // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let raw_id = unsafe { bindings::i2c_match_id(table.as_ptr(), dev.as_raw()) }; + + if raw_id.is_null() { + return None; + } + + // SAFETY: `DeviceId` is a `#[repr(transparent)` wrapper of `struct i2c_device_id` and + // does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*raw_id.cast::<DeviceId>() }; + + Some(table.info(<DeviceId as RawDeviceIdIndex>::index(id))) + } +} + +impl<T: Driver + 'static> driver::Adapter for Adapter<T> { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option<of::IdTable<Self::IdInfo>> { + T::OF_ID_TABLE + } + + fn acpi_id_table() -> Option<acpi::IdTable<Self::IdInfo>> { + T::ACPI_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single i2c driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_i2c_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_i2c_driver { + ($($f:tt)*) => { + $crate::module_driver!(<T>, $crate::i2c::Adapter<T>, { $($f)* }); + }; +} + +/// The i2c driver trait. +/// +/// Drivers must implement this trait in order to get a i2c driver registered. +/// +/// # Example +/// +///``` +/// # use kernel::{acpi, bindings, c_str, device::Core, i2c, of}; +/// +/// struct MyDriver; +/// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// MODULE_ACPI_TABLE, +/// <MyDriver as i2c::Driver>::IdInfo, +/// [ +/// (acpi::DeviceId::new(c_str!("LNUXBEEF")), ()) +/// ] +/// ); +/// +/// kernel::i2c_device_table!( +/// I2C_TABLE, +/// MODULE_I2C_TABLE, +/// <MyDriver as i2c::Driver>::IdInfo, +/// [ +/// (i2c::DeviceId::new(c_str!("rust_driver_i2c")), ()) +/// ] +/// ); +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// MODULE_OF_TABLE, +/// <MyDriver as i2c::Driver>::IdInfo, +/// [ +/// (of::DeviceId::new(c_str!("test,device")), ()) +/// ] +/// ); +/// +/// impl i2c::Driver for MyDriver { +/// type IdInfo = (); +/// const I2C_ID_TABLE: Option<i2c::IdTable<Self::IdInfo>> = Some(&I2C_TABLE); +/// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = Some(&ACPI_TABLE); +/// +/// fn probe( +/// _idev: &i2c::I2cClient<Core>, +/// _id_info: Option<&Self::IdInfo>, +/// ) -> impl PinInit<Self, Error> { +/// Err(ENODEV) +/// } +/// +/// fn shutdown(_idev: &i2c::I2cClient<Core>, this: Pin<&Self>) { +/// } +/// } +///``` +pub trait Driver: Send { + /// The type holding information about each device id supported by the driver. + // TODO: Use `associated_type_defaults` once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The table of device ids supported by the driver. + const I2C_ID_TABLE: Option<IdTable<Self::IdInfo>> = None; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = None; + + /// I2C driver probe. + /// + /// Called when a new i2c client is added or discovered. + /// Implementers should attempt to initialize the client here. + fn probe( + dev: &I2cClient<device::Core>, + id_info: Option<&Self::IdInfo>, + ) -> impl PinInit<Self, Error>; + + /// I2C driver shutdown. + /// + /// Called by the kernel during system reboot or power-off to allow the [`Driver`] to bring the + /// [`I2cClient`] into a safe state. Implementing this callback is optional. + /// + /// Typical actions include stopping transfers, disabling interrupts, or resetting the hardware + /// to prevent undesired behavior during shutdown. + /// + /// This callback is distinct from final resource cleanup, as the driver instance remains valid + /// after it returns. Any deallocation or teardown of driver-owned resources should instead be + /// handled in `Self::drop`. + fn shutdown(dev: &I2cClient<device::Core>, this: Pin<&Self>) { + let _ = (dev, this); + } + + /// I2C driver unbind. + /// + /// Called when the [`I2cClient`] is unbound from its bound [`Driver`]. Implementing this + /// callback is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Self::drop`. + fn unbind(dev: &I2cClient<device::Core>, this: Pin<&Self>) { + let _ = (dev, this); + } +} + +/// The i2c adapter representation. +/// +/// This structure represents the Rust abstraction for a C `struct i2c_adapter`. The +/// implementation abstracts the usage of an existing C `struct i2c_adapter` that +/// gets passed from the C side +/// +/// # Invariants +/// +/// A [`I2cAdapter`] instance represents a valid `struct i2c_adapter` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct I2cAdapter<Ctx: device::DeviceContext = device::Normal>( + Opaque<bindings::i2c_adapter>, + PhantomData<Ctx>, +); + +impl<Ctx: device::DeviceContext> I2cAdapter<Ctx> { + fn as_raw(&self) -> *mut bindings::i2c_adapter { + self.0.get() + } +} + +impl I2cAdapter { + /// Returns the I2C Adapter index. + #[inline] + pub fn index(&self) -> i32 { + // SAFETY: `self.as_raw` is a valid pointer to a `struct i2c_adapter`. + unsafe { (*self.as_raw()).nr } + } + + /// Gets pointer to an `i2c_adapter` by index. + pub fn get(index: i32) -> Result<ARef<Self>> { + // SAFETY: `index` must refer to a valid I2C adapter; the kernel + // guarantees that `i2c_get_adapter(index)` returns either a valid + // pointer or NULL. `NonNull::new` guarantees the correct check. + let adapter = NonNull::new(unsafe { bindings::i2c_get_adapter(index) }).ok_or(ENODEV)?; + + // SAFETY: `adapter` is non-null and points to a live `i2c_adapter`. + // `I2cAdapter` is #[repr(transparent)], so this cast is valid. + Ok(unsafe { (&*adapter.as_ptr().cast::<I2cAdapter<device::Normal>>()).into() }) + } +} + +// SAFETY: `I2cAdapter` is a transparent wrapper of a type that doesn't depend on +// `I2cAdapter`'s generic argument. +kernel::impl_device_context_deref!(unsafe { I2cAdapter }); +kernel::impl_device_context_into_aref!(I2cAdapter); + +// SAFETY: Instances of `I2cAdapter` are always reference-counted. +unsafe impl crate::types::AlwaysRefCounted for I2cAdapter { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::i2c_get_adapter(self.index()) }; + } + + unsafe fn dec_ref(obj: NonNull<Self>) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::i2c_put_adapter(obj.as_ref().as_raw()) } + } +} + +/// The i2c board info representation +/// +/// This structure represents the Rust abstraction for a C `struct i2c_board_info` structure, +/// which is used for manual I2C client creation. +#[repr(transparent)] +pub struct I2cBoardInfo(bindings::i2c_board_info); + +impl I2cBoardInfo { + const I2C_TYPE_SIZE: usize = 20; + /// Create a new [`I2cBoardInfo`] for a kernel driver. + #[inline(always)] + pub const fn new(type_: &'static CStr, addr: u16) -> Self { + let src = type_.to_bytes_with_nul(); + build_assert!(src.len() <= Self::I2C_TYPE_SIZE, "Type exceeds 20 bytes"); + let mut i2c_board_info: bindings::i2c_board_info = pin_init::zeroed(); + let mut i: usize = 0; + while i < src.len() { + i2c_board_info.type_[i] = src[i]; + i += 1; + } + + i2c_board_info.addr = addr; + Self(i2c_board_info) + } + + fn as_raw(&self) -> *const bindings::i2c_board_info { + from_ref(&self.0) + } +} + +/// The i2c client representation. +/// +/// This structure represents the Rust abstraction for a C `struct i2c_client`. The +/// implementation abstracts the usage of an existing C `struct i2c_client` that +/// gets passed from the C side +/// +/// # Invariants +/// +/// A [`I2cClient`] instance represents a valid `struct i2c_client` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct I2cClient<Ctx: device::DeviceContext = device::Normal>( + Opaque<bindings::i2c_client>, + PhantomData<Ctx>, +); + +impl<Ctx: device::DeviceContext> I2cClient<Ctx> { + fn as_raw(&self) -> *mut bindings::i2c_client { + self.0.get() + } +} + +// SAFETY: `I2cClient` is a transparent wrapper of `struct i2c_client`. +// The offset is guaranteed to point to a valid device field inside `I2cClient`. +unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for I2cClient<Ctx> { + const OFFSET: usize = offset_of!(bindings::i2c_client, dev); +} + +// SAFETY: `I2cClient` is a transparent wrapper of a type that doesn't depend on +// `I2cClient`'s generic argument. +kernel::impl_device_context_deref!(unsafe { I2cClient }); +kernel::impl_device_context_into_aref!(I2cClient); + +// SAFETY: Instances of `I2cClient` are always reference-counted. +unsafe impl AlwaysRefCounted for I2cClient { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. + unsafe { bindings::get_device(self.as_ref().as_raw()) }; + } + + unsafe fn dec_ref(obj: NonNull<Self>) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::put_device(&raw mut (*obj.as_ref().as_raw()).dev) } + } +} + +impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for I2cClient<Ctx> { + fn as_ref(&self) -> &device::Device<Ctx> { + let raw = self.as_raw(); + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct i2c_client`. + let dev = unsafe { &raw mut (*raw).dev }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +impl<Ctx: device::DeviceContext> TryFrom<&device::Device<Ctx>> for &I2cClient<Ctx> { + type Error = kernel::error::Error; + + fn try_from(dev: &device::Device<Ctx>) -> Result<Self, Self::Error> { + // SAFETY: By the type invariant of `Device`, `dev.as_raw()` is a valid pointer to a + // `struct device`. + if unsafe { bindings::i2c_verify_client(dev.as_raw()).is_null() } { + return Err(EINVAL); + } + + // SAFETY: We've just verified that the type of `dev` equals to + // `bindings::i2c_client_type`, hence `dev` must be embedded in a valid + // `struct i2c_client` as guaranteed by the corresponding C code. + let idev = unsafe { container_of!(dev.as_raw(), bindings::i2c_client, dev) }; + + // SAFETY: `idev` is a valid pointer to a `struct i2c_client`. + Ok(unsafe { &*idev.cast() }) + } +} + +// SAFETY: A `I2cClient` is always reference-counted and can be released from any thread. +unsafe impl Send for I2cClient {} + +// SAFETY: `I2cClient` can be shared among threads because all methods of `I2cClient` +// (i.e. `I2cClient<Normal>) are thread safe. +unsafe impl Sync for I2cClient {} + +/// The registration of an i2c client device. +/// +/// This type represents the registration of a [`struct i2c_client`]. When an instance of this +/// type is dropped, its respective i2c client device will be unregistered from the system. +/// +/// # Invariants +/// +/// `self.0` always holds a valid pointer to an initialized and registered +/// [`struct i2c_client`]. +#[repr(transparent)] +pub struct Registration(NonNull<bindings::i2c_client>); + +impl Registration { + /// The C `i2c_new_client_device` function wrapper for manual I2C client creation. + pub fn new<'a>( + i2c_adapter: &I2cAdapter, + i2c_board_info: &I2cBoardInfo, + parent_dev: &'a device::Device<device::Bound>, + ) -> impl PinInit<Devres<Self>, Error> + 'a { + Devres::new(parent_dev, Self::try_new(i2c_adapter, i2c_board_info)) + } + + fn try_new(i2c_adapter: &I2cAdapter, i2c_board_info: &I2cBoardInfo) -> Result<Self> { + // SAFETY: the kernel guarantees that `i2c_new_client_device()` returns either a valid + // pointer or NULL. `from_err_ptr` separates errors. Following `NonNull::new` + // checks for NULL. + let raw_dev = from_err_ptr(unsafe { + bindings::i2c_new_client_device(i2c_adapter.as_raw(), i2c_board_info.as_raw()) + })?; + + let dev_ptr = NonNull::new(raw_dev).ok_or(ENODEV)?; + + Ok(Self(dev_ptr)) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: `Drop` is only called for a valid `Registration`, which by invariant + // always contains a non-null pointer to an `i2c_client`. + unsafe { bindings::i2c_unregister_device(self.0.as_ptr()) } + } +} + +// SAFETY: A `Registration` of a `struct i2c_client` can be released from any thread. +unsafe impl Send for Registration {} + +// SAFETY: `Registration` offers no interior mutability (no mutation through &self +// and no mutable access is exposed) +unsafe impl Sync for Registration {} diff --git a/rust/kernel/id_pool.rs b/rust/kernel/id_pool.rs index a41a3404213c..384753fe0e44 100644 --- a/rust/kernel/id_pool.rs +++ b/rust/kernel/id_pool.rs @@ -7,8 +7,6 @@ use crate::alloc::{AllocError, Flags}; use crate::bitmap::BitmapVec; -const BITS_PER_LONG: usize = bindings::BITS_PER_LONG as usize; - /// Represents a dynamic ID pool backed by a [`BitmapVec`]. /// /// Clients acquire and release IDs from unset bits in a bitmap. @@ -25,22 +23,22 @@ const BITS_PER_LONG: usize = bindings::BITS_PER_LONG as usize; /// Basic usage /// /// ``` -/// use kernel::alloc::{AllocError, flags::GFP_KERNEL}; -/// use kernel::id_pool::IdPool; +/// use kernel::alloc::AllocError; +/// use kernel::id_pool::{IdPool, UnusedId}; /// -/// let mut pool = IdPool::new(64, GFP_KERNEL)?; +/// let mut pool = IdPool::with_capacity(64, GFP_KERNEL)?; /// for i in 0..64 { -/// assert_eq!(i, pool.acquire_next_id(i).ok_or(ENOSPC)?); +/// assert_eq!(i, pool.find_unused_id(i).ok_or(ENOSPC)?.acquire()); /// } /// /// pool.release_id(23); -/// assert_eq!(23, pool.acquire_next_id(0).ok_or(ENOSPC)?); +/// assert_eq!(23, pool.find_unused_id(0).ok_or(ENOSPC)?.acquire()); /// -/// assert_eq!(None, pool.acquire_next_id(0)); // time to realloc. +/// assert!(pool.find_unused_id(0).is_none()); // time to realloc. /// let resizer = pool.grow_request().ok_or(ENOSPC)?.realloc(GFP_KERNEL)?; /// pool.grow(resizer); /// -/// assert_eq!(pool.acquire_next_id(0), Some(64)); +/// assert_eq!(pool.find_unused_id(0).ok_or(ENOSPC)?.acquire(), 64); /// # Ok::<(), Error>(()) /// ``` /// @@ -54,8 +52,8 @@ const BITS_PER_LONG: usize = bindings::BITS_PER_LONG as usize; /// fn get_id_maybe_realloc(guarded_pool: &SpinLock<IdPool>) -> Result<usize, AllocError> { /// let mut pool = guarded_pool.lock(); /// loop { -/// match pool.acquire_next_id(0) { -/// Some(index) => return Ok(index), +/// match pool.find_unused_id(0) { +/// Some(index) => return Ok(index.acquire()), /// None => { /// let alloc_request = pool.grow_request(); /// drop(pool); @@ -97,13 +95,24 @@ impl ReallocRequest { impl IdPool { /// Constructs a new [`IdPool`]. /// - /// A capacity below [`BITS_PER_LONG`] is adjusted to - /// [`BITS_PER_LONG`]. + /// The pool will have a capacity of [`MAX_INLINE_LEN`]. + /// + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN + #[inline] + pub fn new() -> Self { + Self { + map: BitmapVec::new_inline(), + } + } + + /// Constructs a new [`IdPool`] with space for a specific number of bits. + /// + /// A capacity below [`MAX_INLINE_LEN`] is adjusted to [`MAX_INLINE_LEN`]. /// - /// [`BITS_PER_LONG`]: srctree/include/asm-generic/bitsperlong.h + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN #[inline] - pub fn new(num_ids: usize, flags: Flags) -> Result<Self, AllocError> { - let num_ids = core::cmp::max(num_ids, BITS_PER_LONG); + pub fn with_capacity(num_ids: usize, flags: Flags) -> Result<Self, AllocError> { + let num_ids = usize::max(num_ids, BitmapVec::MAX_INLINE_LEN); let map = BitmapVec::new(num_ids, flags)?; Ok(Self { map }) } @@ -116,28 +125,34 @@ impl IdPool { /// Returns a [`ReallocRequest`] if the [`IdPool`] can be shrunk, [`None`] otherwise. /// - /// The capacity of an [`IdPool`] cannot be shrunk below [`BITS_PER_LONG`]. + /// The capacity of an [`IdPool`] cannot be shrunk below [`MAX_INLINE_LEN`]. /// - /// [`BITS_PER_LONG`]: srctree/include/asm-generic/bitsperlong.h + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN /// /// # Examples /// /// ``` - /// use kernel::alloc::{AllocError, flags::GFP_KERNEL}; - /// use kernel::id_pool::{ReallocRequest, IdPool}; + /// use kernel::{ + /// alloc::AllocError, + /// bitmap::BitmapVec, + /// id_pool::{ + /// IdPool, + /// ReallocRequest, + /// }, + /// }; /// - /// let mut pool = IdPool::new(1024, GFP_KERNEL)?; + /// let mut pool = IdPool::with_capacity(1024, GFP_KERNEL)?; /// let alloc_request = pool.shrink_request().ok_or(AllocError)?; /// let resizer = alloc_request.realloc(GFP_KERNEL)?; /// pool.shrink(resizer); - /// assert_eq!(pool.capacity(), kernel::bindings::BITS_PER_LONG as usize); + /// assert_eq!(pool.capacity(), BitmapVec::MAX_INLINE_LEN); /// # Ok::<(), AllocError>(()) /// ``` #[inline] pub fn shrink_request(&self) -> Option<ReallocRequest> { let cap = self.capacity(); - // Shrinking below [`BITS_PER_LONG`] is never possible. - if cap <= BITS_PER_LONG { + // Shrinking below `MAX_INLINE_LEN` is never possible. + if cap <= BitmapVec::MAX_INLINE_LEN { return None; } // Determine if the bitmap can shrink based on the position of @@ -146,13 +161,13 @@ impl IdPool { // bitmap should shrink to half its current size. let Some(bit) = self.map.last_bit() else { return Some(ReallocRequest { - num_ids: BITS_PER_LONG, + num_ids: BitmapVec::MAX_INLINE_LEN, }); }; if bit >= (cap / 4) { return None; } - let num_ids = usize::max(BITS_PER_LONG, cap / 2); + let num_ids = usize::max(BitmapVec::MAX_INLINE_LEN, cap / 2); Some(ReallocRequest { num_ids }) } @@ -177,11 +192,13 @@ impl IdPool { /// Returns a [`ReallocRequest`] for growing this [`IdPool`], if possible. /// - /// The capacity of an [`IdPool`] cannot be grown above [`i32::MAX`]. + /// The capacity of an [`IdPool`] cannot be grown above [`MAX_LEN`]. + /// + /// [`MAX_LEN`]: BitmapVec::MAX_LEN #[inline] pub fn grow_request(&self) -> Option<ReallocRequest> { let num_ids = self.capacity() * 2; - if num_ids > i32::MAX.try_into().unwrap() { + if num_ids > BitmapVec::MAX_LEN { return None; } Some(ReallocRequest { num_ids }) @@ -204,18 +221,18 @@ impl IdPool { self.map = resizer.new; } - /// Acquires a new ID by finding and setting the next zero bit in the - /// bitmap. + /// Finds an unused ID in the bitmap. /// /// Upon success, returns its index. Otherwise, returns [`None`] /// to indicate that a [`Self::grow_request`] is needed. #[inline] - pub fn acquire_next_id(&mut self, offset: usize) -> Option<usize> { - let next_zero_bit = self.map.next_zero_bit(offset); - if let Some(nr) = next_zero_bit { - self.map.set_bit(nr); - } - next_zero_bit + #[must_use] + pub fn find_unused_id(&mut self, offset: usize) -> Option<UnusedId<'_>> { + // INVARIANT: `next_zero_bit()` returns None or an integer less than `map.len()` + Some(UnusedId { + id: self.map.next_zero_bit(offset)?, + pool: self, + }) } /// Releases an ID. @@ -224,3 +241,55 @@ impl IdPool { self.map.clear_bit(id); } } + +/// Represents an unused id in an [`IdPool`]. +/// +/// # Invariants +/// +/// The value of `id` is less than `pool.map.len()`. +pub struct UnusedId<'pool> { + id: usize, + pool: &'pool mut IdPool, +} + +impl<'pool> UnusedId<'pool> { + /// Get the unused id as an usize. + /// + /// Be aware that the id has not yet been acquired in the pool. The + /// [`acquire`] method must be called to prevent others from taking the id. + /// + /// [`acquire`]: UnusedId::acquire() + #[inline] + #[must_use] + pub fn as_usize(&self) -> usize { + self.id + } + + /// Get the unused id as an u32. + /// + /// Be aware that the id has not yet been acquired in the pool. The + /// [`acquire`] method must be called to prevent others from taking the id. + /// + /// [`acquire`]: UnusedId::acquire() + #[inline] + #[must_use] + pub fn as_u32(&self) -> u32 { + // CAST: By the type invariants: + // `self.id < pool.map.len() <= BitmapVec::MAX_LEN = i32::MAX`. + self.id as u32 + } + + /// Acquire the unused id. + #[inline] + pub fn acquire(self) -> usize { + self.pool.map.set_bit(self.id); + self.id + } +} + +impl Default for IdPool { + #[inline] + fn default() -> Self { + Self::new() + } +} diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index ee182b0b5452..98e8b84e68d1 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -4,8 +4,10 @@ //! //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) -use crate::error::{code::EINVAL, Result}; -use crate::{bindings, build_assert, ffi::c_void}; +use crate::{ + bindings, + prelude::*, // +}; pub mod mem; pub mod poll; @@ -13,6 +15,18 @@ pub mod resource; pub use resource::Resource; +/// Physical address type. +/// +/// This is a type alias to either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. +pub type PhysAddr = bindings::phys_addr_t; + +/// Resource Size type. +/// +/// This is a type alias to either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. +pub type ResourceSize = bindings::resource_size_t; + /// Raw representation of an MMIO region. /// /// By itself, the existence of an instance of this structure does not provide any guarantees that @@ -62,8 +76,16 @@ impl<const SIZE: usize> IoRaw<SIZE> { /// # Examples /// /// ```no_run -/// # use kernel::{bindings, ffi::c_void, io::{Io, IoRaw}}; -/// # use core::ops::Deref; +/// use kernel::{ +/// bindings, +/// ffi::c_void, +/// io::{ +/// Io, +/// IoRaw, +/// PhysAddr, +/// }, +/// }; +/// use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. /// struct IoMem<const SIZE: usize>(IoRaw<SIZE>); @@ -76,7 +98,7 @@ impl<const SIZE: usize> IoRaw<SIZE> { /// unsafe fn new(paddr: usize) -> Result<Self>{ /// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is /// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as bindings::phys_addr_t, SIZE) }; +/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; /// if addr.is_null() { /// return Err(ENOMEM); /// } diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 6f99510bfc3a..b03b82cd531b 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -4,16 +4,24 @@ use core::ops::Deref; -use crate::c_str; -use crate::device::Bound; -use crate::device::Device; -use crate::devres::Devres; -use crate::io; -use crate::io::resource::Region; -use crate::io::resource::Resource; -use crate::io::Io; -use crate::io::IoRaw; -use crate::prelude::*; +use crate::{ + c_str, + device::{ + Bound, + Device, // + }, + devres::Devres, + io::{ + self, + resource::{ + Region, + Resource, // + }, + Io, + IoRaw, // + }, + prelude::*, +}; /// An IO request for a specific device and resource. pub struct IoRequest<'a> { @@ -53,7 +61,7 @@ impl<'a> IoRequest<'a> { /// fn probe( /// pdev: &platform::Device<Core>, /// info: Option<&Self::IdInfo>, - /// ) -> Result<Pin<KBox<Self>>> { + /// ) -> impl PinInit<Self, Error> { /// let offset = 0; // Some offset. /// /// // If the size is known at compile time, use [`Self::iomap_sized`]. @@ -70,7 +78,7 @@ impl<'a> IoRequest<'a> { /// /// io.write32_relaxed(data, offset); /// - /// # Ok(KBox::new(SampleDriver, GFP_KERNEL)?.into()) + /// # Ok(SampleDriver) /// } /// } /// ``` @@ -111,7 +119,7 @@ impl<'a> IoRequest<'a> { /// fn probe( /// pdev: &platform::Device<Core>, /// info: Option<&Self::IdInfo>, - /// ) -> Result<Pin<KBox<Self>>> { + /// ) -> impl PinInit<Self, Error> { /// let offset = 0; // Some offset. /// /// // Unlike [`Self::iomap_sized`], here the size of the memory region @@ -128,7 +136,7 @@ impl<'a> IoRequest<'a> { /// /// io.try_write32_relaxed(data, offset)?; /// - /// # Ok(KBox::new(SampleDriver, GFP_KERNEL)?.into()) + /// # Ok(SampleDriver) /// } /// } /// ``` diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs index 613eb25047ef..b1a2570364f4 100644 --- a/rust/kernel/io/poll.rs +++ b/rust/kernel/io/poll.rs @@ -5,10 +5,18 @@ //! C header: [`include/linux/iopoll.h`](srctree/include/linux/iopoll.h). use crate::{ - error::{code::*, Result}, + prelude::*, processor::cpu_relax, task::might_sleep, - time::{delay::fsleep, Delta, Instant, Monotonic}, + time::{ + delay::{ + fsleep, + udelay, // + }, + Delta, + Instant, + Monotonic, // + }, }; /// Polls periodically until a condition is met, an error occurs, @@ -42,8 +50,8 @@ use crate::{ /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware<const SIZE: usize>(io: &Io<SIZE>) -> Result<()> { -/// match read_poll_timeout( +/// fn wait_for_hardware<const SIZE: usize>(io: &Io<SIZE>) -> Result { +/// read_poll_timeout( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), /// // The `cond` closure takes a reference to the value returned by `op` @@ -51,14 +59,8 @@ use crate::{ /// |val: &u16| *val == HW_READY, /// Delta::from_millis(50), /// Delta::from_secs(3), -/// ) { -/// Ok(_) => { -/// // The hardware is ready. The returned value of the `op` closure -/// // isn't used. -/// Ok(()) -/// } -/// Err(e) => Err(e), -/// } +/// )?; +/// Ok(()) /// } /// ``` #[track_caller] @@ -102,3 +104,70 @@ where cpu_relax(); } } + +/// Polls periodically until a condition is met, an error occurs, +/// or the attempt limit is reached. +/// +/// The function repeatedly executes the given operation `op` closure and +/// checks its result using the condition closure `cond`. +/// +/// If `cond` returns `true`, the function returns successfully with the result of `op`. +/// Otherwise, it performs a busy wait for a duration specified by `delay_delta` +/// before executing `op` again. +/// +/// This process continues until either `op` returns an error, `cond` +/// returns `true`, or the attempt limit specified by `retry` is reached. +/// +/// # Errors +/// +/// If `op` returns an error, then that error is returned directly. +/// +/// If the attempt limit specified by `retry` is reached, then +/// `Err(ETIMEDOUT)` is returned. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::io::{poll::read_poll_timeout_atomic, Io}; +/// use kernel::time::Delta; +/// +/// const HW_READY: u16 = 0x01; +/// +/// fn wait_for_hardware<const SIZE: usize>(io: &Io<SIZE>) -> Result { +/// read_poll_timeout_atomic( +/// // The `op` closure reads the value of a specific status register. +/// || io.try_read16(0x1000), +/// // The `cond` closure takes a reference to the value returned by `op` +/// // and checks whether the hardware is ready. +/// |val: &u16| *val == HW_READY, +/// Delta::from_micros(50), +/// 1000, +/// )?; +/// Ok(()) +/// } +/// ``` +pub fn read_poll_timeout_atomic<Op, Cond, T>( + mut op: Op, + mut cond: Cond, + delay_delta: Delta, + retry: usize, +) -> Result<T> +where + Op: FnMut() -> Result<T>, + Cond: FnMut(&T) -> bool, +{ + for _ in 0..retry { + let val = op()?; + if cond(&val) { + return Ok(val); + } + + if !delay_delta.is_zero() { + udelay(delay_delta); + } + + cpu_relax(); + } + + Err(ETIMEDOUT) +} diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs index bea3ee0ed87b..56cfde97ce87 100644 --- a/rust/kernel/io/resource.rs +++ b/rust/kernel/io/resource.rs @@ -5,18 +5,21 @@ //! //! C header: [`include/linux/ioport.h`](srctree/include/linux/ioport.h) -use core::ops::Deref; -use core::ptr::NonNull; - -use crate::prelude::*; -use crate::str::{CStr, CString}; -use crate::types::Opaque; - -/// Resource Size type. -/// -/// This is a type alias to either `u32` or `u64` depending on the config option -/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. -pub type ResourceSize = bindings::phys_addr_t; +use core::{ + ops::Deref, + ptr::NonNull, // +}; + +use crate::{ + prelude::*, + str::CString, + types::Opaque, // +}; + +pub use super::{ + PhysAddr, + ResourceSize, // +}; /// A region allocated from a parent [`Resource`]. /// @@ -97,7 +100,7 @@ impl Resource { /// the region, or a part of it, is already in use. pub fn request_region( &self, - start: ResourceSize, + start: PhysAddr, size: ResourceSize, name: CString, flags: Flags, @@ -131,7 +134,7 @@ impl Resource { } /// Returns the start address of the resource. - pub fn start(&self) -> ResourceSize { + pub fn start(&self) -> PhysAddr { let inner = self.0.get(); // SAFETY: Safe as per the invariants of `Resource`. unsafe { (*inner).start } diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index a4eee75cf07a..6083dec1f190 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -97,6 +97,8 @@ pub mod faux; pub mod firmware; pub mod fmt; pub mod fs; +#[cfg(CONFIG_I2C = "y")] +pub mod i2c; pub mod id_pool; pub mod init; pub mod io; @@ -110,6 +112,7 @@ pub mod list; pub mod maple_tree; pub mod miscdevice; pub mod mm; +pub mod module_param; #[cfg(CONFIG_NET)] pub mod net; pub mod num; diff --git a/rust/kernel/mm/virt.rs b/rust/kernel/mm/virt.rs index a1bfa4e19293..da21d65ccd20 100644 --- a/rust/kernel/mm/virt.rs +++ b/rust/kernel/mm/virt.rs @@ -250,7 +250,7 @@ impl VmaNew { // SAFETY: This is not a data race: the vma is undergoing initial setup, so it's not yet // shared. Additionally, `VmaNew` is `!Sync`, so it cannot be used to write in parallel. // The caller promises that this does not set the flags to an invalid value. - unsafe { (*self.as_ptr()).__bindgen_anon_2.__vm_flags = flags }; + unsafe { (*self.as_ptr()).__bindgen_anon_2.vm_flags = flags }; } /// Set the `VM_MIXEDMAP` flag on this vma. diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs new file mode 100644 index 000000000000..6a8a7a875643 --- /dev/null +++ b/rust/kernel/module_param.rs @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Support for module parameters. +//! +//! C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h) + +use crate::prelude::*; +use crate::str::BStr; +use bindings; +use kernel::sync::SetOnce; + +/// Newtype to make `bindings::kernel_param` [`Sync`]. +#[repr(transparent)] +#[doc(hidden)] +pub struct KernelParam(bindings::kernel_param); + +impl KernelParam { + #[doc(hidden)] + pub const fn new(val: bindings::kernel_param) -> Self { + Self(val) + } +} + +// SAFETY: C kernel handles serializing access to this type. We never access it +// from Rust module. +unsafe impl Sync for KernelParam {} + +/// Types that can be used for module parameters. +// NOTE: This trait is `Copy` because drop could produce unsoundness during teardown. +pub trait ModuleParam: Sized + Copy { + /// Parse a parameter argument into the parameter value. + fn try_from_param_arg(arg: &BStr) -> Result<Self>; +} + +/// Set the module parameter from a string. +/// +/// Used to set the parameter value at kernel initialization, when loading +/// the module or when set through `sysfs`. +/// +/// See `struct kernel_param_ops.set`. +/// +/// # Safety +/// +/// - If `val` is non-null then it must point to a valid null-terminated string that must be valid +/// for reads for the duration of the call. +/// - `param` must be a pointer to a `bindings::kernel_param` initialized by the rust module macro. +/// The pointee must be valid for reads for the duration of the call. +/// +/// # Note +/// +/// - The safety requirements are satisfied by C API contract when this function is invoked by the +/// module subsystem C code. +/// - Currently, we only support read-only parameters that are not readable from `sysfs`. Thus, this +/// function is only called at kernel initialization time, or at module load time, and we have +/// exclusive access to the parameter for the duration of the function. +/// +/// [`module!`]: macros::module +unsafe extern "C" fn set_param<T>(val: *const c_char, param: *const bindings::kernel_param) -> c_int +where + T: ModuleParam, +{ + // NOTE: If we start supporting arguments without values, val _is_ allowed + // to be null here. + if val.is_null() { + // TODO: Use pr_warn_once available. + crate::pr_warn!("Null pointer passed to `module_param::set_param`"); + return EINVAL.to_errno(); + } + + // SAFETY: By function safety requirement, val is non-null, null-terminated + // and valid for reads for the duration of this function. + let arg = unsafe { CStr::from_char_ptr(val) }; + let arg: &BStr = arg.as_ref(); + + crate::error::from_result(|| { + let new_value = T::try_from_param_arg(arg)?; + + // SAFETY: By function safety requirements, this access is safe. + let container = unsafe { &*((*param).__bindgen_anon_1.arg.cast::<SetOnce<T>>()) }; + + container + .populate(new_value) + .then_some(0) + .ok_or(kernel::error::code::EEXIST) + }) +} + +macro_rules! impl_int_module_param { + ($ty:ident) => { + impl ModuleParam for $ty { + fn try_from_param_arg(arg: &BStr) -> Result<Self> { + <$ty as crate::str::parse_int::ParseInt>::from_str(arg) + } + } + }; +} + +impl_int_module_param!(i8); +impl_int_module_param!(u8); +impl_int_module_param!(i16); +impl_int_module_param!(u16); +impl_int_module_param!(i32); +impl_int_module_param!(u32); +impl_int_module_param!(i64); +impl_int_module_param!(u64); +impl_int_module_param!(isize); +impl_int_module_param!(usize); + +/// A wrapper for kernel parameters. +/// +/// This type is instantiated by the [`module!`] macro when module parameters are +/// defined. You should never need to instantiate this type directly. +/// +/// Note: This type is `pub` because it is used by module crates to access +/// parameter values. +pub struct ModuleParamAccess<T> { + value: SetOnce<T>, + default: T, +} + +// SAFETY: We only create shared references to the contents of this container, +// so if `T` is `Sync`, so is `ModuleParamAccess`. +unsafe impl<T: Sync> Sync for ModuleParamAccess<T> {} + +impl<T> ModuleParamAccess<T> { + #[doc(hidden)] + pub const fn new(default: T) -> Self { + Self { + value: SetOnce::new(), + default, + } + } + + /// Get a shared reference to the parameter value. + // Note: When sysfs access to parameters are enabled, we have to pass in a + // held lock guard here. + pub fn value(&self) -> &T { + self.value.as_ref().unwrap_or(&self.default) + } + + /// Get a mutable pointer to `self`. + /// + /// NOTE: In most cases it is not safe deref the returned pointer. + pub const fn as_void_ptr(&self) -> *mut c_void { + core::ptr::from_ref(self).cast_mut().cast() + } +} + +#[doc(hidden)] +/// Generate a static [`kernel_param_ops`](srctree/include/linux/moduleparam.h) struct. +/// +/// # Examples +/// +/// ```ignore +/// make_param_ops!( +/// /// Documentation for new param ops. +/// PARAM_OPS_MYTYPE, // Name for the static. +/// MyType // A type which implements [`ModuleParam`]. +/// ); +/// ``` +macro_rules! make_param_ops { + ($ops:ident, $ty:ty) => { + #[doc(hidden)] + pub static $ops: $crate::bindings::kernel_param_ops = $crate::bindings::kernel_param_ops { + flags: 0, + set: Some(set_param::<$ty>), + get: None, + free: None, + }; + }; +} + +make_param_ops!(PARAM_OPS_I8, i8); +make_param_ops!(PARAM_OPS_U8, u8); +make_param_ops!(PARAM_OPS_I16, i16); +make_param_ops!(PARAM_OPS_U16, u16); +make_param_ops!(PARAM_OPS_I32, i32); +make_param_ops!(PARAM_OPS_U32, u32); +make_param_ops!(PARAM_OPS_I64, i64); +make_param_ops!(PARAM_OPS_U64, u64); +make_param_ops!(PARAM_OPS_ISIZE, isize); +make_param_ops!(PARAM_OPS_USIZE, usize); diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 7fcc5f6022c1..82e128431f08 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -5,28 +5,47 @@ //! C header: [`include/linux/pci.h`](srctree/include/linux/pci.h) use crate::{ - bindings, container_of, device, - device_id::{RawDeviceId, RawDeviceIdIndex}, - devres::Devres, + bindings, + container_of, + device, + device_id::{ + RawDeviceId, + RawDeviceIdIndex, // + }, driver, - error::{from_result, to_result, Result}, - io::{Io, IoRaw}, - irq::{self, IrqRequest}, + error::{ + from_result, + to_result, // + }, + prelude::*, str::CStr, - sync::aref::ARef, types::Opaque, - ThisModule, + ThisModule, // }; use core::{ marker::PhantomData, - ops::Deref, - ptr::{addr_of_mut, NonNull}, + mem::offset_of, + ptr::{ + addr_of_mut, + NonNull, // + }, }; -use kernel::prelude::*; mod id; +mod io; +mod irq; -pub use self::id::{Class, ClassMask, Vendor}; +pub use self::id::{ + Class, + ClassMask, + Vendor, // +}; +pub use self::io::Bar; +pub use self::irq::{ + IrqType, + IrqTypes, + IrqVector, // +}; /// An adapter for the registration of PCI drivers. pub struct Adapter<T: Driver>(T); @@ -78,9 +97,9 @@ impl<T: Driver + 'static> Adapter<T> { let info = T::ID_TABLE.info(id.index()); from_result(|| { - let data = T::probe(pdev, info)?; + let data = T::probe(pdev, info); - pdev.as_ref().set_drvdata(data); + pdev.as_ref().set_drvdata(data)?; Ok(0) }) } @@ -95,7 +114,7 @@ impl<T: Driver + 'static> Adapter<T> { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called // and stored a `Pin<KBox<T>>`. - let data = unsafe { pdev.as_ref().drvdata_obtain::<Pin<KBox<T>>>() }; + let data = unsafe { pdev.as_ref().drvdata_obtain::<T>() }; T::unbind(pdev, data.as_ref()); } @@ -249,7 +268,7 @@ macro_rules! pci_device_table { /// fn probe( /// _pdev: &pci::Device<Core>, /// _id_info: &Self::IdInfo, -/// ) -> Result<Pin<KBox<Self>>> { +/// ) -> impl PinInit<Self, Error> { /// Err(ENODEV) /// } /// } @@ -272,7 +291,7 @@ pub trait Driver: Send { /// /// Called when a new pci device is added or discovered. Implementers should /// attempt to initialize the device here. - fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>; + fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> impl PinInit<Self, Error>; /// PCI driver unbind. /// @@ -305,112 +324,6 @@ pub struct Device<Ctx: device::DeviceContext = device::Normal>( PhantomData<Ctx>, ); -/// A PCI BAR to perform I/O-Operations on. -/// -/// # Invariants -/// -/// `Bar` always holds an `IoRaw` inststance that holds a valid pointer to the start of the I/O -/// memory mapped PCI bar and its size. -pub struct Bar<const SIZE: usize = 0> { - pdev: ARef<Device>, - io: IoRaw<SIZE>, - num: i32, -} - -impl<const SIZE: usize> Bar<SIZE> { - fn new(pdev: &Device, num: u32, name: &CStr) -> Result<Self> { - let len = pdev.resource_len(num)?; - if len == 0 { - return Err(ENOMEM); - } - - // Convert to `i32`, since that's what all the C bindings use. - let num = i32::try_from(num)?; - - // SAFETY: - // `pdev` is valid by the invariants of `Device`. - // `num` is checked for validity by a previous call to `Device::resource_len`. - // `name` is always valid. - let ret = unsafe { bindings::pci_request_region(pdev.as_raw(), num, name.as_char_ptr()) }; - if ret != 0 { - return Err(EBUSY); - } - - // SAFETY: - // `pdev` is valid by the invariants of `Device`. - // `num` is checked for validity by a previous call to `Device::resource_len`. - // `name` is always valid. - let ioptr: usize = unsafe { bindings::pci_iomap(pdev.as_raw(), num, 0) } as usize; - if ioptr == 0 { - // SAFETY: - // `pdev` valid by the invariants of `Device`. - // `num` is checked for validity by a previous call to `Device::resource_len`. - unsafe { bindings::pci_release_region(pdev.as_raw(), num) }; - return Err(ENOMEM); - } - - let io = match IoRaw::new(ioptr, len as usize) { - Ok(io) => io, - Err(err) => { - // SAFETY: - // `pdev` is valid by the invariants of `Device`. - // `ioptr` is guaranteed to be the start of a valid I/O mapped memory region. - // `num` is checked for validity by a previous call to `Device::resource_len`. - unsafe { Self::do_release(pdev, ioptr, num) }; - return Err(err); - } - }; - - Ok(Bar { - pdev: pdev.into(), - io, - num, - }) - } - - /// # Safety - /// - /// `ioptr` must be a valid pointer to the memory mapped PCI bar number `num`. - unsafe fn do_release(pdev: &Device, ioptr: usize, num: i32) { - // SAFETY: - // `pdev` is valid by the invariants of `Device`. - // `ioptr` is valid by the safety requirements. - // `num` is valid by the safety requirements. - unsafe { - bindings::pci_iounmap(pdev.as_raw(), ioptr as *mut c_void); - bindings::pci_release_region(pdev.as_raw(), num); - } - } - - fn release(&self) { - // SAFETY: The safety requirements are guaranteed by the type invariant of `self.pdev`. - unsafe { Self::do_release(&self.pdev, self.io.addr(), self.num) }; - } -} - -impl Bar { - #[inline] - fn index_is_valid(index: u32) -> bool { - // A `struct pci_dev` owns an array of resources with at most `PCI_NUM_RESOURCES` entries. - index < bindings::PCI_NUM_RESOURCES - } -} - -impl<const SIZE: usize> Drop for Bar<SIZE> { - fn drop(&mut self) { - self.release(); - } -} - -impl<const SIZE: usize> Deref for Bar<SIZE> { - type Target = Io<SIZE>; - - fn deref(&self) -> &Self::Target { - // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. - unsafe { Io::from_raw(&self.io) } - } -} - impl<Ctx: device::DeviceContext> Device<Ctx> { #[inline] fn as_raw(&self) -> *mut bindings::pci_dev { @@ -484,7 +397,7 @@ impl Device { unsafe { (*self.as_raw()).subsystem_device } } - /// Returns the start of the given PCI bar resource. + /// Returns the start of the given PCI BAR resource. pub fn resource_start(&self, bar: u32) -> Result<bindings::resource_size_t> { if !Bar::index_is_valid(bar) { return Err(EINVAL); @@ -496,7 +409,7 @@ impl Device { Ok(unsafe { bindings::pci_resource_start(self.as_raw(), bar.try_into()?) }) } - /// Returns the size of the given PCI bar resource. + /// Returns the size of the given PCI BAR resource. pub fn resource_len(&self, bar: u32) -> Result<bindings::resource_size_t> { if !Bar::index_is_valid(bar) { return Err(EINVAL); @@ -516,68 +429,6 @@ impl Device { } } -impl Device<device::Bound> { - /// Mapps an entire PCI-BAR after performing a region-request on it. I/O operation bound checks - /// can be performed on compile time for offsets (plus the requested type size) < SIZE. - pub fn iomap_region_sized<'a, const SIZE: usize>( - &'a self, - bar: u32, - name: &'a CStr, - ) -> impl PinInit<Devres<Bar<SIZE>>, Error> + 'a { - Devres::new(self.as_ref(), Bar::<SIZE>::new(self, bar, name)) - } - - /// Mapps an entire PCI-BAR after performing a region-request on it. - pub fn iomap_region<'a>( - &'a self, - bar: u32, - name: &'a CStr, - ) -> impl PinInit<Devres<Bar>, Error> + 'a { - self.iomap_region_sized::<0>(bar, name) - } - - /// Returns an [`IrqRequest`] for the IRQ vector at the given index, if any. - pub fn irq_vector(&self, index: u32) -> Result<IrqRequest<'_>> { - // SAFETY: `self.as_raw` returns a valid pointer to a `struct pci_dev`. - let irq = unsafe { crate::bindings::pci_irq_vector(self.as_raw(), index) }; - if irq < 0 { - return Err(crate::error::Error::from_errno(irq)); - } - // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. - Ok(unsafe { IrqRequest::new(self.as_ref(), irq as u32) }) - } - - /// Returns a [`kernel::irq::Registration`] for the IRQ vector at the given - /// index. - pub fn request_irq<'a, T: crate::irq::Handler + 'static>( - &'a self, - index: u32, - flags: irq::Flags, - name: &'static CStr, - handler: impl PinInit<T, Error> + 'a, - ) -> Result<impl PinInit<irq::Registration<T>, Error> + 'a> { - let request = self.irq_vector(index)?; - - Ok(irq::Registration::<T>::new(request, flags, name, handler)) - } - - /// Returns a [`kernel::irq::ThreadedRegistration`] for the IRQ vector at - /// the given index. - pub fn request_threaded_irq<'a, T: crate::irq::ThreadedHandler + 'static>( - &'a self, - index: u32, - flags: irq::Flags, - name: &'static CStr, - handler: impl PinInit<T, Error> + 'a, - ) -> Result<impl PinInit<irq::ThreadedRegistration<T>, Error> + 'a> { - let request = self.irq_vector(index)?; - - Ok(irq::ThreadedRegistration::<T>::new( - request, flags, name, handler, - )) - } -} - impl Device<device::Core> { /// Enable memory resources for this device. pub fn enable_device_mem(&self) -> Result { @@ -593,6 +444,12 @@ impl Device<device::Core> { } } +// SAFETY: `pci::Device` is a transparent wrapper of `struct pci_dev`. +// The offset is guaranteed to point to a valid device field inside `pci::Device`. +unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Device<Ctx> { + const OFFSET: usize = offset_of!(bindings::pci_dev, dev); +} + // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic // argument. kernel::impl_device_context_deref!(unsafe { Device }); diff --git a/rust/kernel/pci/id.rs b/rust/kernel/pci/id.rs index 5f5d59ff49fc..c09125946d9e 100644 --- a/rust/kernel/pci/id.rs +++ b/rust/kernel/pci/id.rs @@ -4,7 +4,11 @@ //! //! This module contains PCI class codes, Vendor IDs, and supporting types. -use crate::{bindings, error::code::EINVAL, error::Error, fmt, prelude::*}; +use crate::{ + bindings, + fmt, + prelude::*, // +}; /// PCI device class codes. /// diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs new file mode 100644 index 000000000000..0d55c3139b6f --- /dev/null +++ b/rust/kernel/pci/io.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! PCI memory-mapped I/O infrastructure. + +use super::Device; +use crate::{ + bindings, + device, + devres::Devres, + io::{ + Io, + IoRaw, // + }, + prelude::*, + sync::aref::ARef, // +}; +use core::ops::Deref; + +/// A PCI BAR to perform I/O-Operations on. +/// +/// # Invariants +/// +/// `Bar` always holds an `IoRaw` inststance that holds a valid pointer to the start of the I/O +/// memory mapped PCI BAR and its size. +pub struct Bar<const SIZE: usize = 0> { + pdev: ARef<Device>, + io: IoRaw<SIZE>, + num: i32, +} + +impl<const SIZE: usize> Bar<SIZE> { + pub(super) fn new(pdev: &Device, num: u32, name: &CStr) -> Result<Self> { + let len = pdev.resource_len(num)?; + if len == 0 { + return Err(ENOMEM); + } + + // Convert to `i32`, since that's what all the C bindings use. + let num = i32::try_from(num)?; + + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + // `name` is always valid. + let ret = unsafe { bindings::pci_request_region(pdev.as_raw(), num, name.as_char_ptr()) }; + if ret != 0 { + return Err(EBUSY); + } + + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + // `name` is always valid. + let ioptr: usize = unsafe { bindings::pci_iomap(pdev.as_raw(), num, 0) } as usize; + if ioptr == 0 { + // SAFETY: + // `pdev` valid by the invariants of `Device`. + // `num` is checked for validity by a previous call to `Device::resource_len`. + unsafe { bindings::pci_release_region(pdev.as_raw(), num) }; + return Err(ENOMEM); + } + + let io = match IoRaw::new(ioptr, len as usize) { + Ok(io) => io, + Err(err) => { + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `ioptr` is guaranteed to be the start of a valid I/O mapped memory region. + // `num` is checked for validity by a previous call to `Device::resource_len`. + unsafe { Self::do_release(pdev, ioptr, num) }; + return Err(err); + } + }; + + Ok(Bar { + pdev: pdev.into(), + io, + num, + }) + } + + /// # Safety + /// + /// `ioptr` must be a valid pointer to the memory mapped PCI BAR number `num`. + unsafe fn do_release(pdev: &Device, ioptr: usize, num: i32) { + // SAFETY: + // `pdev` is valid by the invariants of `Device`. + // `ioptr` is valid by the safety requirements. + // `num` is valid by the safety requirements. + unsafe { + bindings::pci_iounmap(pdev.as_raw(), ioptr as *mut c_void); + bindings::pci_release_region(pdev.as_raw(), num); + } + } + + fn release(&self) { + // SAFETY: The safety requirements are guaranteed by the type invariant of `self.pdev`. + unsafe { Self::do_release(&self.pdev, self.io.addr(), self.num) }; + } +} + +impl Bar { + #[inline] + pub(super) fn index_is_valid(index: u32) -> bool { + // A `struct pci_dev` owns an array of resources with at most `PCI_NUM_RESOURCES` entries. + index < bindings::PCI_NUM_RESOURCES + } +} + +impl<const SIZE: usize> Drop for Bar<SIZE> { + fn drop(&mut self) { + self.release(); + } +} + +impl<const SIZE: usize> Deref for Bar<SIZE> { + type Target = Io<SIZE>; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. + unsafe { Io::from_raw(&self.io) } + } +} + +impl Device<device::Bound> { + /// Maps an entire PCI BAR after performing a region-request on it. I/O operation bound checks + /// can be performed on compile time for offsets (plus the requested type size) < SIZE. + pub fn iomap_region_sized<'a, const SIZE: usize>( + &'a self, + bar: u32, + name: &'a CStr, + ) -> impl PinInit<Devres<Bar<SIZE>>, Error> + 'a { + Devres::new(self.as_ref(), Bar::<SIZE>::new(self, bar, name)) + } + + /// Maps an entire PCI BAR after performing a region-request on it. + pub fn iomap_region<'a>( + &'a self, + bar: u32, + name: &'a CStr, + ) -> impl PinInit<Devres<Bar>, Error> + 'a { + self.iomap_region_sized::<0>(bar, name) + } +} diff --git a/rust/kernel/pci/irq.rs b/rust/kernel/pci/irq.rs new file mode 100644 index 000000000000..d9230e105541 --- /dev/null +++ b/rust/kernel/pci/irq.rs @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! PCI interrupt infrastructure. + +use super::Device; +use crate::{ + bindings, + device, + device::Bound, + devres, + error::to_result, + irq::{ + self, + IrqRequest, // + }, + prelude::*, + str::CStr, + sync::aref::ARef, // +}; +use core::ops::RangeInclusive; + +/// IRQ type flags for PCI interrupt allocation. +#[derive(Debug, Clone, Copy)] +pub enum IrqType { + /// INTx interrupts. + Intx, + /// Message Signaled Interrupts (MSI). + Msi, + /// Extended Message Signaled Interrupts (MSI-X). + MsiX, +} + +impl IrqType { + /// Convert to the corresponding kernel flags. + const fn as_raw(self) -> u32 { + match self { + IrqType::Intx => bindings::PCI_IRQ_INTX, + IrqType::Msi => bindings::PCI_IRQ_MSI, + IrqType::MsiX => bindings::PCI_IRQ_MSIX, + } + } +} + +/// Set of IRQ types that can be used for PCI interrupt allocation. +#[derive(Debug, Clone, Copy, Default)] +pub struct IrqTypes(u32); + +impl IrqTypes { + /// Create a set containing all IRQ types (MSI-X, MSI, and INTx). + pub const fn all() -> Self { + Self(bindings::PCI_IRQ_ALL_TYPES) + } + + /// Build a set of IRQ types. + /// + /// # Examples + /// + /// ```ignore + /// // Create a set with only MSI and MSI-X (no INTx interrupts). + /// let msi_only = IrqTypes::default() + /// .with(IrqType::Msi) + /// .with(IrqType::MsiX); + /// ``` + pub const fn with(self, irq_type: IrqType) -> Self { + Self(self.0 | irq_type.as_raw()) + } + + /// Get the raw flags value. + const fn as_raw(self) -> u32 { + self.0 + } +} + +/// Represents an allocated IRQ vector for a specific PCI device. +/// +/// This type ties an IRQ vector to the device it was allocated for, +/// ensuring the vector is only used with the correct device. +#[derive(Clone, Copy)] +pub struct IrqVector<'a> { + dev: &'a Device<Bound>, + index: u32, +} + +impl<'a> IrqVector<'a> { + /// Creates a new [`IrqVector`] for the given device and index. + /// + /// # Safety + /// + /// - `index` must be a valid IRQ vector index for `dev`. + /// - `dev` must point to a [`Device`] that has successfully allocated IRQ vectors. + unsafe fn new(dev: &'a Device<Bound>, index: u32) -> Self { + Self { dev, index } + } + + /// Returns the raw vector index. + fn index(&self) -> u32 { + self.index + } +} + +impl<'a> TryInto<IrqRequest<'a>> for IrqVector<'a> { + type Error = Error; + + fn try_into(self) -> Result<IrqRequest<'a>> { + // SAFETY: `self.as_raw` returns a valid pointer to a `struct pci_dev`. + let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), self.index()) }; + if irq < 0 { + return Err(crate::error::Error::from_errno(irq)); + } + // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. + Ok(unsafe { IrqRequest::new(self.dev.as_ref(), irq as u32) }) + } +} + +/// Represents an IRQ vector allocation for a PCI device. +/// +/// This type ensures that IRQ vectors are properly allocated and freed by +/// tying the allocation to the lifetime of this registration object. +/// +/// # Invariants +/// +/// The [`Device`] has successfully allocated IRQ vectors. +struct IrqVectorRegistration { + dev: ARef<Device>, +} + +impl IrqVectorRegistration { + /// Allocate and register IRQ vectors for the given PCI device. + /// + /// Allocates IRQ vectors and registers them with devres for automatic cleanup. + /// Returns a range of valid IRQ vectors. + fn register<'a>( + dev: &'a Device<Bound>, + min_vecs: u32, + max_vecs: u32, + irq_types: IrqTypes, + ) -> Result<RangeInclusive<IrqVector<'a>>> { + // SAFETY: + // - `dev.as_raw()` is guaranteed to be a valid pointer to a `struct pci_dev` + // by the type invariant of `Device`. + // - `pci_alloc_irq_vectors` internally validates all other parameters + // and returns error codes. + let ret = unsafe { + bindings::pci_alloc_irq_vectors(dev.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) + }; + + to_result(ret)?; + let count = ret as u32; + + // SAFETY: + // - `pci_alloc_irq_vectors` returns the number of allocated vectors on success. + // - Vectors are 0-based, so valid indices are [0, count-1]. + // - `pci_alloc_irq_vectors` guarantees `count >= min_vecs > 0`, so both `0` and + // `count - 1` are valid IRQ vector indices for `dev`. + let range = unsafe { IrqVector::new(dev, 0)..=IrqVector::new(dev, count - 1) }; + + // INVARIANT: The IRQ vector allocation for `dev` above was successful. + let irq_vecs = Self { dev: dev.into() }; + devres::register(dev.as_ref(), irq_vecs, GFP_KERNEL)?; + + Ok(range) + } +} + +impl Drop for IrqVectorRegistration { + fn drop(&mut self) { + // SAFETY: + // - By the type invariant, `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + // - `self.dev` has successfully allocated IRQ vectors. + unsafe { bindings::pci_free_irq_vectors(self.dev.as_raw()) }; + } +} + +impl Device<device::Bound> { + /// Returns a [`kernel::irq::Registration`] for the given IRQ vector. + pub fn request_irq<'a, T: crate::irq::Handler + 'static>( + &'a self, + vector: IrqVector<'a>, + flags: irq::Flags, + name: &'static CStr, + handler: impl PinInit<T, Error> + 'a, + ) -> impl PinInit<irq::Registration<T>, Error> + 'a { + pin_init::pin_init_scope(move || { + let request = vector.try_into()?; + + Ok(irq::Registration::<T>::new(request, flags, name, handler)) + }) + } + + /// Returns a [`kernel::irq::ThreadedRegistration`] for the given IRQ vector. + pub fn request_threaded_irq<'a, T: crate::irq::ThreadedHandler + 'static>( + &'a self, + vector: IrqVector<'a>, + flags: irq::Flags, + name: &'static CStr, + handler: impl PinInit<T, Error> + 'a, + ) -> impl PinInit<irq::ThreadedRegistration<T>, Error> + 'a { + pin_init::pin_init_scope(move || { + let request = vector.try_into()?; + + Ok(irq::ThreadedRegistration::<T>::new( + request, flags, name, handler, + )) + }) + } + + /// Allocate IRQ vectors for this PCI device with automatic cleanup. + /// + /// Allocates between `min_vecs` and `max_vecs` interrupt vectors for the device. + /// The allocation will use MSI-X, MSI, or INTx interrupts based on the `irq_types` + /// parameter and hardware capabilities. When multiple types are specified, the kernel + /// will try them in order of preference: MSI-X first, then MSI, then INTx interrupts. + /// + /// The allocated vectors are automatically freed when the device is unbound, using the + /// devres (device resource management) system. + /// + /// # Arguments + /// + /// * `min_vecs` - Minimum number of vectors required. + /// * `max_vecs` - Maximum number of vectors to allocate. + /// * `irq_types` - Types of interrupts that can be used. + /// + /// # Returns + /// + /// Returns a range of IRQ vectors that were successfully allocated, or an error if the + /// allocation fails or cannot meet the minimum requirement. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{ device::Bound, pci}; + /// # fn no_run(dev: &pci::Device<Bound>) -> Result { + /// // Allocate using any available interrupt type in the order mentioned above. + /// let vectors = dev.alloc_irq_vectors(1, 32, pci::IrqTypes::all())?; + /// + /// // Allocate MSI or MSI-X only (no INTx interrupts). + /// let msi_only = pci::IrqTypes::default() + /// .with(pci::IrqType::Msi) + /// .with(pci::IrqType::MsiX); + /// let vectors = dev.alloc_irq_vectors(4, 16, msi_only)?; + /// # Ok(()) + /// # } + /// ``` + pub fn alloc_irq_vectors( + &self, + min_vecs: u32, + max_vecs: u32, + irq_types: IrqTypes, + ) -> Result<RangeInclusive<IrqVector<'_>>> { + IrqVectorRegistration::register(self, min_vecs, max_vecs, irq_types) + } +} diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 7205fe3416d3..ed889f079cab 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -19,6 +19,7 @@ use crate::{ use core::{ marker::PhantomData, + mem::offset_of, ptr::{addr_of_mut, NonNull}, }; @@ -74,9 +75,9 @@ impl<T: Driver + 'static> Adapter<T> { let info = <Self as driver::Adapter>::id_info(pdev.as_ref()); from_result(|| { - let data = T::probe(pdev, info)?; + let data = T::probe(pdev, info); - pdev.as_ref().set_drvdata(data); + pdev.as_ref().set_drvdata(data)?; Ok(0) }) } @@ -91,7 +92,7 @@ impl<T: Driver + 'static> Adapter<T> { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called // and stored a `Pin<KBox<T>>`. - let data = unsafe { pdev.as_ref().drvdata_obtain::<Pin<KBox<T>>>() }; + let data = unsafe { pdev.as_ref().drvdata_obtain::<T>() }; T::unbind(pdev, data.as_ref()); } @@ -166,7 +167,7 @@ macro_rules! module_platform_driver { /// fn probe( /// _pdev: &platform::Device<Core>, /// _id_info: Option<&Self::IdInfo>, -/// ) -> Result<Pin<KBox<Self>>> { +/// ) -> impl PinInit<Self, Error> { /// Err(ENODEV) /// } /// } @@ -190,8 +191,10 @@ pub trait Driver: Send { /// /// Called when a new platform device is added or discovered. /// Implementers should attempt to initialize the device here. - fn probe(dev: &Device<device::Core>, id_info: Option<&Self::IdInfo>) - -> Result<Pin<KBox<Self>>>; + fn probe( + dev: &Device<device::Core>, + id_info: Option<&Self::IdInfo>, + ) -> impl PinInit<Self, Error>; /// Platform driver unbind. /// @@ -285,6 +288,12 @@ impl Device<Bound> { } } +// SAFETY: `platform::Device` is a transparent wrapper of `struct platform_device`. +// The offset is guaranteed to point to a valid device field inside `platform::Device`. +unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Device<Ctx> { + const OFFSET: usize = offset_of!(bindings::platform_device, dev); +} + macro_rules! define_irq_accessor_by_index { ( $(#[$meta:meta])* $fn_name:ident, @@ -299,15 +308,17 @@ macro_rules! define_irq_accessor_by_index { index: u32, name: &'static CStr, handler: impl PinInit<T, Error> + 'a, - ) -> Result<impl PinInit<irq::$reg_type<T>, Error> + 'a> { - let request = self.$request_fn(index)?; - - Ok(irq::$reg_type::<T>::new( - request, - flags, - name, - handler, - )) + ) -> impl PinInit<irq::$reg_type<T>, Error> + 'a { + pin_init::pin_init_scope(move || { + let request = self.$request_fn(index)?; + + Ok(irq::$reg_type::<T>::new( + request, + flags, + name, + handler, + )) + }) } }; } @@ -323,18 +334,20 @@ macro_rules! define_irq_accessor_by_name { pub fn $fn_name<'a, T: irq::$handler_trait + 'static>( &'a self, flags: irq::Flags, - irq_name: &CStr, + irq_name: &'a CStr, name: &'static CStr, handler: impl PinInit<T, Error> + 'a, - ) -> Result<impl PinInit<irq::$reg_type<T>, Error> + 'a> { - let request = self.$request_fn(irq_name)?; - - Ok(irq::$reg_type::<T>::new( - request, - flags, - name, - handler, - )) + ) -> impl PinInit<irq::$reg_type<T>, Error> + 'a { + pin_init::pin_init_scope(move || { + let request = self.$request_fn(irq_name)?; + + Ok(irq::$reg_type::<T>::new( + request, + flags, + name, + handler, + )) + }) } }; } diff --git a/rust/kernel/scatterlist.rs b/rust/kernel/scatterlist.rs index 9709dff60b5a..196fdb9a75e7 100644 --- a/rust/kernel/scatterlist.rs +++ b/rust/kernel/scatterlist.rs @@ -35,7 +35,7 @@ use crate::{ device::{Bound, Device}, devres::Devres, dma, error, - io::resource::ResourceSize, + io::ResourceSize, page, prelude::*, types::{ARef, Opaque}, diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 6fcc9d47f12e..fa87779d2253 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -15,6 +15,8 @@ use core::{ pub use crate::prelude::CStr; +pub mod parse_int; + /// Byte string without UTF-8 validity guarantee. #[repr(transparent)] pub struct BStr([u8]); diff --git a/rust/kernel/str/parse_int.rs b/rust/kernel/str/parse_int.rs new file mode 100644 index 000000000000..48eb4c202984 --- /dev/null +++ b/rust/kernel/str/parse_int.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Integer parsing functions. +//! +//! Integer parsing functions for parsing signed and unsigned integers +//! potentially prefixed with `0x`, `0o`, or `0b`. + +use crate::prelude::*; +use crate::str::BStr; +use core::ops::Deref; + +// Make `FromStrRadix` a public type with a private name. This seals +// `ParseInt`, that is, prevents downstream users from implementing the +// trait. +mod private { + use crate::prelude::*; + use crate::str::BStr; + + /// Trait that allows parsing a [`&BStr`] to an integer with a radix. + pub trait FromStrRadix: Sized { + /// Parse `src` to [`Self`] using radix `radix`. + fn from_str_radix(src: &BStr, radix: u32) -> Result<Self>; + + /// Tries to convert `value` into [`Self`] and negates the resulting value. + fn from_u64_negated(value: u64) -> Result<Self>; + } +} + +/// Extract the radix from an integer literal optionally prefixed with +/// one of `0x`, `0X`, `0o`, `0O`, `0b`, `0B`, `0`. +fn strip_radix(src: &BStr) -> (u32, &BStr) { + match src.deref() { + [b'0', b'x' | b'X', rest @ ..] => (16, rest.as_ref()), + [b'0', b'o' | b'O', rest @ ..] => (8, rest.as_ref()), + [b'0', b'b' | b'B', rest @ ..] => (2, rest.as_ref()), + // NOTE: We are including the leading zero to be able to parse + // literal `0` here. If we removed it as a radix prefix, we would + // not be able to parse `0`. + [b'0', ..] => (8, src), + _ => (10, src), + } +} + +/// Trait for parsing string representations of integers. +/// +/// Strings beginning with `0x`, `0o`, or `0b` are parsed as hex, octal, or +/// binary respectively. Strings beginning with `0` otherwise are parsed as +/// octal. Anything else is parsed as decimal. A leading `+` or `-` is also +/// permitted. Any string parsed by [`kstrtol()`] or [`kstrtoul()`] will be +/// successfully parsed. +/// +/// [`kstrtol()`]: https://docs.kernel.org/core-api/kernel-api.html#c.kstrtol +/// [`kstrtoul()`]: https://docs.kernel.org/core-api/kernel-api.html#c.kstrtoul +/// +/// # Examples +/// +/// ``` +/// # use kernel::str::parse_int::ParseInt; +/// # use kernel::b_str; +/// +/// assert_eq!(Ok(0u8), u8::from_str(b_str!("0"))); +/// +/// assert_eq!(Ok(0xa2u8), u8::from_str(b_str!("0xa2"))); +/// assert_eq!(Ok(-0xa2i32), i32::from_str(b_str!("-0xa2"))); +/// +/// assert_eq!(Ok(-0o57i8), i8::from_str(b_str!("-0o57"))); +/// assert_eq!(Ok(0o57i8), i8::from_str(b_str!("057"))); +/// +/// assert_eq!(Ok(0b1001i16), i16::from_str(b_str!("0b1001"))); +/// assert_eq!(Ok(-0b1001i16), i16::from_str(b_str!("-0b1001"))); +/// +/// assert_eq!(Ok(127i8), i8::from_str(b_str!("127"))); +/// assert!(i8::from_str(b_str!("128")).is_err()); +/// assert_eq!(Ok(-128i8), i8::from_str(b_str!("-128"))); +/// assert!(i8::from_str(b_str!("-129")).is_err()); +/// assert_eq!(Ok(255u8), u8::from_str(b_str!("255"))); +/// assert!(u8::from_str(b_str!("256")).is_err()); +/// ``` +pub trait ParseInt: private::FromStrRadix + TryFrom<u64> { + /// Parse a string according to the description in [`Self`]. + fn from_str(src: &BStr) -> Result<Self> { + match src.deref() { + [b'-', rest @ ..] => { + let (radix, digits) = strip_radix(rest.as_ref()); + // 2's complement values range from -2^(b-1) to 2^(b-1)-1. + // So if we want to parse negative numbers as positive and + // later multiply by -1, we have to parse into a larger + // integer. We choose `u64` as sufficiently large. + // + // NOTE: 128 bit integers are not available on all + // platforms, hence the choice of 64 bits. + let val = + u64::from_str_radix(core::str::from_utf8(digits).map_err(|_| EINVAL)?, radix) + .map_err(|_| EINVAL)?; + Self::from_u64_negated(val) + } + _ => { + let (radix, digits) = strip_radix(src); + Self::from_str_radix(digits, radix).map_err(|_| EINVAL) + } + } + } +} + +macro_rules! impl_parse_int { + ($($ty:ty),*) => { + $( + impl private::FromStrRadix for $ty { + fn from_str_radix(src: &BStr, radix: u32) -> Result<Self> { + <$ty>::from_str_radix(core::str::from_utf8(src).map_err(|_| EINVAL)?, radix) + .map_err(|_| EINVAL) + } + + fn from_u64_negated(value: u64) -> Result<Self> { + const ABS_MIN: u64 = { + #[allow(unused_comparisons)] + if <$ty>::MIN < 0 { + 1u64 << (<$ty>::BITS - 1) + } else { + 0 + } + }; + + if value > ABS_MIN { + return Err(EINVAL); + } + + if value == ABS_MIN { + return Ok(<$ty>::MIN); + } + + // SAFETY: The above checks guarantee that `value` fits into `Self`: + // - if `Self` is unsigned, then `ABS_MIN == 0` and thus we have returned above + // (either `EINVAL` or `MIN`). + // - if `Self` is signed, then we have that `0 <= value < ABS_MIN`. And since + // `ABS_MIN - 1` fits into `Self` by construction, `value` also does. + let value: Self = unsafe { value.try_into().unwrap_unchecked() }; + + Ok((!value).wrapping_add(1)) + } + } + + impl ParseInt for $ty {} + )* + }; +} + +impl_parse_int![i8, u8, i16, u16, i32, u32, i64, u64, isize, usize]; diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index c94753d6413e..5df87e2bd212 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -20,6 +20,7 @@ mod locked_by; pub mod poll; pub mod rcu; mod refcount; +mod set_once; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use completion::Completion; @@ -29,6 +30,7 @@ pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; pub use locked_by::LockedBy; pub use refcount::Refcount; +pub use set_once::SetOnce; /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. #[repr(transparent)] diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs index 3afc376be42d..4aebeacb961a 100644 --- a/rust/kernel/sync/atomic.rs +++ b/rust/kernel/sync/atomic.rs @@ -22,6 +22,7 @@ mod predefine; pub use internal::AtomicImpl; pub use ordering::{Acquire, Full, Relaxed, Release}; + pub(crate) use internal::{AtomicArithmeticOps, AtomicBasicOps, AtomicExchangeOps}; use crate::build_error; diff --git a/rust/kernel/sync/set_once.rs b/rust/kernel/sync/set_once.rs new file mode 100644 index 000000000000..bdba601807d8 --- /dev/null +++ b/rust/kernel/sync/set_once.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A container that can be initialized at most once. + +use super::atomic::{ + ordering::{Acquire, Relaxed, Release}, + Atomic, +}; +use core::{cell::UnsafeCell, mem::MaybeUninit}; + +/// A container that can be populated at most once. Thread safe. +/// +/// Once the a [`SetOnce`] is populated, it remains populated by the same object for the +/// lifetime `Self`. +/// +/// # Invariants +/// +/// - `init` may only increase in value. +/// - `init` may only assume values in the range `0..=2`. +/// - `init == 0` if and only if `value` is uninitialized. +/// - `init == 1` if and only if there is exactly one thread with exclusive +/// access to `self.value`. +/// - `init == 2` if and only if `value` is initialized and valid for shared +/// access. +/// +/// # Example +/// +/// ``` +/// # use kernel::sync::SetOnce; +/// let value = SetOnce::new(); +/// assert_eq!(None, value.as_ref()); +/// +/// let status = value.populate(42u8); +/// assert_eq!(true, status); +/// assert_eq!(Some(&42u8), value.as_ref()); +/// assert_eq!(Some(42u8), value.copy()); +/// +/// let status = value.populate(101u8); +/// assert_eq!(false, status); +/// assert_eq!(Some(&42u8), value.as_ref()); +/// assert_eq!(Some(42u8), value.copy()); +/// ``` +pub struct SetOnce<T> { + init: Atomic<u32>, + value: UnsafeCell<MaybeUninit<T>>, +} + +impl<T> Default for SetOnce<T> { + fn default() -> Self { + Self::new() + } +} + +impl<T> SetOnce<T> { + /// Create a new [`SetOnce`]. + /// + /// The returned instance will be empty. + pub const fn new() -> Self { + // INVARIANT: The container is empty and we initialize `init` to `0`. + Self { + value: UnsafeCell::new(MaybeUninit::uninit()), + init: Atomic::new(0), + } + } + + /// Get a reference to the contained object. + /// + /// Returns [`None`] if this [`SetOnce`] is empty. + pub fn as_ref(&self) -> Option<&T> { + if self.init.load(Acquire) == 2 { + // SAFETY: By the type invariants of `Self`, `self.init == 2` means that `self.value` + // is initialized and valid for shared access. + Some(unsafe { &*self.value.get().cast() }) + } else { + None + } + } + + /// Populate the [`SetOnce`]. + /// + /// Returns `true` if the [`SetOnce`] was successfully populated. + pub fn populate(&self, value: T) -> bool { + // INVARIANT: If the swap succeeds: + // - We increase `init`. + // - We write the valid value `1` to `init`. + // - Only one thread can succeed in this write, so we have exclusive access after the + // write. + if let Ok(0) = self.init.cmpxchg(0, 1, Relaxed) { + // SAFETY: By the type invariants of `Self`, the fact that we succeeded in writing `1` + // to `self.init` means we obtained exclusive access to `self.value`. + unsafe { core::ptr::write(self.value.get().cast(), value) }; + // INVARIANT: + // - We increase `init`. + // - We write the valid value `2` to `init`. + // - We release our exclusive access to `self.value` and it is now valid for shared + // access. + self.init.store(2, Release); + true + } else { + false + } + } + + /// Get a copy of the contained object. + /// + /// Returns [`None`] if the [`SetOnce`] is empty. + pub fn copy(&self) -> Option<T> + where + T: Copy, + { + self.as_ref().copied() + } +} + +impl<T> Drop for SetOnce<T> { + fn drop(&mut self) { + if *self.init.get_mut() == 2 { + let value = self.value.get_mut(); + // SAFETY: By the type invariants of `Self`, `self.init == 2` means that `self.value` + // contains a valid value. We have exclusive access, as we hold a `mut` reference to + // `self`. + unsafe { value.assume_init_drop() }; + } + } +} diff --git a/rust/kernel/time/delay.rs b/rust/kernel/time/delay.rs index eb8838da62bc..b5b1b42797a0 100644 --- a/rust/kernel/time/delay.rs +++ b/rust/kernel/time/delay.rs @@ -47,3 +47,40 @@ pub fn fsleep(delta: Delta) { bindings::fsleep(delta.as_micros_ceil() as c_ulong) } } + +/// Inserts a delay based on microseconds with busy waiting. +/// +/// Equivalent to the C side [`udelay()`], which delays in microseconds. +/// +/// `delta` must be within `[0, MAX_UDELAY_MS]` in milliseconds; +/// otherwise, it is erroneous behavior. That is, it is considered a bug to +/// call this function with an out-of-range value. +/// +/// The behavior above differs from the C side [`udelay()`] for which out-of-range +/// values could lead to an overflow and unexpected behavior. +/// +/// [`udelay()`]: https://docs.kernel.org/timers/delay_sleep_functions.html#c.udelay +pub fn udelay(delta: Delta) { + const MAX_UDELAY_DELTA: Delta = Delta::from_millis(bindings::MAX_UDELAY_MS as i64); + + debug_assert!(delta.as_nanos() >= 0); + debug_assert!(delta <= MAX_UDELAY_DELTA); + + let delta = if (Delta::ZERO..=MAX_UDELAY_DELTA).contains(&delta) { + delta + } else { + MAX_UDELAY_DELTA + }; + + // SAFETY: It is always safe to call `udelay()` with any duration. + // Note that the kernel is compiled with `-fno-strict-overflow` + // so any out-of-range value could lead to unexpected behavior + // but won't lead to undefined behavior. + unsafe { + // Convert the duration to microseconds and round up to preserve + // the guarantee; `udelay()` inserts a delay for at least + // the provided duration, but that it may delay for longer + // under some circumstances. + bindings::udelay(delta.as_micros_ceil() as c_ulong) + } +} diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index a8fb4764185a..f989539a31b4 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -9,6 +9,7 @@ use crate::{ bindings, error::Result, ffi::{c_char, c_void}, + fs::file, prelude::*, transmute::{AsBytes, FromBytes}, }; @@ -287,6 +288,48 @@ impl UserSliceReader { self.read_raw(out) } + /// Reads raw data from the user slice into a kernel buffer partially. + /// + /// This is the same as [`Self::read_slice`] but considers the given `offset` into `out` and + /// truncates the read to the boundaries of `self` and `out`. + /// + /// On success, returns the number of bytes read. + pub fn read_slice_partial(&mut self, out: &mut [u8], offset: usize) -> Result<usize> { + let end = offset.saturating_add(self.len()).min(out.len()); + + let Some(dst) = out.get_mut(offset..end) else { + return Ok(0); + }; + + self.read_slice(dst)?; + Ok(dst.len()) + } + + /// Reads raw data from the user slice into a kernel buffer partially. + /// + /// This is the same as [`Self::read_slice_partial`] but updates the given [`file::Offset`] by + /// the number of bytes read. + /// + /// This is equivalent to C's `simple_write_to_buffer()`. + /// + /// On success, returns the number of bytes read. + pub fn read_slice_file(&mut self, out: &mut [u8], offset: &mut file::Offset) -> Result<usize> { + if offset.is_negative() { + return Err(EINVAL); + } + + let Ok(offset_index) = (*offset).try_into() else { + return Ok(0); + }; + + let read = self.read_slice_partial(out, offset_index)?; + + // OVERFLOW: `offset + read <= data.len() <= isize::MAX <= Offset::MAX` + *offset += read as i64; + + Ok(read) + } + /// Reads a value of the specified type. /// /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of @@ -438,6 +481,48 @@ impl UserSliceWriter { Ok(()) } + /// Writes raw data to this user pointer from a kernel buffer partially. + /// + /// This is the same as [`Self::write_slice`] but considers the given `offset` into `data` and + /// truncates the write to the boundaries of `self` and `data`. + /// + /// On success, returns the number of bytes written. + pub fn write_slice_partial(&mut self, data: &[u8], offset: usize) -> Result<usize> { + let end = offset.saturating_add(self.len()).min(data.len()); + + let Some(src) = data.get(offset..end) else { + return Ok(0); + }; + + self.write_slice(src)?; + Ok(src.len()) + } + + /// Writes raw data to this user pointer from a kernel buffer partially. + /// + /// This is the same as [`Self::write_slice_partial`] but updates the given [`file::Offset`] by + /// the number of bytes written. + /// + /// This is equivalent to C's `simple_read_from_buffer()`. + /// + /// On success, returns the number of bytes written. + pub fn write_slice_file(&mut self, data: &[u8], offset: &mut file::Offset) -> Result<usize> { + if offset.is_negative() { + return Err(EINVAL); + } + + let Ok(offset_index) = (*offset).try_into() else { + return Ok(0); + }; + + let written = self.write_slice_partial(data, offset_index)?; + + // OVERFLOW: `offset + written <= data.len() <= isize::MAX <= Offset::MAX` + *offset += written as i64; + + Ok(written) + } + /// Writes the provided Rust value to this userspace pointer. /// /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs index 14ddb711bab3..d10b65e9fb6a 100644 --- a/rust/kernel/usb.rs +++ b/rust/kernel/usb.rs @@ -15,7 +15,14 @@ use crate::{ types::{AlwaysRefCounted, Opaque}, ThisModule, }; -use core::{marker::PhantomData, mem::MaybeUninit, ptr::NonNull}; +use core::{ + marker::PhantomData, + mem::{ + offset_of, + MaybeUninit, // + }, + ptr::NonNull, +}; /// An adapter for the registration of USB drivers. pub struct Adapter<T: Driver>(T); @@ -67,10 +74,10 @@ impl<T: Driver + 'static> Adapter<T> { let id = unsafe { &*id.cast::<DeviceId>() }; let info = T::ID_TABLE.info(id.index()); - let data = T::probe(intf, id, info)?; + let data = T::probe(intf, id, info); let dev: &device::Device<device::CoreInternal> = intf.as_ref(); - dev.set_drvdata(data); + dev.set_drvdata(data)?; Ok(0) }) } @@ -87,7 +94,7 @@ impl<T: Driver + 'static> Adapter<T> { // SAFETY: `disconnect_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called // and stored a `Pin<KBox<T>>`. - let data = unsafe { dev.drvdata_obtain::<Pin<KBox<T>>>() }; + let data = unsafe { dev.drvdata_obtain::<T>() }; T::disconnect(intf, data.as_ref()); } @@ -270,7 +277,7 @@ macro_rules! usb_device_table { /// _interface: &usb::Interface<Core>, /// _id: &usb::DeviceId, /// _info: &Self::IdInfo, -/// ) -> Result<Pin<KBox<Self>>> { +/// ) -> impl PinInit<Self, Error> { /// Err(ENODEV) /// } /// @@ -292,7 +299,7 @@ pub trait Driver { interface: &Interface<device::Core>, id: &DeviceId, id_info: &Self::IdInfo, - ) -> Result<Pin<KBox<Self>>>; + ) -> impl PinInit<Self, Error>; /// USB driver disconnect. /// @@ -324,6 +331,12 @@ impl<Ctx: device::DeviceContext> Interface<Ctx> { } } +// SAFETY: `usb::Interface` is a transparent wrapper of `struct usb_interface`. +// The offset is guaranteed to point to a valid device field inside `usb::Interface`. +unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Interface<Ctx> { + const OFFSET: usize = offset_of!(bindings::usb_interface, dev); +} + // SAFETY: `Interface` is a transparent wrapper of a type that doesn't depend on // `Interface`'s generic argument. kernel::impl_device_context_deref!(unsafe { Interface }); |