diff options
Diffstat (limited to 'rust')
107 files changed, 5971 insertions, 1522 deletions
diff --git a/rust/Makefile b/rust/Makefile index 27dec7904c3a..4263462b8470 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -34,6 +34,9 @@ obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.o obj-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.o always-$(subst y,$(CONFIG_RUST),$(CONFIG_JUMP_LABEL)) += kernel/generated_arch_static_branch_asm.rs +ifndef CONFIG_UML +always-$(subst y,$(CONFIG_RUST),$(CONFIG_BUG)) += kernel/generated_arch_warn_asm.rs kernel/generated_arch_reachable_asm.rs +endif # Avoids running `$(RUSTC)` when it may not be available. ifdef CONFIG_RUST @@ -194,6 +197,7 @@ quiet_cmd_rustdoc_test = RUSTDOC T $< RUST_MODFILE=test.rs \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) --test $(rust_common_flags) \ + -Zcrate-attr='feature(used_with_arg)' \ @$(objtree)/include/generated/rustc_cfg \ $(rustc_target_flags) $(rustdoc_test_target_flags) \ $(rustdoc_test_quiet) \ @@ -540,5 +544,10 @@ $(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/build_error.o $(obj)/pin_init.o \ ifdef CONFIG_JUMP_LABEL $(obj)/kernel.o: $(obj)/kernel/generated_arch_static_branch_asm.rs endif +ifndef CONFIG_UML +ifdef CONFIG_BUG +$(obj)/kernel.o: $(obj)/kernel/generated_arch_warn_asm.rs $(obj)/kernel/generated_arch_reachable_asm.rs +endif +endif endif # CONFIG_RUST diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index bc494745f67b..84d60635e8a9 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -28,6 +28,7 @@ */ #include <linux/hrtimer_types.h> +#include <linux/acpi.h> #include <drm/drm_device.h> #include <drm/drm_drv.h> #include <drm/drm_file.h> @@ -39,6 +40,7 @@ #include <linux/blk_types.h> #include <linux/blkdev.h> #include <linux/clk.h> +#include <linux/completion.h> #include <linux/configfs.h> #include <linux/cpu.h> #include <linux/cpufreq.h> @@ -51,6 +53,7 @@ #include <linux/file.h> #include <linux/firmware.h> #include <linux/fs.h> +#include <linux/ioport.h> #include <linux/jiffies.h> #include <linux/jump_label.h> #include <linux/mdio.h> @@ -64,6 +67,7 @@ #include <linux/poll.h> #include <linux/property.h> #include <linux/refcount.h> +#include <linux/regulator/consumer.h> #include <linux/sched.h> #include <linux/security.h> #include <linux/slab.h> diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index a08eb5518cac..474cc98c48a3 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -25,6 +25,9 @@ )] #[allow(dead_code)] +#[allow(clippy::cast_lossless)] +#[allow(clippy::ptr_as_ptr)] +#[allow(clippy::ref_as_ptr)] #[allow(clippy::undocumented_unsafe_blocks)] #[cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] mod bindings_raw { diff --git a/rust/helpers/auxiliary.c b/rust/helpers/auxiliary.c index 0db3860d774e..8b5e0fea4493 100644 --- a/rust/helpers/auxiliary.c +++ b/rust/helpers/auxiliary.c @@ -2,16 +2,6 @@ #include <linux/auxiliary_bus.h> -void rust_helper_auxiliary_set_drvdata(struct auxiliary_device *adev, void *data) -{ - auxiliary_set_drvdata(adev, data); -} - -void *rust_helper_auxiliary_get_drvdata(struct auxiliary_device *adev) -{ - return auxiliary_get_drvdata(adev); -} - void rust_helper_auxiliary_device_uninit(struct auxiliary_device *adev) { return auxiliary_device_uninit(adev); diff --git a/rust/helpers/bug.c b/rust/helpers/bug.c index e2d13babc737..a62c96f507d1 100644 --- a/rust/helpers/bug.c +++ b/rust/helpers/bug.c @@ -6,3 +6,8 @@ __noreturn void rust_helper_BUG(void) { BUG(); } + +bool rust_helper_WARN_ON(bool cond) +{ + return WARN_ON(cond); +} diff --git a/rust/helpers/completion.c b/rust/helpers/completion.c new file mode 100644 index 000000000000..b2443262a2ae --- /dev/null +++ b/rust/helpers/completion.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/completion.h> + +void rust_helper_init_completion(struct completion *x) +{ + init_completion(x); +} diff --git a/rust/helpers/cpu.c b/rust/helpers/cpu.c new file mode 100644 index 000000000000..824e0adb19d4 --- /dev/null +++ b/rust/helpers/cpu.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/smp.h> + +unsigned int rust_helper_raw_smp_processor_id(void) +{ + return raw_smp_processor_id(); +} diff --git a/rust/helpers/device.c b/rust/helpers/device.c index b2135c6686b0..9a4316bafedf 100644 --- a/rust/helpers/device.c +++ b/rust/helpers/device.c @@ -8,3 +8,20 @@ int rust_helper_devm_add_action(struct device *dev, { return devm_add_action(dev, action, data); } + +int rust_helper_devm_add_action_or_reset(struct device *dev, + void (*action)(void *), + void *data) +{ + return devm_add_action_or_reset(dev, action, data); +} + +void *rust_helper_dev_get_drvdata(const struct device *dev) +{ + return dev_get_drvdata(dev); +} + +void rust_helper_dev_set_drvdata(struct device *dev, void *data) +{ + dev_set_drvdata(dev, data); +} diff --git a/rust/helpers/dma.c b/rust/helpers/dma.c index df8b8a77355a..6e741c197242 100644 --- a/rust/helpers/dma.c +++ b/rust/helpers/dma.c @@ -14,3 +14,8 @@ void rust_helper_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, { dma_free_attrs(dev, size, cpu_addr, dma_handle, attrs); } + +int rust_helper_dma_set_mask_and_coherent(struct device *dev, u64 mask) +{ + return dma_set_mask_and_coherent(dev, mask); +} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 0f1b5d115985..7cf7fe95e41d 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -13,6 +13,8 @@ #include "build_assert.c" #include "build_bug.c" #include "clk.c" +#include "completion.c" +#include "cpu.c" #include "cpufreq.c" #include "cpumask.c" #include "cred.c" @@ -26,19 +28,24 @@ #include "kunit.c" #include "mm.c" #include "mutex.c" +#include "of.c" #include "page.c" -#include "platform.c" #include "pci.c" #include "pid_namespace.c" +#include "platform.c" +#include "poll.c" +#include "property.c" #include "rbtree.c" #include "rcu.c" #include "refcount.c" +#include "regulator.c" #include "security.c" #include "signal.c" #include "slab.c" #include "spinlock.c" #include "sync.c" #include "task.c" +#include "time.c" #include "uaccess.c" #include "vmalloc.c" #include "wait.c" diff --git a/rust/helpers/io.c b/rust/helpers/io.c index 15ea187c5466..c475913c69e6 100644 --- a/rust/helpers/io.c +++ b/rust/helpers/io.c @@ -1,12 +1,18 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/io.h> +#include <linux/ioport.h> void __iomem *rust_helper_ioremap(phys_addr_t offset, size_t size) { return ioremap(offset, size); } +void __iomem *rust_helper_ioremap_np(phys_addr_t offset, size_t size) +{ + return ioremap_np(offset, size); +} + void rust_helper_iounmap(void __iomem *addr) { iounmap(addr); @@ -99,3 +105,38 @@ void rust_helper_writeq_relaxed(u64 value, void __iomem *addr) writeq_relaxed(value, addr); } #endif + +resource_size_t rust_helper_resource_size(struct resource *res) +{ + return resource_size(res); +} + +struct resource *rust_helper_request_mem_region(resource_size_t start, + resource_size_t n, + const char *name) +{ + return request_mem_region(start, n, name); +} + +void rust_helper_release_mem_region(resource_size_t start, resource_size_t n) +{ + release_mem_region(start, n); +} + +struct resource *rust_helper_request_region(resource_size_t start, + resource_size_t n, const char *name) +{ + return request_region(start, n, name); +} + +struct resource *rust_helper_request_muxed_region(resource_size_t start, + resource_size_t n, + const char *name) +{ + return request_muxed_region(start, n, name); +} + +void rust_helper_release_region(resource_size_t start, resource_size_t n) +{ + release_region(start, n); +} diff --git a/rust/helpers/of.c b/rust/helpers/of.c new file mode 100644 index 000000000000..86b51167c913 --- /dev/null +++ b/rust/helpers/of.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/of.h> + +bool rust_helper_is_of_node(const struct fwnode_handle *fwnode) +{ + return is_of_node(fwnode); +} diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c index cd0e6bf2cc4d..ef9cb38c81a6 100644 --- a/rust/helpers/pci.c +++ b/rust/helpers/pci.c @@ -2,16 +2,6 @@ #include <linux/pci.h> -void rust_helper_pci_set_drvdata(struct pci_dev *pdev, void *data) -{ - pci_set_drvdata(pdev, data); -} - -void *rust_helper_pci_get_drvdata(struct pci_dev *pdev) -{ - return pci_get_drvdata(pdev); -} - resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev, int bar) { return pci_resource_len(pdev, bar); diff --git a/rust/helpers/platform.c b/rust/helpers/platform.c index 82171233d12f..1ce89c1a36f7 100644 --- a/rust/helpers/platform.c +++ b/rust/helpers/platform.c @@ -2,16 +2,6 @@ #include <linux/platform_device.h> -void *rust_helper_platform_get_drvdata(const struct platform_device *pdev) -{ - return platform_get_drvdata(pdev); -} - -void rust_helper_platform_set_drvdata(struct platform_device *pdev, void *data) -{ - platform_set_drvdata(pdev, data); -} - bool rust_helper_dev_is_platform(const struct device *dev) { return dev_is_platform(dev); diff --git a/rust/helpers/poll.c b/rust/helpers/poll.c new file mode 100644 index 000000000000..7e5b1751c2d5 --- /dev/null +++ b/rust/helpers/poll.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/export.h> +#include <linux/poll.h> + +void rust_helper_poll_wait(struct file *filp, wait_queue_head_t *wait_address, + poll_table *p) +{ + poll_wait(filp, wait_address, p); +} diff --git a/rust/helpers/property.c b/rust/helpers/property.c new file mode 100644 index 000000000000..08f68e2dac4a --- /dev/null +++ b/rust/helpers/property.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/property.h> + +void rust_helper_fwnode_handle_put(struct fwnode_handle *fwnode) +{ + fwnode_handle_put(fwnode); +} diff --git a/rust/helpers/regulator.c b/rust/helpers/regulator.c new file mode 100644 index 000000000000..cd8b7ba648ee --- /dev/null +++ b/rust/helpers/regulator.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/regulator/consumer.h> + +#ifndef CONFIG_REGULATOR + +void rust_helper_regulator_put(struct regulator *regulator) +{ + regulator_put(regulator); +} + +int rust_helper_regulator_set_voltage(struct regulator *regulator, int min_uV, + int max_uV) +{ + return regulator_set_voltage(regulator, min_uV, max_uV); +} + +int rust_helper_regulator_get_voltage(struct regulator *regulator) +{ + return regulator_get_voltage(regulator); +} + +struct regulator *rust_helper_regulator_get(struct device *dev, const char *id) +{ + return regulator_get(dev, id); +} + +int rust_helper_regulator_enable(struct regulator *regulator) +{ + return regulator_enable(regulator); +} + +int rust_helper_regulator_disable(struct regulator *regulator) +{ + return regulator_disable(regulator); +} + +int rust_helper_regulator_is_enabled(struct regulator *regulator) +{ + return regulator_is_enabled(regulator); +} + +#endif diff --git a/rust/helpers/task.c b/rust/helpers/task.c index 31c33ea2dce6..2c85bbc2727e 100644 --- a/rust/helpers/task.c +++ b/rust/helpers/task.c @@ -1,7 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/kernel.h> #include <linux/sched/task.h> +void rust_helper_might_resched(void) +{ + might_resched(); +} + struct task_struct *rust_helper_get_current(void) { return current; diff --git a/rust/helpers/time.c b/rust/helpers/time.c new file mode 100644 index 000000000000..a318e9fa4408 --- /dev/null +++ b/rust/helpers/time.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/delay.h> +#include <linux/ktime.h> +#include <linux/timekeeping.h> + +void rust_helper_fsleep(unsigned long usecs) +{ + fsleep(usecs); +} + +ktime_t rust_helper_ktime_get_real(void) +{ + return ktime_get_real(); +} + +ktime_t rust_helper_ktime_get_boottime(void) +{ + return ktime_get_boottime(); +} + +ktime_t rust_helper_ktime_get_clocktai(void) +{ + return ktime_get_clocktai(); +} + +s64 rust_helper_ktime_to_us(const ktime_t kt) +{ + return ktime_to_us(kt); +} + +s64 rust_helper_ktime_to_ms(const ktime_t kt) +{ + return ktime_to_ms(kt); +} diff --git a/rust/kernel/.gitignore b/rust/kernel/.gitignore index 6ba39a178f30..f636ad95aaf3 100644 --- a/rust/kernel/.gitignore +++ b/rust/kernel/.gitignore @@ -1,3 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 /generated_arch_static_branch_asm.rs +/generated_arch_warn_asm.rs +/generated_arch_reachable_asm.rs diff --git a/rust/kernel/acpi.rs b/rust/kernel/acpi.rs new file mode 100644 index 000000000000..7ae317368b00 --- /dev/null +++ b/rust/kernel/acpi.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Advanced Configuration and Power Interface abstractions. + +use crate::{ + bindings, + device_id::{RawDeviceId, RawDeviceIdIndex}, + prelude::*, +}; + +/// IdTable type for ACPI drivers. +pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; + +/// An ACPI device id. +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::acpi_device_id); + +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `acpi_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::acpi_device_id; +} + +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { + const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::acpi_device_id, driver_data); + + fn index(&self) -> usize { + self.0.driver_data + } +} + +impl DeviceId { + const ACPI_ID_LEN: usize = 16; + + /// Create a new device id from an ACPI 'id' string. + #[inline(always)] + pub const fn new(id: &'static CStr) -> Self { + build_assert!( + id.len_with_nul() <= Self::ACPI_ID_LEN, + "ID exceeds 16 bytes" + ); + let src = id.as_bytes_with_nul(); + // Replace with `bindings::acpi_device_id::default()` once stabilized for `const`. + // SAFETY: FFI type is valid to be zero-initialized. + let mut acpi: bindings::acpi_device_id = unsafe { core::mem::zeroed() }; + let mut i = 0; + while i < src.len() { + acpi.id[i] = src[i]; + i += 1; + } + + Self(acpi) + } +} + +/// Create an ACPI `IdTable` with an "alias" for modpost. +#[macro_export] +macro_rules! acpi_device_table { + ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { + const $table_name: $crate::device_id::IdArray< + $crate::acpi::DeviceId, + $id_info_type, + { $table_data.len() }, + > = $crate::device_id::IdArray::new($table_data); + + $crate::module_device_table!("acpi", $module_table_name, $table_name); + }; +} diff --git a/rust/kernel/alloc/allocator_test.rs b/rust/kernel/alloc/allocator_test.rs index d19c06ef0498..a3074480bd8d 100644 --- a/rust/kernel/alloc/allocator_test.rs +++ b/rust/kernel/alloc/allocator_test.rs @@ -82,7 +82,7 @@ unsafe impl Allocator for Cmalloc { // SAFETY: Returns either NULL or a pointer to a memory allocation that satisfies or // exceeds the given size and alignment requirements. - let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) } as *mut u8; + let dst = unsafe { libc_aligned_alloc(layout.align(), layout.size()) }.cast::<u8>(); let dst = NonNull::new(dst).ok_or(AllocError)?; if flags.contains(__GFP_ZERO) { diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index c386ff771d50..856d05aa60f1 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -6,6 +6,7 @@ use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; use super::{AllocError, Allocator, Flags}; use core::alloc::Layout; +use core::borrow::{Borrow, BorrowMut}; use core::fmt; use core::marker::PhantomData; use core::mem::ManuallyDrop; @@ -15,6 +16,7 @@ use core::pin::Pin; use core::ptr::NonNull; use core::result::Result; +use crate::ffi::c_void; use crate::init::InPlaceInit; use crate::types::ForeignOwnable; use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption}; @@ -398,70 +400,74 @@ where } } -// SAFETY: The `into_foreign` function returns a pointer that is well-aligned. +// SAFETY: The pointer returned by `into_foreign` comes from a well aligned +// pointer to `T`. unsafe impl<T: 'static, A> ForeignOwnable for Box<T, A> where A: Allocator, { - type PointedTo = T; + const FOREIGN_ALIGN: usize = core::mem::align_of::<T>(); type Borrowed<'a> = &'a T; type BorrowedMut<'a> = &'a mut T; - fn into_foreign(self) -> *mut Self::PointedTo { - Box::into_raw(self) + fn into_foreign(self) -> *mut c_void { + Box::into_raw(self).cast() } - unsafe fn from_foreign(ptr: *mut Self::PointedTo) -> Self { + unsafe fn from_foreign(ptr: *mut c_void) -> Self { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. - unsafe { Box::from_raw(ptr) } + unsafe { Box::from_raw(ptr.cast()) } } - unsafe fn borrow<'a>(ptr: *mut Self::PointedTo) -> &'a T { + unsafe fn borrow<'a>(ptr: *mut c_void) -> &'a T { // SAFETY: The safety requirements of this method ensure that the object remains alive and // immutable for the duration of 'a. - unsafe { &*ptr } + unsafe { &*ptr.cast() } } - unsafe fn borrow_mut<'a>(ptr: *mut Self::PointedTo) -> &'a mut T { + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a mut T { + let ptr = ptr.cast(); // SAFETY: The safety requirements of this method ensure that the pointer is valid and that // nothing else will access the value for the duration of 'a. unsafe { &mut *ptr } } } -// SAFETY: The `into_foreign` function returns a pointer that is well-aligned. +// SAFETY: The pointer returned by `into_foreign` comes from a well aligned +// pointer to `T`. unsafe impl<T: 'static, A> ForeignOwnable for Pin<Box<T, A>> where A: Allocator, { - type PointedTo = T; + const FOREIGN_ALIGN: usize = core::mem::align_of::<T>(); type Borrowed<'a> = Pin<&'a T>; type BorrowedMut<'a> = Pin<&'a mut T>; - fn into_foreign(self) -> *mut Self::PointedTo { + fn into_foreign(self) -> *mut c_void { // SAFETY: We are still treating the box as pinned. - Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) + Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }).cast() } - unsafe fn from_foreign(ptr: *mut Self::PointedTo) -> Self { + unsafe fn from_foreign(ptr: *mut c_void) -> Self { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. - unsafe { Pin::new_unchecked(Box::from_raw(ptr)) } + unsafe { Pin::new_unchecked(Box::from_raw(ptr.cast())) } } - unsafe fn borrow<'a>(ptr: *mut Self::PointedTo) -> Pin<&'a T> { + unsafe fn borrow<'a>(ptr: *mut c_void) -> Pin<&'a T> { // SAFETY: The safety requirements for this function ensure that the object is still alive, // so it is safe to dereference the raw pointer. // The safety requirements of `from_foreign` also ensure that the object remains alive for // the lifetime of the returned value. - let r = unsafe { &*ptr }; + let r = unsafe { &*ptr.cast() }; // SAFETY: This pointer originates from a `Pin<Box<T>>`. unsafe { Pin::new_unchecked(r) } } - unsafe fn borrow_mut<'a>(ptr: *mut Self::PointedTo) -> Pin<&'a mut T> { + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Pin<&'a mut T> { + let ptr = ptr.cast(); // SAFETY: The safety requirements for this function ensure that the object is still alive, // so it is safe to dereference the raw pointer. // The safety requirements of `from_foreign` also ensure that the object remains alive for @@ -499,6 +505,62 @@ where } } +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// # use kernel::alloc::KBox; +/// struct Foo<B: Borrow<u32>>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `KBox`. +/// let owned_kbox = Foo(KBox::new(1, GFP_KERNEL)?); +/// +/// let i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&i); +/// # Ok::<(), Error>(()) +/// ``` +impl<T, A> Borrow<T> for Box<T, A> +where + T: ?Sized, + A: Allocator, +{ + fn borrow(&self) -> &T { + self.deref() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::BorrowMut; +/// # use kernel::alloc::KBox; +/// struct Foo<B: BorrowMut<u32>>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `KBox`. +/// let owned_kbox = Foo(KBox::new(1, GFP_KERNEL)?); +/// +/// let mut i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&mut i); +/// # Ok::<(), Error>(()) +/// ``` +impl<T, A> BorrowMut<T> for Box<T, A> +where + T: ?Sized, + A: Allocator, +{ + fn borrow_mut(&mut self) -> &mut T { + self.deref_mut() + } +} + impl<T, A> fmt::Display for Box<T, A> where T: ?Sized + fmt::Display, diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 1a0dd852a468..3c72e0bdddb8 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -8,6 +8,7 @@ use super::{ AllocError, Allocator, Box, Flags, }; use core::{ + borrow::{Borrow, BorrowMut}, fmt, marker::PhantomData, mem::{ManuallyDrop, MaybeUninit}, @@ -288,7 +289,7 @@ where // - `self.len` is smaller than `self.capacity` by the type invariant and hence, the // resulting pointer is guaranteed to be part of the same allocated object. // - `self.len` can not overflow `isize`. - let ptr = unsafe { self.as_mut_ptr().add(self.len) } as *mut MaybeUninit<T>; + let ptr = unsafe { self.as_mut_ptr().add(self.len) }.cast::<MaybeUninit<T>>(); // SAFETY: The memory between `self.len` and `self.capacity` is guaranteed to be allocated // and valid, but uninitialized. @@ -847,11 +848,11 @@ where // - `ptr` points to memory with at least a size of `size_of::<T>() * len`, // - all elements within `b` are initialized values of `T`, // - `len` does not exceed `isize::MAX`. - unsafe { Vec::from_raw_parts(ptr as _, len, len) } + unsafe { Vec::from_raw_parts(ptr.cast(), len, len) } } } -impl<T> Default for KVec<T> { +impl<T, A: Allocator> Default for Vec<T, A> { #[inline] fn default() -> Self { Self::new() @@ -890,6 +891,58 @@ where } } +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// struct Foo<B: Borrow<[u32]>>(B); +/// +/// // Owned array. +/// let owned_array = Foo([1, 2, 3]); +/// +/// // Owned vector. +/// let owned_vec = Foo(KVec::from_elem(0, 3, GFP_KERNEL)?); +/// +/// let arr = [1, 2, 3]; +/// // Borrowed slice from `arr`. +/// let borrowed_slice = Foo(&arr[..]); +/// # Ok::<(), Error>(()) +/// ``` +impl<T, A> Borrow<[T]> for Vec<T, A> +where + A: Allocator, +{ + fn borrow(&self) -> &[T] { + self.as_slice() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::BorrowMut; +/// struct Foo<B: BorrowMut<[u32]>>(B); +/// +/// // Owned array. +/// let owned_array = Foo([1, 2, 3]); +/// +/// // Owned vector. +/// let owned_vec = Foo(KVec::from_elem(0, 3, GFP_KERNEL)?); +/// +/// let mut arr = [1, 2, 3]; +/// // Borrowed slice from `arr`. +/// let borrowed_slice = Foo(&mut arr[..]); +/// # Ok::<(), Error>(()) +/// ``` +impl<T, A> BorrowMut<[T]> for Vec<T, A> +where + A: Allocator, +{ + fn borrow_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + impl<T: Eq, A> Eq for Vec<T, A> where A: Allocator {} impl<T, I: SliceIndex<[T]>, A> Index<I> for Vec<T, A> diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index d2cfe1eeefb6..4749fb6bffef 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -6,12 +6,11 @@ use crate::{ bindings, container_of, device, - device_id::RawDeviceId, + device_id::{RawDeviceId, RawDeviceIdIndex}, driver, - error::{to_result, Result}, + error::{from_result, to_result, Result}, prelude::*, - str::CStr, - types::{ForeignOwnable, Opaque}, + types::Opaque, ThisModule, }; use core::{ @@ -61,37 +60,32 @@ impl<T: Driver + 'static> Adapter<T> { // `struct auxiliary_device`. // // INVARIANT: `adev` is valid for the duration of `probe_callback()`. - let adev = unsafe { &*adev.cast::<Device<device::Core>>() }; + let adev = unsafe { &*adev.cast::<Device<device::CoreInternal>>() }; // SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id` // and does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::<DeviceId>() }; let info = T::ID_TABLE.info(id.index()); - match T::probe(adev, info) { - Ok(data) => { - // Let the `struct auxiliary_device` own a reference of the driver's private data. - // SAFETY: By the type invariant `adev.as_raw` returns a valid pointer to a - // `struct auxiliary_device`. - unsafe { - bindings::auxiliary_set_drvdata(adev.as_raw(), data.into_foreign().cast()) - }; - } - Err(err) => return Error::to_errno(err), - } + from_result(|| { + let data = T::probe(adev, info)?; - 0 + adev.as_ref().set_drvdata(data); + Ok(0) + }) } extern "C" fn remove_callback(adev: *mut bindings::auxiliary_device) { - // SAFETY: The auxiliary bus only ever calls the remove callback with a valid pointer to a + // SAFETY: The auxiliary bus only ever calls the probe callback with a valid pointer to a // `struct auxiliary_device`. - let ptr = unsafe { bindings::auxiliary_get_drvdata(adev) }; + // + // INVARIANT: `adev` is valid for the duration of `probe_callback()`. + let adev = unsafe { &*adev.cast::<Device<device::CoreInternal>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to - // `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized - // `KBox<T>` pointer created through `KBox::into_foreign`. - drop(unsafe { KBox::<T>::from_foreign(ptr.cast()) }); + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<T>>`. + drop(unsafe { adev.as_ref().drvdata_obtain::<Pin<KBox<T>>>() }); } } @@ -140,13 +134,14 @@ impl DeviceId { } } -// SAFETY: -// * `DeviceId` is a `#[repr(transparent)`] wrapper of `auxiliary_device_id` and does not add -// additional invariants, so it's safe to transmute to `RawType`. -// * `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `auxiliary_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. unsafe impl RawDeviceId for DeviceId { type RawType = bindings::auxiliary_device_id; +} +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::auxiliary_device_id, driver_data); @@ -276,7 +271,7 @@ impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> { let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; // SAFETY: `dev` points to a valid `struct device`. - unsafe { device::Device::as_ref(dev) } + unsafe { device::Device::from_raw(dev) } } } diff --git a/rust/kernel/bits.rs b/rust/kernel/bits.rs new file mode 100644 index 000000000000..553d50265883 --- /dev/null +++ b/rust/kernel/bits.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bit manipulation macros. +//! +//! C header: [`include/linux/bits.h`](srctree/include/linux/bits.h) + +use crate::prelude::*; +use core::ops::RangeInclusive; +use macros::paste; + +macro_rules! impl_bit_fn { + ( + $ty:ty + ) => { + paste! { + /// Computes `1 << n` if `n` is in bounds, i.e.: if `n` is smaller than + /// the maximum number of bits supported by the type. + /// + /// Returns [`None`] otherwise. + #[inline] + pub fn [<checked_bit_ $ty>](n: u32) -> Option<$ty> { + (1 as $ty).checked_shl(n) + } + + /// Computes `1 << n` by performing a compile-time assertion that `n` is + /// in bounds. + /// + /// This version is the default and should be used if `n` is known at + /// compile time. + #[inline] + pub const fn [<bit_ $ty>](n: u32) -> $ty { + build_assert!(n < <$ty>::BITS); + (1 as $ty) << n + } + } + }; +} + +impl_bit_fn!(u64); +impl_bit_fn!(u32); +impl_bit_fn!(u16); +impl_bit_fn!(u8); + +macro_rules! impl_genmask_fn { + ( + $ty:ty, + $(#[$genmask_checked_ex:meta])*, + $(#[$genmask_ex:meta])* + ) => { + paste! { + /// Creates a contiguous bitmask for the given range by validating + /// the range at runtime. + /// + /// Returns [`None`] if the range is invalid, i.e.: if the start is + /// greater than the end or if the range is outside of the + /// representable range for the type. + $(#[$genmask_checked_ex])* + #[inline] + pub fn [<genmask_checked_ $ty>](range: RangeInclusive<u32>) -> Option<$ty> { + let start = *range.start(); + let end = *range.end(); + + if start > end { + return None; + } + + let high = [<checked_bit_ $ty>](end)?; + let low = [<checked_bit_ $ty>](start)?; + Some((high | (high - 1)) & !(low - 1)) + } + + /// Creates a compile-time contiguous bitmask for the given range by + /// performing a compile-time assertion that the range is valid. + /// + /// This version is the default and should be used if the range is known + /// at compile time. + $(#[$genmask_ex])* + #[inline] + pub const fn [<genmask_ $ty>](range: RangeInclusive<u32>) -> $ty { + let start = *range.start(); + let end = *range.end(); + + build_assert!(start <= end); + + let high = [<bit_ $ty>](end); + let low = [<bit_ $ty>](start); + (high | (high - 1)) & !(low - 1) + } + } + }; +} + +impl_genmask_fn!( + u64, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u64; + /// assert_eq!(genmask_checked_u64(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u64(0..=63), Some(u64::MAX)); + /// assert_eq!(genmask_checked_u64(21..=39), Some(0x0000_00ff_ffe0_0000)); + /// + /// // `80` is out of the supported bit range. + /// assert_eq!(genmask_checked_u64(21..=80), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u64(15..=8), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u64; + /// assert_eq!(genmask_u64(21..=39), 0x0000_00ff_ffe0_0000); + /// assert_eq!(genmask_u64(0..=0), 0b1); + /// assert_eq!(genmask_u64(0..=63), u64::MAX); + /// ``` +); + +impl_genmask_fn!( + u32, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u32; + /// assert_eq!(genmask_checked_u32(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u32(0..=31), Some(u32::MAX)); + /// assert_eq!(genmask_checked_u32(21..=31), Some(0xffe0_0000)); + /// + /// // `40` is out of the supported bit range. + /// assert_eq!(genmask_checked_u32(21..=40), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u32(15..=8), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u32; + /// assert_eq!(genmask_u32(21..=31), 0xffe0_0000); + /// assert_eq!(genmask_u32(0..=0), 0b1); + /// assert_eq!(genmask_u32(0..=31), u32::MAX); + /// ``` +); + +impl_genmask_fn!( + u16, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u16; + /// assert_eq!(genmask_checked_u16(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u16(0..=15), Some(u16::MAX)); + /// assert_eq!(genmask_checked_u16(6..=15), Some(0xffc0)); + /// + /// // `20` is out of the supported bit range. + /// assert_eq!(genmask_checked_u16(6..=20), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u16(10..=5), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u16; + /// assert_eq!(genmask_u16(6..=15), 0xffc0); + /// assert_eq!(genmask_u16(0..=0), 0b1); + /// assert_eq!(genmask_u16(0..=15), u16::MAX); + /// ``` +); + +impl_genmask_fn!( + u8, + /// # Examples + /// + /// ``` + /// # #![expect(clippy::reversed_empty_ranges)] + /// # use kernel::bits::genmask_checked_u8; + /// assert_eq!(genmask_checked_u8(0..=0), Some(0b1)); + /// assert_eq!(genmask_checked_u8(0..=7), Some(u8::MAX)); + /// assert_eq!(genmask_checked_u8(6..=7), Some(0xc0)); + /// + /// // `10` is out of the supported bit range. + /// assert_eq!(genmask_checked_u8(6..=10), None); + /// + /// // Invalid range where the start is bigger than the end. + /// assert_eq!(genmask_checked_u8(5..=2), None); + /// ``` + , + /// # Examples + /// + /// ``` + /// # use kernel::bits::genmask_u8; + /// assert_eq!(genmask_u8(6..=7), 0xc0); + /// assert_eq!(genmask_u8(0..=0), 0b1); + /// assert_eq!(genmask_u8(0..=7), u8::MAX); + /// ``` +); diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs index fb0f393c1cea..831445d37181 100644 --- a/rust/kernel/block/mq.rs +++ b/rust/kernel/block/mq.rs @@ -53,7 +53,7 @@ //! [`GenDiskBuilder`]: gen_disk::GenDiskBuilder //! [`GenDiskBuilder::build`]: gen_disk::GenDiskBuilder::build //! -//! # Example +//! # Examples //! //! ```rust //! use kernel::{ diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs index 864ff379dc91..c2b98f507bcb 100644 --- a/rust/kernel/block/mq/operations.rs +++ b/rust/kernel/block/mq/operations.rs @@ -101,7 +101,7 @@ impl<T: Operations> OperationsVTable<T> { if let Err(e) = ret { e.to_blk_status() } else { - bindings::BLK_STS_OK as _ + bindings::BLK_STS_OK as bindings::blk_status_t } } diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs index 4a5b7ec914ef..fefd394f064a 100644 --- a/rust/kernel/block/mq/request.rs +++ b/rust/kernel/block/mq/request.rs @@ -69,7 +69,7 @@ impl<T: Operations> Request<T> { // INVARIANT: By the safety requirements of this function, invariants are upheld. // SAFETY: By the safety requirement of this function, we own a // reference count that we can pass to `ARef`. - unsafe { ARef::from_raw(NonNull::new_unchecked(ptr as *const Self as *mut Self)) } + unsafe { ARef::from_raw(NonNull::new_unchecked(ptr.cast())) } } /// Notify the block layer that a request is going to be processed now. @@ -125,7 +125,12 @@ impl<T: Operations> Request<T> { // success of the call to `try_set_end` guarantees that there are no // `ARef`s pointing to this request. Therefore it is safe to hand it // back to the block layer. - unsafe { bindings::blk_mq_end_request(request_ptr, bindings::BLK_STS_OK as _) }; + unsafe { + bindings::blk_mq_end_request( + request_ptr, + bindings::BLK_STS_OK as bindings::blk_status_t, + ) + }; Ok(()) } @@ -155,7 +160,7 @@ impl<T: Operations> Request<T> { // the private data associated with this request is initialized and // valid. The existence of `&self` guarantees that the private data is // valid as a shared reference. - unsafe { Self::wrapper_ptr(self as *const Self as *mut Self).as_ref() } + unsafe { Self::wrapper_ptr(core::ptr::from_ref(self).cast_mut()).as_ref() } } } diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs index bcf4214ad149..c3cf56d52bee 100644 --- a/rust/kernel/block/mq/tag_set.rs +++ b/rust/kernel/block/mq/tag_set.rs @@ -9,7 +9,7 @@ use core::pin::Pin; use crate::{ bindings, block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations}, - error, + error::{self, Result}, prelude::try_pin_init, types::Opaque, }; @@ -41,7 +41,7 @@ impl<T: Operations> TagSet<T> { // SAFETY: `blk_mq_tag_set` only contains integers and pointers, which // all are allowed to be 0. let tag_set: bindings::blk_mq_tag_set = unsafe { core::mem::zeroed() }; - let tag_set = core::mem::size_of::<RequestDataWrapper>() + let tag_set: Result<_> = core::mem::size_of::<RequestDataWrapper>() .try_into() .map(|cmd_size| { bindings::blk_mq_tag_set { @@ -56,12 +56,14 @@ impl<T: Operations> TagSet<T> { nr_maps: num_maps, ..tag_set } - }); + }) + .map(Opaque::new) + .map_err(|e| e.into()); try_pin_init!(TagSet { - inner <- PinInit::<_, error::Error>::pin_chain(Opaque::new(tag_set?), |tag_set| { + inner <- tag_set.pin_chain(|tag_set| { // SAFETY: we do not move out of `tag_set`. - let tag_set = unsafe { Pin::get_unchecked_mut(tag_set) }; + let tag_set: &mut Opaque<_> = unsafe { Pin::get_unchecked_mut(tag_set) }; // SAFETY: `tag_set` is a reference to an initialized `blk_mq_tag_set`. error::to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())}) }), diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs new file mode 100644 index 000000000000..36aef43e5ebe --- /dev/null +++ b/rust/kernel/bug.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024, 2025 FUJITA Tomonori <fujita.tomonori@gmail.com> + +//! Support for BUG and WARN functionality. +//! +//! C header: [`include/asm-generic/bug.h`](srctree/include/asm-generic/bug.h) + +#[macro_export] +#[doc(hidden)] +#[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] +#[cfg(CONFIG_DEBUG_BUGVERBOSE)] +macro_rules! warn_flags { + ($flags:expr) => { + const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags; + const _FILE: &[u8] = file!().as_bytes(); + // Plus one for null-terminator. + static FILE: [u8; _FILE.len() + 1] = { + let mut bytes = [0; _FILE.len() + 1]; + let mut i = 0; + while i < _FILE.len() { + bytes[i] = _FILE[i]; + i += 1; + } + bytes + }; + + // SAFETY: + // - `file`, `line`, `flags`, and `size` are all compile-time constants or + // symbols, preventing any invalid memory access. + // - The asm block has no side effects and does not modify any registers + // or memory. It is purely for embedding metadata into the ELF section. + unsafe { + $crate::asm!( + concat!( + "/* {size} */", + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_warn_asm.rs")), + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_reachable_asm.rs"))); + file = sym FILE, + line = const line!(), + flags = const FLAGS, + size = const ::core::mem::size_of::<$crate::bindings::bug_entry>(), + ); + } + } +} + +#[macro_export] +#[doc(hidden)] +#[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] +#[cfg(not(CONFIG_DEBUG_BUGVERBOSE))] +macro_rules! warn_flags { + ($flags:expr) => { + const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags; + + // SAFETY: + // - `flags` and `size` are all compile-time constants, preventing + // any invalid memory access. + // - The asm block has no side effects and does not modify any registers + // or memory. It is purely for embedding metadata into the ELF section. + unsafe { + $crate::asm!( + concat!( + "/* {size} */", + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_warn_asm.rs")), + include!(concat!(env!("OBJTREE"), "/rust/kernel/generated_arch_reachable_asm.rs"))); + flags = const FLAGS, + size = const ::core::mem::size_of::<$crate::bindings::bug_entry>(), + ); + } + } +} + +#[macro_export] +#[doc(hidden)] +#[cfg(all(CONFIG_BUG, CONFIG_UML))] +macro_rules! warn_flags { + ($flags:expr) => { + // SAFETY: It is always safe to call `warn_slowpath_fmt()` + // with a valid null-terminated string. + unsafe { + $crate::bindings::warn_slowpath_fmt( + $crate::c_str!(::core::file!()).as_char_ptr(), + line!() as $crate::ffi::c_int, + $flags as $crate::ffi::c_uint, + ::core::ptr::null(), + ); + } + }; +} + +#[macro_export] +#[doc(hidden)] +#[cfg(all(CONFIG_BUG, any(CONFIG_LOONGARCH, CONFIG_ARM)))] +macro_rules! warn_flags { + ($flags:expr) => { + // SAFETY: It is always safe to call `WARN_ON()`. + unsafe { $crate::bindings::WARN_ON(true) } + }; +} + +#[macro_export] +#[doc(hidden)] +#[cfg(not(CONFIG_BUG))] +macro_rules! warn_flags { + ($flags:expr) => {}; +} + +#[doc(hidden)] +pub const fn bugflag_taint(value: u32) -> u32 { + value << 8 +} + +/// Report a warning if `cond` is true and return the condition's evaluation result. +#[macro_export] +macro_rules! warn_on { + ($cond:expr) => {{ + let cond = $cond; + if cond { + const WARN_ON_FLAGS: u32 = $crate::bug::bugflag_taint($crate::bindings::TAINT_WARN); + + $crate::warn_flags!(WARN_ON_FLAGS); + } + cond + }}; +} diff --git a/rust/kernel/clk.rs b/rust/kernel/clk.rs index 6041c6d07527..1e6c8c42fb3a 100644 --- a/rust/kernel/clk.rs +++ b/rust/kernel/clk.rs @@ -12,7 +12,7 @@ use crate::ffi::c_ulong; /// /// Represents a frequency in hertz, wrapping a [`c_ulong`] value. /// -/// ## Examples +/// # Examples /// /// ``` /// use kernel::clk::Hertz; @@ -30,39 +30,43 @@ use crate::ffi::c_ulong; pub struct Hertz(pub c_ulong); impl Hertz { + const KHZ_TO_HZ: c_ulong = 1_000; + const MHZ_TO_HZ: c_ulong = 1_000_000; + const GHZ_TO_HZ: c_ulong = 1_000_000_000; + /// Create a new instance from kilohertz (kHz) - pub fn from_khz(khz: c_ulong) -> Self { - Self(khz * 1_000) + pub const fn from_khz(khz: c_ulong) -> Self { + Self(khz * Self::KHZ_TO_HZ) } /// Create a new instance from megahertz (MHz) - pub fn from_mhz(mhz: c_ulong) -> Self { - Self(mhz * 1_000_000) + pub const fn from_mhz(mhz: c_ulong) -> Self { + Self(mhz * Self::MHZ_TO_HZ) } /// Create a new instance from gigahertz (GHz) - pub fn from_ghz(ghz: c_ulong) -> Self { - Self(ghz * 1_000_000_000) + pub const fn from_ghz(ghz: c_ulong) -> Self { + Self(ghz * Self::GHZ_TO_HZ) } /// Get the frequency in hertz - pub fn as_hz(&self) -> c_ulong { + pub const fn as_hz(&self) -> c_ulong { self.0 } /// Get the frequency in kilohertz - pub fn as_khz(&self) -> c_ulong { - self.0 / 1_000 + pub const fn as_khz(&self) -> c_ulong { + self.0 / Self::KHZ_TO_HZ } /// Get the frequency in megahertz - pub fn as_mhz(&self) -> c_ulong { - self.0 / 1_000_000 + pub const fn as_mhz(&self) -> c_ulong { + self.0 / Self::MHZ_TO_HZ } /// Get the frequency in gigahertz - pub fn as_ghz(&self) -> c_ulong { - self.0 / 1_000_000_000 + pub const fn as_ghz(&self) -> c_ulong { + self.0 / Self::GHZ_TO_HZ } } @@ -95,7 +99,7 @@ mod common_clk { /// Instances of this type are reference-counted. Calling [`Clk::get`] ensures that the /// allocation remains valid for the lifetime of the [`Clk`]. /// - /// ## Examples + /// # Examples /// /// The following example demonstrates how to obtain and configure a clock for a device. /// @@ -132,11 +136,7 @@ mod common_clk { /// /// [`clk_get`]: https://docs.kernel.org/core-api/kernel-api.html#c.clk_get pub fn get(dev: &Device, name: Option<&CStr>) -> Result<Self> { - let con_id = if let Some(name) = name { - name.as_ptr() - } else { - ptr::null() - }; + let con_id = name.map_or(ptr::null(), |n| n.as_ptr()); // SAFETY: It is safe to call [`clk_get`] for a valid device pointer. // @@ -266,7 +266,7 @@ mod common_clk { /// Instances of this type are reference-counted. Calling [`OptionalClk::get`] ensures that the /// allocation remains valid for the lifetime of the [`OptionalClk`]. /// - /// ## Examples + /// # Examples /// /// The following example demonstrates how to obtain and configure an optional clock for a /// device. The code functions correctly whether or not the clock is available. @@ -304,11 +304,7 @@ mod common_clk { /// [`clk_get_optional`]: /// https://docs.kernel.org/core-api/kernel-api.html#c.clk_get_optional pub fn get(dev: &Device, name: Option<&CStr>) -> Result<Self> { - let con_id = if let Some(name) = name { - name.as_ptr() - } else { - ptr::null() - }; + let con_id = name.map_or(ptr::null(), |n| n.as_ptr()); // SAFETY: It is safe to call [`clk_get_optional`] for a valid device pointer. // diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs index 34d0bea4f9a5..2736b798cdc6 100644 --- a/rust/kernel/configfs.rs +++ b/rust/kernel/configfs.rs @@ -17,7 +17,7 @@ //! //! C header: [`include/linux/configfs.h`](srctree/include/linux/configfs.h) //! -//! # Example +//! # Examples //! //! ```ignore //! use kernel::alloc::flags; @@ -151,7 +151,7 @@ impl<Data> Subsystem<Data> { data: impl PinInit<Data, Error>, ) -> impl PinInit<Self, Error> { try_pin_init!(Self { - subsystem <- pin_init::zeroed().chain( + subsystem <- pin_init::init_zeroed().chain( |place: &mut Opaque<bindings::configfs_subsystem>| { // SAFETY: We initialized the required fields of `place.group` above. unsafe { @@ -261,7 +261,7 @@ impl<Data> Group<Data> { data: impl PinInit<Data, Error>, ) -> impl PinInit<Self, Error> { try_pin_init!(Self { - group <- pin_init::zeroed().chain(|v: &mut Opaque<bindings::config_group>| { + group <- pin_init::init_zeroed().chain(|v: &mut Opaque<bindings::config_group>| { let place = v.get(); let name = name.as_bytes_with_nul().as_ptr(); // SAFETY: It is safe to initialize a group once it has been zeroed. @@ -279,7 +279,7 @@ impl<Data> Group<Data> { // within the `group` field. unsafe impl<Data> HasGroup<Data> for Group<Data> { unsafe fn group(this: *const Self) -> *const bindings::config_group { - Opaque::raw_get( + Opaque::cast_into( // SAFETY: By impl and function safety requirements this field // projection is within bounds of the allocation. unsafe { &raw const (*this).group }, @@ -426,7 +426,7 @@ where }; const fn vtable_ptr() -> *const bindings::configfs_group_operations { - &Self::VTABLE as *const bindings::configfs_group_operations + &Self::VTABLE } } @@ -464,7 +464,7 @@ where }; const fn vtable_ptr() -> *const bindings::configfs_item_operations { - &Self::VTABLE as *const bindings::configfs_item_operations + &Self::VTABLE } } @@ -476,7 +476,7 @@ impl<Data> ItemOperationsVTable<Subsystem<Data>, Data> { }; const fn vtable_ptr() -> *const bindings::configfs_item_operations { - &Self::VTABLE as *const bindings::configfs_item_operations + &Self::VTABLE } } @@ -561,7 +561,7 @@ where let data: &Data = unsafe { get_group_data(c_group) }; // SAFETY: By function safety requirements, `page` is writable for `PAGE_SIZE`. - let ret = O::show(data, unsafe { &mut *(page as *mut [u8; PAGE_SIZE]) }); + let ret = O::show(data, unsafe { &mut *(page.cast::<[u8; PAGE_SIZE]>()) }); match ret { Ok(size) => size as isize, @@ -717,11 +717,7 @@ impl<const N: usize, Data> AttributeList<N, Data> { // SAFETY: By function safety requirements, we have exclusive access to // `self` and the reference created below will be exclusive. - unsafe { - (&mut *self.0.get())[I] = (attribute as *const Attribute<ID, O, Data>) - .cast_mut() - .cast() - }; + unsafe { (&mut *self.0.get())[I] = core::ptr::from_ref(attribute).cast_mut().cast() }; } } @@ -761,9 +757,7 @@ macro_rules! impl_item_type { ct_owner: owner.as_ptr(), ct_group_ops: GroupOperationsVTable::<Data, Child>::vtable_ptr().cast_mut(), ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(), - ct_attrs: (attributes as *const AttributeList<N, Data>) - .cast_mut() - .cast(), + ct_attrs: core::ptr::from_ref(attributes).cast_mut().cast(), ct_bin_attrs: core::ptr::null_mut(), }), _p: PhantomData, @@ -780,9 +774,7 @@ macro_rules! impl_item_type { ct_owner: owner.as_ptr(), ct_group_ops: core::ptr::null_mut(), ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(), - ct_attrs: (attributes as *const AttributeList<N, Data>) - .cast_mut() - .cast(), + ct_attrs: core::ptr::from_ref(attributes).cast_mut().cast(), ct_bin_attrs: core::ptr::null_mut(), }), _p: PhantomData, diff --git a/rust/kernel/cpu.rs b/rust/kernel/cpu.rs index 10c5c3b25873..5de730c8d817 100644 --- a/rust/kernel/cpu.rs +++ b/rust/kernel/cpu.rs @@ -6,6 +6,127 @@ use crate::{bindings, device::Device, error::Result, prelude::ENODEV}; +/// Returns the maximum number of possible CPUs in the current system configuration. +#[inline] +pub fn nr_cpu_ids() -> u32 { + #[cfg(any(NR_CPUS_1, CONFIG_FORCE_NR_CPUS))] + { + bindings::NR_CPUS + } + + #[cfg(not(any(NR_CPUS_1, CONFIG_FORCE_NR_CPUS)))] + // SAFETY: `nr_cpu_ids` is a valid global provided by the kernel. + unsafe { + bindings::nr_cpu_ids + } +} + +/// The CPU ID. +/// +/// Represents a CPU identifier as a wrapper around an [`u32`]. +/// +/// # Invariants +/// +/// The CPU ID lies within the range `[0, nr_cpu_ids())`. +/// +/// # Examples +/// +/// ``` +/// use kernel::cpu::CpuId; +/// +/// let cpu = 0; +/// +/// // SAFETY: 0 is always a valid CPU number. +/// let id = unsafe { CpuId::from_u32_unchecked(cpu) }; +/// +/// assert_eq!(id.as_u32(), cpu); +/// assert!(CpuId::from_i32(0).is_some()); +/// assert!(CpuId::from_i32(-1).is_none()); +/// ``` +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct CpuId(u32); + +impl CpuId { + /// Creates a new [`CpuId`] from the given `id` without checking bounds. + /// + /// # Safety + /// + /// The caller must ensure that `id` is a valid CPU ID (i.e., `0 <= id < nr_cpu_ids()`). + #[inline] + pub unsafe fn from_i32_unchecked(id: i32) -> Self { + debug_assert!(id >= 0); + debug_assert!((id as u32) < nr_cpu_ids()); + + // INVARIANT: The function safety guarantees `id` is a valid CPU id. + Self(id as u32) + } + + /// Creates a new [`CpuId`] from the given `id`, checking that it is valid. + pub fn from_i32(id: i32) -> Option<Self> { + if id < 0 || id as u32 >= nr_cpu_ids() { + None + } else { + // INVARIANT: `id` has just been checked as a valid CPU ID. + Some(Self(id as u32)) + } + } + + /// Creates a new [`CpuId`] from the given `id` without checking bounds. + /// + /// # Safety + /// + /// The caller must ensure that `id` is a valid CPU ID (i.e., `0 <= id < nr_cpu_ids()`). + #[inline] + pub unsafe fn from_u32_unchecked(id: u32) -> Self { + debug_assert!(id < nr_cpu_ids()); + + // Ensure the `id` fits in an [`i32`] as it's also representable that way. + debug_assert!(id <= i32::MAX as u32); + + // INVARIANT: The function safety guarantees `id` is a valid CPU id. + Self(id) + } + + /// Creates a new [`CpuId`] from the given `id`, checking that it is valid. + pub fn from_u32(id: u32) -> Option<Self> { + if id >= nr_cpu_ids() { + None + } else { + // INVARIANT: `id` has just been checked as a valid CPU ID. + Some(Self(id)) + } + } + + /// Returns CPU number. + #[inline] + pub fn as_u32(&self) -> u32 { + self.0 + } + + /// Returns the ID of the CPU the code is currently running on. + /// + /// The returned value is considered unstable because it may change + /// unexpectedly due to preemption or CPU migration. It should only be + /// used when the context ensures that the task remains on the same CPU + /// or the users could use a stale (yet valid) CPU ID. + pub fn current() -> Self { + // SAFETY: raw_smp_processor_id() always returns a valid CPU ID. + unsafe { Self::from_u32_unchecked(bindings::raw_smp_processor_id()) } + } +} + +impl From<CpuId> for u32 { + fn from(id: CpuId) -> Self { + id.as_u32() + } +} + +impl From<CpuId> for i32 { + fn from(id: CpuId) -> Self { + id.as_u32() as i32 + } +} + /// Creates a new instance of CPU's device. /// /// # Safety @@ -17,14 +138,14 @@ use crate::{bindings, device::Device, error::Result, prelude::ENODEV}; /// Callers must ensure that the CPU device is not used after it has been unregistered. /// This can be achieved, for example, by registering a CPU hotplug notifier and removing /// any references to the CPU device within the notifier's callback. -pub unsafe fn from_cpu(cpu: u32) -> Result<&'static Device> { +pub unsafe fn from_cpu(cpu: CpuId) -> Result<&'static Device> { // SAFETY: It is safe to call `get_cpu_device()` for any CPU. - let ptr = unsafe { bindings::get_cpu_device(cpu) }; + let ptr = unsafe { bindings::get_cpu_device(u32::from(cpu)) }; if ptr.is_null() { return Err(ENODEV); } // SAFETY: The pointer returned by `get_cpu_device()`, if not `NULL`, is a valid pointer to // a `struct device` and is never freed by the C code. - Ok(unsafe { Device::as_ref(ptr) }) + Ok(unsafe { Device::from_raw(ptr) }) } diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index b0a9c6182aec..afc15e72a7c3 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -10,9 +10,10 @@ use crate::{ clk::Hertz, + cpu::CpuId, cpumask, device::{Bound, Device}, - devres::Devres, + devres, error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR}, ffi::{c_char, c_ulong}, prelude::*, @@ -201,7 +202,7 @@ impl From<TableIndex> for usize { /// The callers must ensure that the `struct cpufreq_frequency_table` is valid for access and /// remains valid for the lifetime of the returned reference. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to read a frequency value from [`Table`]. /// @@ -317,7 +318,7 @@ impl Deref for TableBox { /// /// This is used by the CPU frequency drivers to build a frequency table dynamically. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to create a CPU frequency table. /// @@ -394,7 +395,7 @@ impl TableBuilder { /// The callers must ensure that the `struct cpufreq_policy` is valid for access and remains valid /// for the lifetime of the returned reference. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to create a CPU frequency table. /// @@ -465,8 +466,9 @@ impl Policy { /// Returns the primary CPU for the [`Policy`]. #[inline] - pub fn cpu(&self) -> u32 { - self.as_ref().cpu + pub fn cpu(&self) -> CpuId { + // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. + unsafe { CpuId::from_u32_unchecked(self.as_ref().cpu) } } /// Returns the minimum frequency for the [`Policy`]. @@ -525,7 +527,7 @@ impl Policy { #[inline] pub fn generic_get(&self) -> Result<u32> { // SAFETY: By the type invariant, the pointer stored in `self` is valid. - Ok(unsafe { bindings::cpufreq_generic_get(self.cpu()) }) + Ok(unsafe { bindings::cpufreq_generic_get(u32::from(self.cpu())) }) } /// Provides a wrapper to the register with energy model using the OPP core. @@ -647,7 +649,7 @@ impl Policy { fn set_data<T: ForeignOwnable>(&mut self, data: T) -> Result { if self.as_ref().driver_data.is_null() { // Transfer the ownership of the data to the foreign interface. - self.as_mut_ref().driver_data = <T as ForeignOwnable>::into_foreign(data) as _; + self.as_mut_ref().driver_data = <T as ForeignOwnable>::into_foreign(data).cast(); Ok(()) } else { Err(EBUSY) @@ -678,9 +680,9 @@ impl Policy { struct PolicyCpu<'a>(&'a mut Policy); impl<'a> PolicyCpu<'a> { - fn from_cpu(cpu: u32) -> Result<Self> { + fn from_cpu(cpu: CpuId) -> Result<Self> { // SAFETY: It is safe to call `cpufreq_cpu_get` for any valid CPU. - let ptr = from_err_ptr(unsafe { bindings::cpufreq_cpu_get(cpu) })?; + let ptr = from_err_ptr(unsafe { bindings::cpufreq_cpu_get(u32::from(cpu)) })?; Ok(Self( // SAFETY: The `ptr` is guaranteed to be valid and remains valid for the lifetime of @@ -832,7 +834,7 @@ pub trait Driver { /// CPU frequency driver Registration. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to register a cpufreq driver. /// @@ -1044,10 +1046,13 @@ impl<T: Driver> Registration<T> { /// Same as [`Registration::new`], but does not return a [`Registration`] instance. /// - /// Instead the [`Registration`] is owned by [`Devres`] and will be revoked / dropped, once the + /// Instead the [`Registration`] is owned by [`devres::register`] and will be dropped, once the /// device is detached. - pub fn new_foreign_owned(dev: &Device<Bound>) -> Result { - Devres::new_foreign_owned(dev, Self::new()?, GFP_KERNEL) + pub fn new_foreign_owned(dev: &Device<Bound>) -> Result + where + T: 'static, + { + devres::register(dev, Self::new()?, GFP_KERNEL) } } @@ -1055,8 +1060,11 @@ impl<T: Driver> Registration<T> { impl<T: Driver> Registration<T> { /// Driver's `init` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn init_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn init_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1070,8 +1078,11 @@ impl<T: Driver> Registration<T> { /// Driver's `exit` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn exit_callback(ptr: *mut bindings::cpufreq_policy) { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn exit_callback(ptr: *mut bindings::cpufreq_policy) { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. let policy = unsafe { Policy::from_raw_mut(ptr) }; @@ -1082,8 +1093,11 @@ impl<T: Driver> Registration<T> { /// Driver's `online` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn online_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn online_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1094,8 +1108,11 @@ impl<T: Driver> Registration<T> { /// Driver's `offline` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn offline_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn offline_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1106,8 +1123,11 @@ impl<T: Driver> Registration<T> { /// Driver's `suspend` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn suspend_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn suspend_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1118,8 +1138,11 @@ impl<T: Driver> Registration<T> { /// Driver's `resume` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn resume_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn resume_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1130,8 +1153,11 @@ impl<T: Driver> Registration<T> { /// Driver's `ready` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn ready_callback(ptr: *mut bindings::cpufreq_policy) { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn ready_callback(ptr: *mut bindings::cpufreq_policy) { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. let policy = unsafe { Policy::from_raw_mut(ptr) }; @@ -1140,8 +1166,11 @@ impl<T: Driver> Registration<T> { /// Driver's `verify` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn verify_callback(ptr: *mut bindings::cpufreq_policy_data) -> kernel::ffi::c_int { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn verify_callback(ptr: *mut bindings::cpufreq_policy_data) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1152,8 +1181,11 @@ impl<T: Driver> Registration<T> { /// Driver's `setpolicy` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn setpolicy_callback(ptr: *mut bindings::cpufreq_policy) -> kernel::ffi::c_int { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn setpolicy_callback(ptr: *mut bindings::cpufreq_policy) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1164,12 +1196,15 @@ impl<T: Driver> Registration<T> { /// Driver's `target` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn target_callback( + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn target_callback( ptr: *mut bindings::cpufreq_policy, - target_freq: u32, - relation: u32, - ) -> kernel::ffi::c_int { + target_freq: c_uint, + relation: c_uint, + ) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1180,11 +1215,14 @@ impl<T: Driver> Registration<T> { /// Driver's `target_index` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn target_index_callback( + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn target_index_callback( ptr: *mut bindings::cpufreq_policy, - index: u32, - ) -> kernel::ffi::c_int { + index: c_uint, + ) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1200,11 +1238,14 @@ impl<T: Driver> Registration<T> { /// Driver's `fast_switch` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn fast_switch_callback( + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn fast_switch_callback( ptr: *mut bindings::cpufreq_policy, - target_freq: u32, - ) -> kernel::ffi::c_uint { + target_freq: c_uint, + ) -> c_uint { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. let policy = unsafe { Policy::from_raw_mut(ptr) }; @@ -1212,24 +1253,34 @@ impl<T: Driver> Registration<T> { } /// Driver's `adjust_perf` callback. - extern "C" fn adjust_perf_callback( - cpu: u32, - min_perf: usize, - target_perf: usize, - capacity: usize, + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + unsafe extern "C" fn adjust_perf_callback( + cpu: c_uint, + min_perf: c_ulong, + target_perf: c_ulong, + capacity: c_ulong, ) { - if let Ok(mut policy) = PolicyCpu::from_cpu(cpu) { + // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. + let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; + + if let Ok(mut policy) = PolicyCpu::from_cpu(cpu_id) { T::adjust_perf(&mut policy, min_perf, target_perf, capacity); } } /// Driver's `get_intermediate` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn get_intermediate_callback( + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn get_intermediate_callback( ptr: *mut bindings::cpufreq_policy, - index: u32, - ) -> kernel::ffi::c_uint { + index: c_uint, + ) -> c_uint { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. let policy = unsafe { Policy::from_raw_mut(ptr) }; @@ -1243,11 +1294,14 @@ impl<T: Driver> Registration<T> { /// Driver's `target_intermediate` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn target_intermediate_callback( + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn target_intermediate_callback( ptr: *mut bindings::cpufreq_policy, - index: u32, - ) -> kernel::ffi::c_int { + index: c_uint, + ) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1262,12 +1316,24 @@ impl<T: Driver> Registration<T> { } /// Driver's `get` callback. - extern "C" fn get_callback(cpu: u32) -> kernel::ffi::c_uint { - PolicyCpu::from_cpu(cpu).map_or(0, |mut policy| T::get(&mut policy).map_or(0, |f| f)) + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + unsafe extern "C" fn get_callback(cpu: c_uint) -> c_uint { + // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. + let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; + + PolicyCpu::from_cpu(cpu_id).map_or(0, |mut policy| T::get(&mut policy).map_or(0, |f| f)) } /// Driver's `update_limit` callback. - extern "C" fn update_limits_callback(ptr: *mut bindings::cpufreq_policy) { + /// + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn update_limits_callback(ptr: *mut bindings::cpufreq_policy) { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. let policy = unsafe { Policy::from_raw_mut(ptr) }; @@ -1276,10 +1342,16 @@ impl<T: Driver> Registration<T> { /// Driver's `bios_limit` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn bios_limit_callback(cpu: i32, limit: *mut u32) -> kernel::ffi::c_int { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn bios_limit_callback(cpu: c_int, limit: *mut c_uint) -> c_int { + // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. + let cpu_id = unsafe { CpuId::from_i32_unchecked(cpu) }; + from_result(|| { - let mut policy = PolicyCpu::from_cpu(cpu as u32)?; + let mut policy = PolicyCpu::from_cpu(cpu_id)?; // SAFETY: `limit` is guaranteed by the C code to be valid. T::bios_limit(&mut policy, &mut (unsafe { *limit })).map(|()| 0) @@ -1288,11 +1360,14 @@ impl<T: Driver> Registration<T> { /// Driver's `set_boost` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn set_boost_callback( + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn set_boost_callback( ptr: *mut bindings::cpufreq_policy, - state: i32, - ) -> kernel::ffi::c_int { + state: c_int, + ) -> c_int { from_result(|| { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. @@ -1303,8 +1378,11 @@ impl<T: Driver> Registration<T> { /// Driver's `register_em` callback. /// - /// SAFETY: Called from C. Inputs must be valid pointers. - extern "C" fn register_em_callback(ptr: *mut bindings::cpufreq_policy) { + /// # Safety + /// + /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. + unsafe extern "C" fn register_em_callback(ptr: *mut bindings::cpufreq_policy) { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. let policy = unsafe { Policy::from_raw_mut(ptr) }; diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs index c90bfac9346a..3fcbff438670 100644 --- a/rust/kernel/cpumask.rs +++ b/rust/kernel/cpumask.rs @@ -6,6 +6,7 @@ use crate::{ alloc::{AllocError, Flags}, + cpu::CpuId, prelude::*, types::Opaque, }; @@ -13,9 +14,6 @@ use crate::{ #[cfg(CONFIG_CPUMASK_OFFSTACK)] use core::ptr::{self, NonNull}; -#[cfg(not(CONFIG_CPUMASK_OFFSTACK))] -use core::mem::MaybeUninit; - use core::ops::{Deref, DerefMut}; /// A CPU Mask. @@ -29,15 +27,16 @@ use core::ops::{Deref, DerefMut}; /// The callers must ensure that the `struct cpumask` is valid for access and /// remains valid for the lifetime of the returned reference. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to update a [`Cpumask`]. /// /// ``` /// use kernel::bindings; +/// use kernel::cpu::CpuId; /// use kernel::cpumask::Cpumask; /// -/// fn set_clear_cpu(ptr: *mut bindings::cpumask, set_cpu: u32, clear_cpu: i32) { +/// fn set_clear_cpu(ptr: *mut bindings::cpumask, set_cpu: CpuId, clear_cpu: CpuId) { /// // SAFETY: The `ptr` is valid for writing and remains valid for the lifetime of the /// // returned reference. /// let mask = unsafe { Cpumask::as_mut_ref(ptr) }; @@ -90,9 +89,9 @@ impl Cpumask { /// This mismatches kernel naming convention and corresponds to the C /// function `__cpumask_set_cpu()`. #[inline] - pub fn set(&mut self, cpu: u32) { + pub fn set(&mut self, cpu: CpuId) { // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `__cpumask_set_cpu`. - unsafe { bindings::__cpumask_set_cpu(cpu, self.as_raw()) }; + unsafe { bindings::__cpumask_set_cpu(u32::from(cpu), self.as_raw()) }; } /// Clear `cpu` in the cpumask. @@ -101,19 +100,19 @@ impl Cpumask { /// This mismatches kernel naming convention and corresponds to the C /// function `__cpumask_clear_cpu()`. #[inline] - pub fn clear(&mut self, cpu: i32) { + pub fn clear(&mut self, cpu: CpuId) { // SAFETY: By the type invariant, `self.as_raw` is a valid argument to // `__cpumask_clear_cpu`. - unsafe { bindings::__cpumask_clear_cpu(cpu, self.as_raw()) }; + unsafe { bindings::__cpumask_clear_cpu(i32::from(cpu), self.as_raw()) }; } /// Test `cpu` in the cpumask. /// /// Equivalent to the kernel's `cpumask_test_cpu` API. #[inline] - pub fn test(&self, cpu: i32) -> bool { + pub fn test(&self, cpu: CpuId) -> bool { // SAFETY: By the type invariant, `self.as_raw` is a valid argument to `cpumask_test_cpu`. - unsafe { bindings::cpumask_test_cpu(cpu, self.as_raw()) } + unsafe { bindings::cpumask_test_cpu(i32::from(cpu), self.as_raw()) } } /// Set all CPUs in the cpumask. @@ -173,26 +172,45 @@ impl Cpumask { /// The callers must ensure that the `struct cpumask_var_t` is valid for access and remains valid /// for the lifetime of [`CpumaskVar`]. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to create and update a [`CpumaskVar`]. /// /// ``` +/// use kernel::cpu::CpuId; /// use kernel::cpumask::CpumaskVar; /// /// let mut mask = CpumaskVar::new_zero(GFP_KERNEL).unwrap(); /// /// assert!(mask.empty()); -/// mask.set(2); -/// assert!(mask.test(2)); -/// mask.set(3); -/// assert!(mask.test(3)); -/// assert_eq!(mask.weight(), 2); +/// let mut count = 0; +/// +/// let cpu2 = CpuId::from_u32(2); +/// if let Some(cpu) = cpu2 { +/// mask.set(cpu); +/// assert!(mask.test(cpu)); +/// count += 1; +/// } +/// +/// let cpu3 = CpuId::from_u32(3); +/// if let Some(cpu) = cpu3 { +/// mask.set(cpu); +/// assert!(mask.test(cpu)); +/// count += 1; +/// } +/// +/// assert_eq!(mask.weight(), count); /// /// let mask2 = CpumaskVar::try_clone(&mask).unwrap(); -/// assert!(mask2.test(2)); -/// assert!(mask2.test(3)); -/// assert_eq!(mask2.weight(), 2); +/// +/// if let Some(cpu) = cpu2 { +/// assert!(mask2.test(cpu)); +/// } +/// +/// if let Some(cpu) = cpu3 { +/// assert!(mask2.test(cpu)); +/// } +/// assert_eq!(mask2.weight(), count); /// ``` pub struct CpumaskVar { #[cfg(CONFIG_CPUMASK_OFFSTACK)] @@ -218,10 +236,7 @@ impl CpumaskVar { }, #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] - // SAFETY: FFI type is valid to be zero-initialized. - // - // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of scope. - mask: unsafe { core::mem::zeroed() }, + mask: Cpumask(Opaque::zeroed()), }) } @@ -245,10 +260,7 @@ impl CpumaskVar { NonNull::new(ptr.cast()).ok_or(AllocError)? }, #[cfg(not(CONFIG_CPUMASK_OFFSTACK))] - // SAFETY: Guaranteed by the safety requirements of the function. - // - // INVARIANT: The associated memory is freed when the `CpumaskVar` goes out of scope. - mask: unsafe { MaybeUninit::uninit().assume_init() }, + mask: Cpumask(Opaque::uninit()), }) } diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index dea06b79ecb5..b8613289de8e 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -6,14 +6,15 @@ use crate::{ bindings, - str::CStr, - types::{ARef, Opaque}, + types::{ARef, ForeignOwnable, Opaque}, }; use core::{fmt, marker::PhantomData, ptr}; #[cfg(CONFIG_PRINTK)] use crate::c_str; +pub mod property; + /// A reference-counted device. /// /// This structure represents the Rust abstraction for a C `struct device`. This implementation @@ -57,7 +58,73 @@ impl Device { /// While not officially documented, this should be the case for any `struct device`. pub unsafe fn get_device(ptr: *mut bindings::device) -> ARef<Self> { // SAFETY: By the safety requirements ptr is valid - unsafe { Self::as_ref(ptr) }.into() + unsafe { Self::from_raw(ptr) }.into() + } + + /// Convert a [`&Device`](Device) into a [`&Device<Bound>`](Device<Bound>). + /// + /// # Safety + /// + /// The caller is responsible to ensure that the returned [`&Device<Bound>`](Device<Bound>) + /// only lives as long as it can be guaranteed that the [`Device`] is actually bound. + pub unsafe fn as_bound(&self) -> &Device<Bound> { + let ptr = core::ptr::from_ref(self); + + // CAST: By the safety requirements the caller is responsible to guarantee that the + // returned reference only lives as long as the device is actually bound. + let ptr = ptr.cast(); + + // SAFETY: + // - `ptr` comes from `from_ref(self)` above, hence it's guaranteed to be valid. + // - Any valid `Device` pointer is also a valid pointer for `Device<Bound>`. + unsafe { &*ptr } + } +} + +impl Device<CoreInternal> { + /// Store a pointer to the bound driver's private data. + pub fn set_drvdata(&self, data: impl ForeignOwnable) { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + unsafe { bindings::dev_set_drvdata(self.as_raw(), data.into_foreign().cast()) } + } + + /// Take ownership of the private data stored in this [`Device`]. + /// + /// # Safety + /// + /// - Must only be called once after a preceding call to [`Device::set_drvdata`]. + /// - The type `T` must match the type of the `ForeignOwnable` previously stored by + /// [`Device::set_drvdata`]. + pub unsafe fn drvdata_obtain<T: ForeignOwnable>(&self) -> T { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; + + // SAFETY: + // - By the safety requirements of this function, `ptr` comes from a previous call to + // `into_foreign()`. + // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` + // in `into_foreign()`. + unsafe { T::from_foreign(ptr.cast()) } + } + + /// Borrow the driver's private data bound to this [`Device`]. + /// + /// # Safety + /// + /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before + /// [`Device::drvdata_obtain`]. + /// - The type `T` must match the type of the `ForeignOwnable` previously stored by + /// [`Device::set_drvdata`]. + pub unsafe fn drvdata_borrow<T: ForeignOwnable>(&self) -> T::Borrowed<'_> { + // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. + let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; + + // SAFETY: + // - By the safety requirements of this function, `ptr` comes from a previous call to + // `into_foreign()`. + // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` + // in `into_foreign()`. + unsafe { T::borrow(ptr.cast()) } } } @@ -82,7 +149,7 @@ impl<Ctx: DeviceContext> Device<Ctx> { // - Since `parent` is not NULL, it must be a valid pointer to a `struct device`. // - `parent` is valid for the lifetime of `self`, since a `struct device` holds a // reference count of its parent. - Some(unsafe { Self::as_ref(parent) }) + Some(unsafe { Self::from_raw(parent) }) } } @@ -94,7 +161,7 @@ impl<Ctx: DeviceContext> Device<Ctx> { /// i.e. it must be ensured that the reference count of the C `struct device` `ptr` points to /// can't drop to zero, for the duration of this function call and the entire duration when the /// returned reference exists. - pub unsafe fn as_ref<'a>(ptr: *mut bindings::device) -> &'a Self { + pub unsafe fn from_raw<'a>(ptr: *mut bindings::device) -> &'a Self { // SAFETY: Guaranteed by the safety requirements of the function. unsafe { &*ptr.cast() } } @@ -195,18 +262,27 @@ impl<Ctx: DeviceContext> Device<Ctx> { #[cfg(CONFIG_PRINTK)] unsafe { bindings::_dev_printk( - klevel as *const _ as *const crate::ffi::c_char, + klevel.as_ptr().cast::<crate::ffi::c_char>(), self.as_raw(), c_str!("%pA").as_char_ptr(), - &msg as *const _ as *const crate::ffi::c_void, + core::ptr::from_ref(&msg).cast::<crate::ffi::c_void>(), ) }; } - /// Checks if property is present or not. - pub fn property_present(&self, name: &CStr) -> bool { - // SAFETY: By the invariant of `CStr`, `name` is null-terminated. - unsafe { bindings::device_property_present(self.as_raw().cast_const(), name.as_char_ptr()) } + /// Obtain the [`FwNode`](property::FwNode) corresponding to this [`Device`]. + pub fn fwnode(&self) -> Option<&property::FwNode> { + // SAFETY: `self` is valid. + let fwnode_handle = unsafe { bindings::__dev_fwnode(self.as_raw()) }; + if fwnode_handle.is_null() { + return None; + } + // SAFETY: `fwnode_handle` is valid. Its lifetime is tied to `&self`. We + // return a reference instead of an `ARef<FwNode>` because `dev_fwnode()` + // doesn't increment the refcount. It is safe to cast from a + // `struct fwnode_handle*` to a `*const FwNode` because `FwNode` is + // defined as a `#[repr(transparent)]` wrapper around `fwnode_handle`. + Some(unsafe { &*fwnode_handle.cast() }) } } @@ -251,6 +327,10 @@ pub struct Normal; /// any of the bus callbacks, such as `probe()`. pub struct Core; +/// Semantically the same as [`Core`] but reserved for internal usage of the corresponding bus +/// abstraction. +pub struct CoreInternal; + /// The [`Bound`] context is the context of a bus specific device reference when it is guaranteed to /// be bound for the duration of its lifetime. pub struct Bound; @@ -260,11 +340,13 @@ mod private { impl Sealed for super::Bound {} impl Sealed for super::Core {} + impl Sealed for super::CoreInternal {} impl Sealed for super::Normal {} } impl DeviceContext for Bound {} impl DeviceContext for Core {} +impl DeviceContext for CoreInternal {} impl DeviceContext for Normal {} /// # Safety @@ -306,6 +388,13 @@ macro_rules! impl_device_context_deref { // `__impl_device_context_deref!`. ::kernel::__impl_device_context_deref!(unsafe { $device, + $crate::device::CoreInternal => $crate::device::Core + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, $crate::device::Core => $crate::device::Bound }); @@ -335,6 +424,7 @@ macro_rules! __impl_device_context_into_aref { #[macro_export] macro_rules! impl_device_context_into_aref { ($device:tt) => { + ::kernel::__impl_device_context_into_aref!($crate::device::CoreInternal, $device); ::kernel::__impl_device_context_into_aref!($crate::device::Core, $device); ::kernel::__impl_device_context_into_aref!($crate::device::Bound, $device); }; diff --git a/rust/kernel/device/property.rs b/rust/kernel/device/property.rs new file mode 100644 index 000000000000..49ee12a906db --- /dev/null +++ b/rust/kernel/device/property.rs @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Unified device property interface. +//! +//! C header: [`include/linux/property.h`](srctree/include/linux/property.h) + +use core::{mem::MaybeUninit, ptr}; + +use super::private::Sealed; +use crate::{ + alloc::KVec, + bindings, + error::{to_result, Result}, + prelude::*, + str::{CStr, CString}, + types::{ARef, Opaque}, +}; + +/// A reference-counted fwnode_handle. +/// +/// This structure represents the Rust abstraction for a +/// C `struct fwnode_handle`. This implementation abstracts the usage of an +/// already existing C `struct fwnode_handle` within Rust code that we get +/// passed from the C side. +/// +/// # Invariants +/// +/// A `FwNode` instance represents a valid `struct fwnode_handle` created by the +/// C portion of the kernel. +/// +/// Instances of this type are always reference-counted, that is, a call to +/// `fwnode_handle_get` ensures that the allocation remains valid at least until +/// the matching call to `fwnode_handle_put`. +#[repr(transparent)] +pub struct FwNode(Opaque<bindings::fwnode_handle>); + +impl FwNode { + /// # Safety + /// + /// Callers must ensure that: + /// - The reference count was incremented at least once. + /// - They relinquish that increment. That is, if there is only one + /// increment, callers must not use the underlying object anymore -- it is + /// only safe to do so via the newly created `ARef<FwNode>`. + unsafe fn from_raw(raw: *mut bindings::fwnode_handle) -> ARef<Self> { + // SAFETY: As per the safety requirements of this function: + // - `NonNull::new_unchecked`: + // - `raw` is not null. + // - `ARef::from_raw`: + // - `raw` has an incremented refcount. + // - that increment is relinquished, i.e. it won't be decremented + // elsewhere. + // CAST: It is safe to cast from a `*mut fwnode_handle` to + // `*mut FwNode`, because `FwNode` is defined as a + // `#[repr(transparent)]` wrapper around `fwnode_handle`. + unsafe { ARef::from_raw(ptr::NonNull::new_unchecked(raw.cast())) } + } + + /// Obtain the raw `struct fwnode_handle *`. + pub(crate) fn as_raw(&self) -> *mut bindings::fwnode_handle { + self.0.get() + } + + /// Returns `true` if `&self` is an OF node, `false` otherwise. + pub fn is_of_node(&self) -> bool { + // SAFETY: The type invariant of `Self` guarantees that `self.as_raw() is a pointer to a + // valid `struct fwnode_handle`. + unsafe { bindings::is_of_node(self.as_raw()) } + } + + /// Returns an object that implements [`Display`](core::fmt::Display) for + /// printing the name of a node. + /// + /// This is an alternative to the default `Display` implementation, which + /// prints the full path. + pub fn display_name(&self) -> impl core::fmt::Display + '_ { + struct FwNodeDisplayName<'a>(&'a FwNode); + + impl core::fmt::Display for FwNodeDisplayName<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // SAFETY: `self` is valid by its type invariant. + let name = unsafe { bindings::fwnode_get_name(self.0.as_raw()) }; + if name.is_null() { + return Ok(()); + } + // SAFETY: + // - `fwnode_get_name` returns null or a valid C string. + // - `name` was checked to be non-null. + let name = unsafe { CStr::from_char_ptr(name) }; + write!(f, "{name}") + } + } + + FwNodeDisplayName(self) + } + + /// Checks if property is present or not. + pub fn property_present(&self, name: &CStr) -> bool { + // SAFETY: By the invariant of `CStr`, `name` is null-terminated. + unsafe { bindings::fwnode_property_present(self.as_raw().cast_const(), name.as_char_ptr()) } + } + + /// Returns firmware property `name` boolean value. + pub fn property_read_bool(&self, name: &CStr) -> bool { + // SAFETY: + // - `name` is non-null and null-terminated. + // - `self.as_raw()` is valid because `self` is valid. + unsafe { bindings::fwnode_property_read_bool(self.as_raw(), name.as_char_ptr()) } + } + + /// Returns the index of matching string `match_str` for firmware string + /// property `name`. + pub fn property_match_string(&self, name: &CStr, match_str: &CStr) -> Result<usize> { + // SAFETY: + // - `name` and `match_str` are non-null and null-terminated. + // - `self.as_raw` is valid because `self` is valid. + let ret = unsafe { + bindings::fwnode_property_match_string( + self.as_raw(), + name.as_char_ptr(), + match_str.as_char_ptr(), + ) + }; + to_result(ret)?; + Ok(ret as usize) + } + + /// Returns firmware property `name` integer array values in a [`KVec`]. + pub fn property_read_array_vec<'fwnode, 'name, T: PropertyInt>( + &'fwnode self, + name: &'name CStr, + len: usize, + ) -> Result<PropertyGuard<'fwnode, 'name, KVec<T>>> { + let mut val: KVec<T> = KVec::with_capacity(len, GFP_KERNEL)?; + + let res = T::read_array_from_fwnode_property(self, name, val.spare_capacity_mut()); + let res = match res { + Ok(_) => { + // SAFETY: + // - `len` is equal to `val.capacity - val.len`, because + // `val.capacity` is `len` and `val.len` is zero. + // - All elements within the interval [`0`, `len`) were initialized + // by `read_array_from_fwnode_property`. + unsafe { val.inc_len(len) } + Ok(val) + } + Err(e) => Err(e), + }; + Ok(PropertyGuard { + inner: res, + fwnode: self, + name, + }) + } + + /// Returns integer array length for firmware property `name`. + pub fn property_count_elem<T: PropertyInt>(&self, name: &CStr) -> Result<usize> { + T::read_array_len_from_fwnode_property(self, name) + } + + /// Returns the value of firmware property `name`. + /// + /// This method is generic over the type of value to read. The types that + /// can be read are strings, integers and arrays of integers. + /// + /// Reading a [`KVec`] of integers is done with the separate + /// method [`Self::property_read_array_vec`], because it takes an + /// additional `len` argument. + /// + /// Reading a boolean is done with the separate method + /// [`Self::property_read_bool`], because this operation is infallible. + /// + /// For more precise documentation about what types can be read, see + /// the [implementors of Property][Property#implementors] and [its + /// implementations on foreign types][Property#foreign-impls]. + /// + /// # Examples + /// + /// ``` + /// # use kernel::{c_str, device::{Device, property::FwNode}, str::CString}; + /// fn examples(dev: &Device) -> Result { + /// let fwnode = dev.fwnode().ok_or(ENOENT)?; + /// let b: u32 = fwnode.property_read(c_str!("some-number")).required_by(dev)?; + /// if let Some(s) = fwnode.property_read::<CString>(c_str!("some-str")).optional() { + /// // ... + /// } + /// Ok(()) + /// } + /// ``` + pub fn property_read<'fwnode, 'name, T: Property>( + &'fwnode self, + name: &'name CStr, + ) -> PropertyGuard<'fwnode, 'name, T> { + PropertyGuard { + inner: T::read_from_fwnode_property(self, name), + fwnode: self, + name, + } + } + + /// Returns first matching named child node handle. + pub fn get_child_by_name(&self, name: &CStr) -> Option<ARef<Self>> { + // SAFETY: `self` and `name` are valid by their type invariants. + let child = + unsafe { bindings::fwnode_get_named_child_node(self.as_raw(), name.as_char_ptr()) }; + if child.is_null() { + return None; + } + // SAFETY: + // - `fwnode_get_named_child_node` returns a pointer with its refcount + // incremented. + // - That increment is relinquished, i.e. the underlying object is not + // used anymore except via the newly created `ARef`. + Some(unsafe { Self::from_raw(child) }) + } + + /// Returns an iterator over a node's children. + pub fn children<'a>(&'a self) -> impl Iterator<Item = ARef<FwNode>> + 'a { + let mut prev: Option<ARef<FwNode>> = None; + + core::iter::from_fn(move || { + let prev_ptr = match prev.take() { + None => ptr::null_mut(), + Some(prev) => { + // We will pass `prev` to `fwnode_get_next_child_node`, + // which decrements its refcount, so we use + // `ARef::into_raw` to avoid decrementing the refcount + // twice. + let prev = ARef::into_raw(prev); + prev.as_ptr().cast() + } + }; + // SAFETY: + // - `self.as_raw()` is valid by its type invariant. + // - `prev_ptr` may be null, which is allowed and corresponds to + // getting the first child. Otherwise, `prev_ptr` is valid, as it + // is the stored return value from the previous invocation. + // - `prev_ptr` has its refount incremented. + // - The increment of `prev_ptr` is relinquished, i.e. the + // underlying object won't be used anymore. + let next = unsafe { bindings::fwnode_get_next_child_node(self.as_raw(), prev_ptr) }; + if next.is_null() { + return None; + } + // SAFETY: + // - `next` is valid because `fwnode_get_next_child_node` returns a + // pointer with its refcount incremented. + // - That increment is relinquished, i.e. the underlying object + // won't be used anymore, except via the newly created + // `ARef<Self>`. + let next = unsafe { FwNode::from_raw(next) }; + prev = Some(next.clone()); + Some(next) + }) + } + + /// Finds a reference with arguments. + pub fn property_get_reference_args( + &self, + prop: &CStr, + nargs: NArgs<'_>, + index: u32, + ) -> Result<FwNodeReferenceArgs> { + let mut out_args = FwNodeReferenceArgs::default(); + + let (nargs_prop, nargs) = match nargs { + NArgs::Prop(nargs_prop) => (nargs_prop.as_char_ptr(), 0), + NArgs::N(nargs) => (ptr::null(), nargs), + }; + + // SAFETY: + // - `self.0.get()` is valid. + // - `prop.as_char_ptr()` is valid and zero-terminated. + // - `nargs_prop` is valid and zero-terminated if `nargs` + // is zero, otherwise it is allowed to be a null-pointer. + // - The function upholds the type invariants of `out_args`, + // namely: + // - It may fill the field `fwnode` with a valid pointer, + // in which case its refcount is incremented. + // - It may modify the field `nargs`, in which case it + // initializes at least as many elements in `args`. + let ret = unsafe { + bindings::fwnode_property_get_reference_args( + self.0.get(), + prop.as_char_ptr(), + nargs_prop, + nargs, + index, + &mut out_args.0, + ) + }; + to_result(ret)?; + + Ok(out_args) + } +} + +/// The number of arguments to request [`FwNodeReferenceArgs`]. +pub enum NArgs<'a> { + /// The name of the property of the reference indicating the number of + /// arguments. + Prop(&'a CStr), + /// The known number of arguments. + N(u32), +} + +/// The return value of [`FwNode::property_get_reference_args`]. +/// +/// This structure represents the Rust abstraction for a C +/// `struct fwnode_reference_args` which was initialized by the C side. +/// +/// # Invariants +/// +/// If the field `fwnode` is valid, it owns an increment of its refcount. +/// +/// The field `args` contains at least as many initialized elements as indicated +/// by the field `nargs`. +#[repr(transparent)] +#[derive(Default)] +pub struct FwNodeReferenceArgs(bindings::fwnode_reference_args); + +impl Drop for FwNodeReferenceArgs { + fn drop(&mut self) { + if !self.0.fwnode.is_null() { + // SAFETY: + // - By the type invariants of `FwNodeReferenceArgs`, its field + // `fwnode` owns an increment of its refcount. + // - That increment is relinquished. The underlying object won't be + // used anymore because we are dropping it. + let _ = unsafe { FwNode::from_raw(self.0.fwnode) }; + } + } +} + +impl FwNodeReferenceArgs { + /// Returns the slice of reference arguments. + pub fn as_slice(&self) -> &[u64] { + // SAFETY: As per the safety invariant of `FwNodeReferenceArgs`, `nargs` + // is the minimum number of elements in `args` that is valid. + unsafe { core::slice::from_raw_parts(self.0.args.as_ptr(), self.0.nargs as usize) } + } + + /// Returns the number of reference arguments. + pub fn len(&self) -> usize { + self.0.nargs as usize + } + + /// Returns `true` if there are no reference arguments. + pub fn is_empty(&self) -> bool { + self.0.nargs == 0 + } +} + +impl core::fmt::Debug for FwNodeReferenceArgs { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{:?}", self.as_slice()) + } +} + +// SAFETY: Instances of `FwNode` are always reference-counted. +unsafe impl crate::types::AlwaysRefCounted for FwNode { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the + // refcount is non-zero. + unsafe { bindings::fwnode_handle_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: ptr::NonNull<Self>) { + // SAFETY: The safety requirements guarantee that the refcount is + // non-zero. + unsafe { bindings::fwnode_handle_put(obj.cast().as_ptr()) } + } +} + +enum Node<'a> { + Borrowed(&'a FwNode), + Owned(ARef<FwNode>), +} + +impl core::fmt::Display for FwNode { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // The logic here is the same as the one in lib/vsprintf.c + // (fwnode_full_name_string). + + // SAFETY: `self.as_raw()` is valid by its type invariant. + let num_parents = unsafe { bindings::fwnode_count_parents(self.as_raw()) }; + + for depth in (0..=num_parents).rev() { + let fwnode = if depth == 0 { + Node::Borrowed(self) + } else { + // SAFETY: `self.as_raw()` is valid. + let ptr = unsafe { bindings::fwnode_get_nth_parent(self.as_raw(), depth) }; + // SAFETY: + // - The depth passed to `fwnode_get_nth_parent` is + // within the valid range, so the returned pointer is + // not null. + // - The reference count was incremented by + // `fwnode_get_nth_parent`. + // - That increment is relinquished to + // `FwNode::from_raw`. + Node::Owned(unsafe { FwNode::from_raw(ptr) }) + }; + // Take a reference to the owned or borrowed `FwNode`. + let fwnode: &FwNode = match &fwnode { + Node::Borrowed(f) => f, + Node::Owned(f) => f, + }; + + // SAFETY: `fwnode` is valid by its type invariant. + let prefix = unsafe { bindings::fwnode_get_name_prefix(fwnode.as_raw()) }; + if !prefix.is_null() { + // SAFETY: `fwnode_get_name_prefix` returns null or a + // valid C string. + let prefix = unsafe { CStr::from_char_ptr(prefix) }; + write!(f, "{prefix}")?; + } + write!(f, "{}", fwnode.display_name())?; + } + + Ok(()) + } +} + +/// Implemented for types that can be read as properties. +/// +/// This is implemented for strings, integers and arrays of integers. It's used +/// to make [`FwNode::property_read`] generic over the type of property being +/// read. There are also two dedicated methods to read other types, because they +/// require more specialized function signatures: +/// - [`property_read_bool`](FwNode::property_read_bool) +/// - [`property_read_array_vec`](FwNode::property_read_array_vec) +/// +/// It must be public, because it appears in the signatures of other public +/// functions, but its methods shouldn't be used outside the kernel crate. +pub trait Property: Sized + Sealed { + /// Used to make [`FwNode::property_read`] generic. + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<Self>; +} + +impl Sealed for CString {} + +impl Property for CString { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<Self> { + let mut str: *mut u8 = ptr::null_mut(); + let pstr: *mut _ = &mut str; + + // SAFETY: + // - `name` is non-null and null-terminated. + // - `fwnode.as_raw` is valid because `fwnode` is valid. + let ret = unsafe { + bindings::fwnode_property_read_string(fwnode.as_raw(), name.as_char_ptr(), pstr.cast()) + }; + to_result(ret)?; + + // SAFETY: + // - `pstr` is a valid pointer to a NUL-terminated C string. + // - It is valid for at least as long as `fwnode`, but it's only used + // within the current function. + // - The memory it points to is not mutated during that time. + let str = unsafe { CStr::from_char_ptr(*pstr) }; + Ok(str.try_into()?) + } +} + +/// Implemented for all integers that can be read as properties. +/// +/// This helper trait is needed on top of the existing [`Property`] +/// trait to associate the integer types of various sizes with their +/// corresponding `fwnode_property_read_*_array` functions. +/// +/// It must be public, because it appears in the signatures of other public +/// functions, but its methods shouldn't be used outside the kernel crate. +pub trait PropertyInt: Copy + Sealed { + /// Reads a property array. + fn read_array_from_fwnode_property<'a>( + fwnode: &FwNode, + name: &CStr, + out: &'a mut [MaybeUninit<Self>], + ) -> Result<&'a mut [Self]>; + + /// Reads the length of a property array. + fn read_array_len_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<usize>; +} +// This macro generates implementations of the traits `Property` and +// `PropertyInt` for integers of various sizes. Its input is a list +// of pairs separated by commas. The first element of the pair is the +// type of the integer, the second one is the name of its corresponding +// `fwnode_property_read_*_array` function. +macro_rules! impl_property_for_int { + ($($int:ty: $f:ident),* $(,)?) => { $( + impl Sealed for $int {} + impl<const N: usize> Sealed for [$int; N] {} + + impl PropertyInt for $int { + fn read_array_from_fwnode_property<'a>( + fwnode: &FwNode, + name: &CStr, + out: &'a mut [MaybeUninit<Self>], + ) -> Result<&'a mut [Self]> { + // SAFETY: + // - `fwnode`, `name` and `out` are all valid by their type + // invariants. + // - `out.len()` is a valid bound for the memory pointed to by + // `out.as_mut_ptr()`. + // CAST: It's ok to cast from `*mut MaybeUninit<$int>` to a + // `*mut $int` because they have the same memory layout. + let ret = unsafe { + bindings::$f( + fwnode.as_raw(), + name.as_char_ptr(), + out.as_mut_ptr().cast(), + out.len(), + ) + }; + to_result(ret)?; + // SAFETY: Transmuting from `&'a mut [MaybeUninit<Self>]` to + // `&'a mut [Self]` is sound, because the previous call to a + // `fwnode_property_read_*_array` function (which didn't fail) + // fully initialized the slice. + Ok(unsafe { core::mem::transmute::<&mut [MaybeUninit<Self>], &mut [Self]>(out) }) + } + + fn read_array_len_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<usize> { + // SAFETY: + // - `fwnode` and `name` are valid by their type invariants. + // - It's ok to pass a null pointer to the + // `fwnode_property_read_*_array` functions if `nval` is zero. + // This will return the length of the array. + let ret = unsafe { + bindings::$f( + fwnode.as_raw(), + name.as_char_ptr(), + ptr::null_mut(), + 0, + ) + }; + to_result(ret)?; + Ok(ret as usize) + } + } + + impl Property for $int { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<Self> { + let val: [_; 1] = <[$int; 1]>::read_from_fwnode_property(fwnode, name)?; + Ok(val[0]) + } + } + + impl<const N: usize> Property for [$int; N] { + fn read_from_fwnode_property(fwnode: &FwNode, name: &CStr) -> Result<Self> { + let mut val: [MaybeUninit<$int>; N] = [const { MaybeUninit::uninit() }; N]; + + <$int>::read_array_from_fwnode_property(fwnode, name, &mut val)?; + + // SAFETY: `val` is always initialized when + // `fwnode_property_read_*_array` is successful. + Ok(val.map(|v| unsafe { v.assume_init() })) + } + } + )* }; +} +impl_property_for_int! { + u8: fwnode_property_read_u8_array, + u16: fwnode_property_read_u16_array, + u32: fwnode_property_read_u32_array, + u64: fwnode_property_read_u64_array, + i8: fwnode_property_read_u8_array, + i16: fwnode_property_read_u16_array, + i32: fwnode_property_read_u32_array, + i64: fwnode_property_read_u64_array, +} + +/// A helper for reading device properties. +/// +/// Use [`Self::required_by`] if a missing property is considered a bug and +/// [`Self::optional`] otherwise. +/// +/// For convenience, [`Self::or`] and [`Self::or_default`] are provided. +pub struct PropertyGuard<'fwnode, 'name, T> { + /// The result of reading the property. + inner: Result<T>, + /// The fwnode of the property, used for logging in the "required" case. + fwnode: &'fwnode FwNode, + /// The name of the property, used for logging in the "required" case. + name: &'name CStr, +} + +impl<T> PropertyGuard<'_, '_, T> { + /// Access the property, indicating it is required. + /// + /// If the property is not present, the error is automatically logged. If a + /// missing property is not an error, use [`Self::optional`] instead. The + /// device is required to associate the log with it. + pub fn required_by(self, dev: &super::Device) -> Result<T> { + if self.inner.is_err() { + dev_err!( + dev, + "{}: property '{}' is missing\n", + self.fwnode, + self.name + ); + } + self.inner + } + + /// Access the property, indicating it is optional. + /// + /// In contrast to [`Self::required_by`], no error message is logged if + /// the property is not present. + pub fn optional(self) -> Option<T> { + self.inner.ok() + } + + /// Access the property or the specified default value. + /// + /// Do not pass a sentinel value as default to detect a missing property. + /// Use [`Self::required_by`] or [`Self::optional`] instead. + pub fn or(self, default: T) -> T { + self.inner.unwrap_or(default) + } +} + +impl<T: Default> PropertyGuard<'_, '_, T> { + /// Access the property or a default value. + /// + /// Use [`Self::or`] to specify a custom default value. + pub fn or_default(self) -> T { + self.inner.unwrap_or_default() + } +} diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs index 0a4eb56d98f2..70d57814ff79 100644 --- a/rust/kernel/device_id.rs +++ b/rust/kernel/device_id.rs @@ -14,32 +14,41 @@ use core::mem::MaybeUninit; /// /// # Safety /// -/// Implementers must ensure that: -/// - `Self` is layout-compatible with [`RawDeviceId::RawType`]; i.e. it's safe to transmute to -/// `RawDeviceId`. +/// Implementers must ensure that `Self` is layout-compatible with [`RawDeviceId::RawType`]; +/// i.e. it's safe to transmute to `RawDeviceId`. /// -/// This requirement is needed so `IdArray::new` can convert `Self` to `RawType` when building -/// the ID table. +/// This requirement is needed so `IdArray::new` can convert `Self` to `RawType` when building +/// the ID table. /// -/// Ideally, this should be achieved using a const function that does conversion instead of -/// transmute; however, const trait functions relies on `const_trait_impl` unstable feature, -/// which is broken/gone in Rust 1.73. -/// -/// - `DRIVER_DATA_OFFSET` is the offset of context/data field of the device ID (usually named -/// `driver_data`) of the device ID, the field is suitable sized to write a `usize` value. -/// -/// Similar to the previous requirement, the data should ideally be added during `Self` to -/// `RawType` conversion, but there's currently no way to do it when using traits in const. +/// Ideally, this should be achieved using a const function that does conversion instead of +/// transmute; however, const trait functions relies on `const_trait_impl` unstable feature, +/// which is broken/gone in Rust 1.73. pub unsafe trait RawDeviceId { /// The raw type that holds the device id. /// /// Id tables created from [`Self`] are going to hold this type in its zero-terminated array. type RawType: Copy; +} - /// The offset to the context/data field. +/// Extension trait for [`RawDeviceId`] for devices that embed an index or context value. +/// +/// This is typically used when the device ID struct includes a field like `driver_data` +/// that is used to store a pointer-sized value (e.g., an index or context pointer). +/// +/// # Safety +/// +/// Implementers must ensure that `DRIVER_DATA_OFFSET` is the correct offset (in bytes) to +/// the context/data field (e.g., the `driver_data` field) within the raw device ID structure. +/// This field must be correctly sized to hold a `usize`. +/// +/// Ideally, the data should be added during `Self` to `RawType` conversion, +/// but there's currently no way to do it when using traits in const. +pub unsafe trait RawDeviceIdIndex: RawDeviceId { + /// The offset (in bytes) to the context/data field in the raw device ID. const DRIVER_DATA_OFFSET: usize; - /// The index stored at `DRIVER_DATA_OFFSET` of the implementor of the [`RawDeviceId`] trait. + /// The index stored at `DRIVER_DATA_OFFSET` of the implementor of the [`RawDeviceIdIndex`] + /// trait. fn index(&self) -> usize; } @@ -68,7 +77,15 @@ impl<T: RawDeviceId, U, const N: usize> IdArray<T, U, N> { /// Creates a new instance of the array. /// /// The contents are derived from the given identifiers and context information. - pub const fn new(ids: [(T, U); N]) -> Self { + /// + /// # Safety + /// + /// `data_offset` as `None` is always safe. + /// If `data_offset` is `Some(data_offset)`, then: + /// - `data_offset` must be the correct offset (in bytes) to the context/data field + /// (e.g., the `driver_data` field) within the raw device ID structure. + /// - The field at `data_offset` must be correctly sized to hold a `usize`. + const unsafe fn build(ids: [(T, U); N], data_offset: Option<usize>) -> Self { let mut raw_ids = [const { MaybeUninit::<T::RawType>::uninit() }; N]; let mut infos = [const { MaybeUninit::uninit() }; N]; @@ -77,14 +94,16 @@ impl<T: RawDeviceId, U, const N: usize> IdArray<T, U, N> { // SAFETY: by the safety requirement of `RawDeviceId`, we're guaranteed that `T` is // layout-wise compatible with `RawType`. raw_ids[i] = unsafe { core::mem::transmute_copy(&ids[i].0) }; - // SAFETY: by the safety requirement of `RawDeviceId`, this would be effectively - // `raw_ids[i].driver_data = i;`. - unsafe { - raw_ids[i] - .as_mut_ptr() - .byte_offset(T::DRIVER_DATA_OFFSET as _) - .cast::<usize>() - .write(i); + if let Some(data_offset) = data_offset { + // SAFETY: by the safety requirement of this function, this would be effectively + // `raw_ids[i].driver_data = i;`. + unsafe { + raw_ids[i] + .as_mut_ptr() + .byte_add(data_offset) + .cast::<usize>() + .write(i); + } } // SAFETY: this is effectively a move: `infos[i] = ids[i].1`. We make a copy here but @@ -109,12 +128,34 @@ impl<T: RawDeviceId, U, const N: usize> IdArray<T, U, N> { } } + /// Creates a new instance of the array without writing index values. + /// + /// The contents are derived from the given identifiers and context information. + /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead. + pub const fn new_without_index(ids: [(T, U); N]) -> Self { + // SAFETY: Calling `Self::build` with `offset = None` is always safe, + // because no raw memory writes are performed in this case. + unsafe { Self::build(ids, None) } + } + /// Reference to the contained [`RawIdArray`]. pub const fn raw_ids(&self) -> &RawIdArray<T, N> { &self.raw_ids } } +impl<T: RawDeviceId + RawDeviceIdIndex, U, const N: usize> IdArray<T, U, N> { + /// Creates a new instance of the array. + /// + /// The contents are derived from the given identifiers and context information. + pub const fn new(ids: [(T, U); N]) -> Self { + // SAFETY: by the safety requirement of `RawDeviceIdIndex`, + // `T::DRIVER_DATA_OFFSET` is guaranteed to be the correct offset (in bytes) to + // a field within `T::RawType`. + unsafe { Self::build(ids, Some(T::DRIVER_DATA_OFFSET)) } + } +} + /// A device id table. /// /// This trait is only implemented by `IdArray`. @@ -136,7 +177,7 @@ impl<T: RawDeviceId, U, const N: usize> IdTable<T, U> for IdArray<T, U, N> { fn as_ptr(&self) -> *const T::RawType { // This cannot be `self.ids.as_ptr()`, as the return pointer must have correct provenance // to access the sentinel. - (self as *const Self).cast() + core::ptr::from_ref(self).cast() } fn id(&self, index: usize) -> &T::RawType { diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 0f79a2ec9474..da18091143a6 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -9,29 +9,36 @@ use crate::{ alloc::Flags, bindings, device::{Bound, Device}, - error::{Error, Result}, + error::{to_result, Error, Result}, ffi::c_void, prelude::*, - revocable::Revocable, - sync::Arc, - types::ARef, + revocable::{Revocable, RevocableGuard}, + sync::{rcu, Completion}, + types::{ARef, ForeignOwnable, Opaque, ScopeGuard}, }; -use core::ops::Deref; +use pin_init::Wrapper; +/// [`Devres`] inner data accessed from [`Devres::callback`]. #[pin_data] -struct DevresInner<T> { - dev: ARef<Device>, - callback: unsafe extern "C" fn(*mut c_void), +struct Inner<T: Send> { #[pin] data: Revocable<T>, + /// Tracks whether [`Devres::callback`] has been completed. + #[pin] + devm: Completion, + /// Tracks whether revoking [`Self::data`] has been completed. + #[pin] + revoke: Completion, } /// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to /// manage their lifetime. /// /// [`Device`] bound resources should be freed when either the resource goes out of scope or the -/// [`Device`] is unbound respectively, depending on what happens first. +/// [`Device`] is unbound respectively, depending on what happens first. In any case, it is always +/// guaranteed that revoking the device resource is completed before the corresponding [`Device`] +/// is unbound. /// /// To achieve that [`Devres`] registers a devres callback on creation, which is called once the /// [`Device`] is unbound, revoking access to the encapsulated resource (see also [`Revocable`]). @@ -42,10 +49,10 @@ struct DevresInner<T> { /// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s /// [`Drop`] implementation. /// -/// # Example +/// # Examples /// /// ```no_run -/// # use kernel::{bindings, c_str, device::{Bound, Device}, devres::Devres, io::{Io, IoRaw}}; +/// # use kernel::{bindings, device::{Bound, Device}, devres::Devres, io::{Io, IoRaw}}; /// # use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. @@ -59,19 +66,19 @@ struct DevresInner<T> { /// unsafe fn new(paddr: usize) -> Result<Self>{ /// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is /// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as _, SIZE as _) }; +/// let addr = unsafe { bindings::ioremap(paddr as bindings::phys_addr_t, SIZE) }; /// if addr.is_null() { /// return Err(ENOMEM); /// } /// -/// Ok(IoMem(IoRaw::new(addr as _, SIZE)?)) +/// Ok(IoMem(IoRaw::new(addr as usize, SIZE)?)) /// } /// } /// /// impl<const SIZE: usize> Drop for IoMem<SIZE> { /// fn drop(&mut self) { /// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. -/// unsafe { bindings::iounmap(self.0.addr() as _); }; +/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); }; /// } /// } /// @@ -86,100 +93,120 @@ struct DevresInner<T> { /// # fn no_run(dev: &Device<Bound>) -> Result<(), Error> { /// // SAFETY: Invalid usage for example purposes. /// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? }; -/// let devres = Devres::new(dev, iomem, GFP_KERNEL)?; +/// let devres = KBox::pin_init(Devres::new(dev, iomem), GFP_KERNEL)?; /// /// let res = devres.try_access().ok_or(ENXIO)?; /// res.write8(0x42, 0x0); /// # Ok(()) /// # } /// ``` -pub struct Devres<T>(Arc<DevresInner<T>>); - -impl<T> DevresInner<T> { - fn new(dev: &Device<Bound>, data: T, flags: Flags) -> Result<Arc<DevresInner<T>>> { - let inner = Arc::pin_init( - pin_init!( DevresInner { - dev: dev.into(), - callback: Self::devres_callback, - data <- Revocable::new(data), - }), - flags, - )?; - - // Convert `Arc<DevresInner>` into a raw pointer and make devres own this reference until - // `Self::devres_callback` is called. - let data = inner.clone().into_raw(); - - // SAFETY: `devm_add_action` guarantees to call `Self::devres_callback` once `dev` is - // detached. - let ret = - unsafe { bindings::devm_add_action(dev.as_raw(), Some(inner.callback), data as _) }; - - if ret != 0 { - // SAFETY: We just created another reference to `inner` in order to pass it to - // `bindings::devm_add_action`. If `bindings::devm_add_action` fails, we have to drop - // this reference accordingly. - let _ = unsafe { Arc::from_raw(data) }; - return Err(Error::from_errno(ret)); - } +/// +/// # Invariants +/// +/// [`Self::inner`] is guaranteed to be initialized and is always accessed read-only. +#[pin_data(PinnedDrop)] +pub struct Devres<T: Send> { + dev: ARef<Device>, + /// Pointer to [`Self::devres_callback`]. + /// + /// Has to be stored, since Rust does not guarantee to always return the same address for a + /// function. However, the C API uses the address as a key. + callback: unsafe extern "C" fn(*mut c_void), + /// Contains all the fields shared with [`Self::callback`]. + // TODO: Replace with `UnsafePinned`, once available. + // + // Subsequently, the `drop_in_place()` in `Devres::drop` and the explicit `Send` and `Sync' + // impls can be removed. + #[pin] + inner: Opaque<Inner<T>>, +} - Ok(inner) - } +impl<T: Send> Devres<T> { + /// Creates a new [`Devres`] instance of the given `data`. + /// + /// The `data` encapsulated within the returned `Devres` instance' `data` will be + /// (revoked)[`Revocable`] once the device is detached. + pub fn new<'a, E>( + dev: &'a Device<Bound>, + data: impl PinInit<T, E> + 'a, + ) -> impl PinInit<Self, Error> + 'a + where + T: 'a, + Error: From<E>, + { + let callback = Self::devres_callback; + + try_pin_init!(&this in Self { + dev: dev.into(), + callback, + // INVARIANT: `inner` is properly initialized. + inner <- { + // SAFETY: `this` is a valid pointer to uninitialized memory. + let inner = unsafe { &raw mut (*this.as_ptr()).inner }; - fn as_ptr(&self) -> *const Self { - self as _ + // SAFETY: + // - `dev.as_raw()` is a pointer to a valid bound device. + // - `inner` is guaranteed to be a valid for the duration of the lifetime of `Self`. + // - `devm_add_action()` is guaranteed not to call `callback` until `this` has been + // properly initialized, because we require `dev` (i.e. the *bound* device) to + // live at least as long as the returned `impl PinInit<Self, Error>`. + to_result(unsafe { + bindings::devm_add_action(dev.as_raw(), Some(callback), inner.cast()) + })?; + + Opaque::pin_init(try_pin_init!(Inner { + devm <- Completion::new(), + revoke <- Completion::new(), + data <- Revocable::new(data), + })) + }, + }) } - fn remove_action(this: &Arc<Self>) { - // SAFETY: - // - `self.inner.dev` is a valid `Device`, - // - the `action` and `data` pointers are the exact same ones as given to devm_add_action() - // previously, - // - `self` is always valid, even if the action has been released already. - let ret = unsafe { - bindings::devm_remove_action_nowarn( - this.dev.as_raw(), - Some(this.callback), - this.as_ptr() as _, - ) - }; + fn inner(&self) -> &Inner<T> { + // SAFETY: By the type invairants of `Self`, `inner` is properly initialized and always + // accessed read-only. + unsafe { &*self.inner.get() } + } - if ret == 0 { - // SAFETY: We leaked an `Arc` reference to devm_add_action() in `DevresInner::new`; if - // devm_remove_action_nowarn() was successful we can (and have to) claim back ownership - // of this reference. - let _ = unsafe { Arc::from_raw(this.as_ptr()) }; - } + fn data(&self) -> &Revocable<T> { + &self.inner().data } #[allow(clippy::missing_safety_doc)] unsafe extern "C" fn devres_callback(ptr: *mut kernel::ffi::c_void) { - let ptr = ptr as *mut DevresInner<T>; - // Devres owned this memory; now that we received the callback, drop the `Arc` and hence the - // reference. - // SAFETY: Safe, since we leaked an `Arc` reference to devm_add_action() in - // `DevresInner::new`. - let inner = unsafe { Arc::from_raw(ptr) }; - - inner.data.revoke(); - } -} + // SAFETY: In `Self::new` we've passed a valid pointer to `Inner` to `devm_add_action()`, + // hence `ptr` must be a valid pointer to `Inner`. + let inner = unsafe { &*ptr.cast::<Inner<T>>() }; -impl<T> Devres<T> { - /// Creates a new [`Devres`] instance of the given `data`. The `data` encapsulated within the - /// returned `Devres` instance' `data` will be revoked once the device is detached. - pub fn new(dev: &Device<Bound>, data: T, flags: Flags) -> Result<Self> { - let inner = DevresInner::new(dev, data, flags)?; + // Ensure that `inner` can't be used anymore after we signal completion of this callback. + let inner = ScopeGuard::new_with_data(inner, |inner| inner.devm.complete_all()); - Ok(Devres(inner)) + if !inner.data.revoke() { + // If `revoke()` returns false, it means that `Devres::drop` already started revoking + // `data` for us. Hence we have to wait until `Devres::drop` signals that it + // completed revoking `data`. + inner.revoke.wait_for_completion(); + } } - /// Same as [`Devres::new`], but does not return a `Devres` instance. Instead the given `data` - /// is owned by devres and will be revoked / dropped, once the device is detached. - pub fn new_foreign_owned(dev: &Device<Bound>, data: T, flags: Flags) -> Result { - let _ = DevresInner::new(dev, data, flags)?; + fn remove_action(&self) -> bool { + // SAFETY: + // - `self.dev` is a valid `Device`, + // - the `action` and `data` pointers are the exact same ones as given to + // `devm_add_action()` previously, + (unsafe { + bindings::devm_remove_action_nowarn( + self.dev.as_raw(), + Some(self.callback), + core::ptr::from_ref(self.inner()).cast_mut().cast(), + ) + } == 0) + } - Ok(()) + /// Return a reference of the [`Device`] this [`Devres`] instance has been created with. + pub fn device(&self) -> &Device { + &self.dev } /// Obtain `&'a T`, bypassing the [`Revocable`]. @@ -192,7 +219,7 @@ impl<T> Devres<T> { /// An error is returned if `dev` does not match the same [`Device`] this [`Devres`] instance /// has been created with. /// - /// # Example + /// # Examples /// /// ```no_run /// # #![cfg(CONFIG_PCI)] @@ -211,27 +238,123 @@ impl<T> Devres<T> { /// } /// ``` pub fn access<'a>(&'a self, dev: &'a Device<Bound>) -> Result<&'a T> { - if self.0.dev.as_raw() != dev.as_raw() { + if self.dev.as_raw() != dev.as_raw() { return Err(EINVAL); } // SAFETY: `dev` being the same device as the device this `Devres` has been created for - // proves that `self.0.data` hasn't been revoked and is guaranteed to not be revoked as - // long as `dev` lives; `dev` lives at least as long as `self`. - Ok(unsafe { self.deref().access() }) + // proves that `self.data` hasn't been revoked and is guaranteed to not be revoked as long + // as `dev` lives; `dev` lives at least as long as `self`. + Ok(unsafe { self.data().access() }) + } + + /// [`Devres`] accessor for [`Revocable::try_access`]. + pub fn try_access(&self) -> Option<RevocableGuard<'_, T>> { + self.data().try_access() + } + + /// [`Devres`] accessor for [`Revocable::try_access_with`]. + pub fn try_access_with<R, F: FnOnce(&T) -> R>(&self, f: F) -> Option<R> { + self.data().try_access_with(f) + } + + /// [`Devres`] accessor for [`Revocable::try_access_with_guard`]. + pub fn try_access_with_guard<'a>(&'a self, guard: &'a rcu::Guard) -> Option<&'a T> { + self.data().try_access_with_guard(guard) } } -impl<T> Deref for Devres<T> { - type Target = Revocable<T>; +// SAFETY: `Devres` can be send to any task, if `T: Send`. +unsafe impl<T: Send> Send for Devres<T> {} + +// SAFETY: `Devres` can be shared with any task, if `T: Sync`. +unsafe impl<T: Send + Sync> Sync for Devres<T> {} + +#[pinned_drop] +impl<T: Send> PinnedDrop for Devres<T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data + // anymore, hence it is safe not to wait for the grace period to finish. + if unsafe { self.data().revoke_nosync() } { + // We revoked `self.data` before the devres action did, hence try to remove it. + if !self.remove_action() { + // We could not remove the devres action, which means that it now runs concurrently, + // hence signal that `self.data` has been revoked by us successfully. + self.inner().revoke.complete_all(); + + // Wait for `Self::devres_callback` to be done using this object. + self.inner().devm.wait_for_completion(); + } + } else { + // `Self::devres_callback` revokes `self.data` for us, hence wait for it to be done + // using this object. + self.inner().devm.wait_for_completion(); + } - fn deref(&self) -> &Self::Target { - &self.0.data + // INVARIANT: At this point it is guaranteed that `inner` can't be accessed any more. + // + // SAFETY: `inner` is valid for dropping. + unsafe { core::ptr::drop_in_place(self.inner.get()) }; } } -impl<T> Drop for Devres<T> { - fn drop(&mut self) { - DevresInner::remove_action(&self.0); +/// Consume `data` and [`Drop::drop`] `data` once `dev` is unbound. +fn register_foreign<P>(dev: &Device<Bound>, data: P) -> Result +where + P: ForeignOwnable + Send + 'static, +{ + let ptr = data.into_foreign(); + + #[allow(clippy::missing_safety_doc)] + unsafe extern "C" fn callback<P: ForeignOwnable>(ptr: *mut kernel::ffi::c_void) { + // SAFETY: `ptr` is the pointer to the `ForeignOwnable` leaked above and hence valid. + drop(unsafe { P::from_foreign(ptr.cast()) }); } + + // SAFETY: + // - `dev.as_raw()` is a pointer to a valid and bound device. + // - `ptr` is a valid pointer the `ForeignOwnable` devres takes ownership of. + to_result(unsafe { + // `devm_add_action_or_reset()` also calls `callback` on failure, such that the + // `ForeignOwnable` is released eventually. + bindings::devm_add_action_or_reset(dev.as_raw(), Some(callback::<P>), ptr.cast()) + }) +} + +/// Encapsulate `data` in a [`KBox`] and [`Drop::drop`] `data` once `dev` is unbound. +/// +/// # Examples +/// +/// ```no_run +/// use kernel::{device::{Bound, Device}, devres}; +/// +/// /// Registration of e.g. a class device, IRQ, etc. +/// struct Registration; +/// +/// impl Registration { +/// fn new() -> Self { +/// // register +/// +/// Self +/// } +/// } +/// +/// impl Drop for Registration { +/// fn drop(&mut self) { +/// // unregister +/// } +/// } +/// +/// fn from_bound_context(dev: &Device<Bound>) -> Result { +/// devres::register(dev, Registration::new(), GFP_KERNEL) +/// } +/// ``` +pub fn register<T, E>(dev: &Device<Bound>, data: impl PinInit<T, E>, flags: Flags) -> Result +where + T: Send + 'static, + Error: From<E>, +{ + let data = KBox::pin_init(data, flags)?; + + register_foreign(dev, data) } diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index a33261c62e0c..2bc8ab51ec28 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -5,14 +5,156 @@ //! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) use crate::{ - bindings, build_assert, - device::{Bound, Device}, - error::code::*, - error::Result, + bindings, build_assert, device, + device::{Bound, Core}, + error::{to_result, Result}, + prelude::*, transmute::{AsBytes, FromBytes}, types::ARef, }; +/// Trait to be implemented by DMA capable bus devices. +/// +/// The [`dma::Device`](Device) trait should be implemented by bus specific device representations, +/// where the underlying bus is DMA capable, such as [`pci::Device`](::kernel::pci::Device) or +/// [`platform::Device`](::kernel::platform::Device). +pub trait Device: AsRef<device::Device<Core>> { + /// Set up the device's DMA streaming addressing capabilities. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`CoherentAllocation::alloc_attrs`]. + unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) }) + } + + /// Set up the device's DMA coherent addressing capabilities. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`CoherentAllocation::alloc_attrs`]. + unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) }) + } + + /// Set up the device's DMA addressing capabilities. + /// + /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`]. + /// + /// This method is usually called once from `probe()` as soon as the device capabilities are + /// known. + /// + /// # Safety + /// + /// This method must not be called concurrently with any DMA allocation or mapping primitives, + /// such as [`CoherentAllocation::alloc_attrs`]. + unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result { + // SAFETY: + // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid. + // - The safety requirement of this function guarantees that there are no concurrent calls + // to DMA allocation and mapping primitives using this mask. + to_result(unsafe { + bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value()) + }) + } +} + +/// A DMA mask that holds a bitmask with the lowest `n` bits set. +/// +/// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values +/// are guaranteed to never exceed the bit width of `u64`. +/// +/// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DmaMask(u64); + +impl DmaMask { + /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. + /// + /// For `n <= 64`, sets exactly the lowest `n` bits. + /// For `n > 64`, results in a build error. + /// + /// # Examples + /// + /// ``` + /// use kernel::dma::DmaMask; + /// + /// let mask0 = DmaMask::new::<0>(); + /// assert_eq!(mask0.value(), 0); + /// + /// let mask1 = DmaMask::new::<1>(); + /// assert_eq!(mask1.value(), 0b1); + /// + /// let mask64 = DmaMask::new::<64>(); + /// assert_eq!(mask64.value(), u64::MAX); + /// + /// // Build failure. + /// // let mask_overflow = DmaMask::new::<100>(); + /// ``` + #[inline] + pub const fn new<const N: u32>() -> Self { + let Ok(mask) = Self::try_new(N) else { + build_error!("Invalid DMA Mask."); + }; + + mask + } + + /// Constructs a `DmaMask` with the lowest `n` bits set to `1`. + /// + /// For `n <= 64`, sets exactly the lowest `n` bits. + /// For `n > 64`, returns [`EINVAL`]. + /// + /// # Examples + /// + /// ``` + /// use kernel::dma::DmaMask; + /// + /// let mask0 = DmaMask::try_new(0)?; + /// assert_eq!(mask0.value(), 0); + /// + /// let mask1 = DmaMask::try_new(1)?; + /// assert_eq!(mask1.value(), 0b1); + /// + /// let mask64 = DmaMask::try_new(64)?; + /// assert_eq!(mask64.value(), u64::MAX); + /// + /// let mask_overflow = DmaMask::try_new(100); + /// assert!(mask_overflow.is_err()); + /// # Ok::<(), Error>(()) + /// ``` + #[inline] + pub const fn try_new(n: u32) -> Result<Self> { + Ok(Self(match n { + 0 => 0, + 1..=64 => u64::MAX >> (64 - n), + _ => return Err(EINVAL), + })) + } + + /// Returns the underlying `u64` bitmask value. + #[inline] + pub const fn value(&self) -> u64 { + self.0 + } +} + /// Possible attributes associated with a DMA mapping. /// /// They can be combined with the operators `|`, `&`, and `!`. @@ -38,7 +180,7 @@ pub struct Attrs(u32); impl Attrs { /// Get the raw representation of this attribute. pub(crate) fn as_raw(self) -> crate::ffi::c_ulong { - self.0 as _ + self.0 as crate::ffi::c_ulong } /// Check whether `flags` is contained in `self`. @@ -89,7 +231,7 @@ pub mod attrs { /// Forces contiguous allocation of the buffer in physical memory. pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS); - /// This is a hint to the DMA-mapping subsystem that it's probably not worth the time to try + /// Hints DMA-mapping subsystem that it's probably not worth the time to try /// to allocate memory to in a way that gives better TLB efficiency. pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES); @@ -97,7 +239,7 @@ pub mod attrs { /// `__GFP_NOWARN`). pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN); - /// Used to indicate that the buffer is fully accessible at an elevated privilege level (and + /// Indicates that the buffer is fully accessible at an elevated privilege level (and /// ideally inaccessible or at least read-only at lesser-privileged levels). pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED); } @@ -105,7 +247,7 @@ pub mod attrs { /// An abstraction of the `dma_alloc_coherent` API. /// /// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map -/// large consistent DMA regions. +/// large coherent DMA regions. /// /// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the /// processor's virtual address space) and the device address which can be given to the device @@ -114,9 +256,11 @@ pub mod attrs { /// /// # Invariants /// -/// For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer -/// to an allocated region of consistent memory and `dma_handle` is the DMA address base of -/// the region. +/// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer +/// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the +/// region. +/// - The size in bytes of the allocation is equal to `size_of::<T> * count`. +/// - `size_of::<T> * count` fits into a `usize`. // TODO // // DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness @@ -130,7 +274,7 @@ pub mod attrs { // Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the // entire `CoherentAllocation` including the allocated memory itself. pub struct CoherentAllocation<T: AsBytes + FromBytes> { - dev: ARef<Device>, + dev: ARef<device::Device>, dma_handle: bindings::dma_addr_t, count: usize, cpu_addr: *mut T, @@ -138,7 +282,7 @@ pub struct CoherentAllocation<T: AsBytes + FromBytes> { } impl<T: AsBytes + FromBytes> CoherentAllocation<T> { - /// Allocates a region of `size_of::<T> * count` of consistent memory. + /// Allocates a region of `size_of::<T> * count` of coherent memory. /// /// # Examples /// @@ -152,7 +296,7 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { /// # Ok::<(), Error>(()) } /// ``` pub fn alloc_attrs( - dev: &Device<Bound>, + dev: &device::Device<Bound>, count: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -179,14 +323,17 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { if ret.is_null() { return Err(ENOMEM); } - // INVARIANT: We just successfully allocated a coherent region which is accessible for - // `count` elements, hence the cpu address is valid. We also hold a refcounted reference - // to the device. + // INVARIANT: + // - We just successfully allocated a coherent region which is accessible for + // `count` elements, hence the cpu address is valid. We also hold a refcounted reference + // to the device. + // - The allocated `size` is equal to `size_of::<T> * count`. + // - The allocated `size` fits into a `usize`. Ok(Self { dev: dev.into(), dma_handle, count, - cpu_addr: ret as *mut T, + cpu_addr: ret.cast::<T>(), dma_attrs, }) } @@ -194,13 +341,28 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the /// `dma_attrs` is 0 by default. pub fn alloc_coherent( - dev: &Device<Bound>, + dev: &device::Device<Bound>, count: usize, gfp_flags: kernel::alloc::Flags, ) -> Result<CoherentAllocation<T>> { CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0)) } + /// Returns the number of elements `T` in this allocation. + /// + /// Note that this is not the size of the allocation in bytes, which is provided by + /// [`Self::size`]. + pub fn count(&self) -> usize { + self.count + } + + /// Returns the size in bytes of this allocation. + pub fn size(&self) -> usize { + // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits into + // a `usize`. + self.count * core::mem::size_of::<T>() + } + /// Returns the base address to the allocated region in the CPU's virtual address space. pub fn start_ptr(&self) -> *const T { self.cpu_addr @@ -212,12 +374,113 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { self.cpu_addr } - /// Returns a DMA handle which may given to the device as the DMA address base of + /// Returns a DMA handle which may be given to the device as the DMA address base of /// the region. pub fn dma_handle(&self) -> bindings::dma_addr_t { self.dma_handle } + /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the + /// device as the DMA address base of the region. + /// + /// Returns `EINVAL` if `offset` is not within the bounds of the allocation. + pub fn dma_handle_with_offset(&self, offset: usize) -> Result<bindings::dma_addr_t> { + if offset >= self.count { + Err(EINVAL) + } else { + // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits + // into a `usize`, and `offset` is inferior to `count`. + Ok(self.dma_handle + (offset * core::mem::size_of::<T>()) as bindings::dma_addr_t) + } + } + + /// Common helper to validate a range applied from the allocated region in the CPU's virtual + /// address space. + fn validate_range(&self, offset: usize, count: usize) -> Result { + if offset.checked_add(count).ok_or(EOVERFLOW)? > self.count { + return Err(EINVAL); + } + Ok(()) + } + + /// Returns the data from the region starting from `offset` as a slice. + /// `offset` and `count` are in units of `T`, not the number of bytes. + /// + /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed, + /// [`CoherentAllocation::start_ptr`] or [`CoherentAllocation::start_ptr_mut`] could be used + /// instead. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a write to the same region while + /// the returned slice is live. + pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> { + self.validate_range(offset, count)?; + // SAFETY: + // - The pointer is valid due to type invariant on `CoherentAllocation`, + // we've just checked that the range and index is within bounds. The immutability of the + // data is also guaranteed by the safety requirements of the function. + // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked + // that `self.count` won't overflow early in the constructor. + Ok(unsafe { core::slice::from_raw_parts(self.cpu_addr.add(offset), count) }) + } + + /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable + /// slice is returned. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a read or write to the same region + /// while the returned slice is live. + pub unsafe fn as_slice_mut(&mut self, offset: usize, count: usize) -> Result<&mut [T]> { + self.validate_range(offset, count)?; + // SAFETY: + // - The pointer is valid due to type invariant on `CoherentAllocation`, + // we've just checked that the range and index is within bounds. The immutability of the + // data is also guaranteed by the safety requirements of the function. + // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked + // that `self.count` won't overflow early in the constructor. + Ok(unsafe { core::slice::from_raw_parts_mut(self.cpu_addr.add(offset), count) }) + } + + /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the + /// number of bytes. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// slice is live. + /// * Callers must ensure that this call does not race with a read or write to the same region + /// that overlaps with this write. + /// + /// # Examples + /// + /// ``` + /// # fn test(alloc: &mut kernel::dma::CoherentAllocation<u8>) -> Result { + /// let somedata: [u8; 4] = [0xf; 4]; + /// let buf: &[u8] = &somedata; + /// // SAFETY: There is no concurrent HW operation on the device and no other R/W access to the + /// // region. + /// unsafe { alloc.write(buf, 0)?; } + /// # Ok::<(), Error>(()) } + /// ``` + pub unsafe fn write(&mut self, src: &[T], offset: usize) -> Result { + self.validate_range(offset, src.len())?; + // SAFETY: + // - The pointer is valid due to type invariant on `CoherentAllocation` + // and we've just checked that the range and index is within bounds. + // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked + // that `self.count` won't overflow early in the constructor. + unsafe { + core::ptr::copy_nonoverlapping(src.as_ptr(), self.cpu_addr.add(offset), src.len()) + }; + Ok(()) + } + /// Returns a pointer to an element from the region with bounds checking. `offset` is in /// units of `T`, not the number of bytes. /// @@ -293,7 +556,7 @@ impl<T: AsBytes + FromBytes> Drop for CoherentAllocation<T> { bindings::dma_free_attrs( self.dev.as_raw(), size, - self.cpu_addr as _, + self.cpu_addr.cast(), self.dma_handle, self.dma_attrs.as_raw(), ) @@ -328,20 +591,24 @@ unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} #[macro_export] macro_rules! dma_read { ($dma:expr, $idx: expr, $($field:tt)*) => {{ - let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; - // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be - // dereferenced. The compiler also further validates the expression on whether `field` - // is a member of `item` when expanded by the macro. - unsafe { - let ptr_field = ::core::ptr::addr_of!((*item) $($field)*); - $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field) - } + (|| -> ::core::result::Result<_, $crate::error::Error> { + let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; + // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be + // dereferenced. The compiler also further validates the expression on whether `field` + // is a member of `item` when expanded by the macro. + unsafe { + let ptr_field = ::core::ptr::addr_of!((*item) $($field)*); + ::core::result::Result::Ok( + $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field) + ) + } + })() }}; ($dma:ident [ $idx:expr ] $($field:tt)* ) => { - $crate::dma_read!($dma, $idx, $($field)*); + $crate::dma_read!($dma, $idx, $($field)*) }; ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => { - $crate::dma_read!($($dma).*, $idx, $($field)*); + $crate::dma_read!($($dma).*, $idx, $($field)*) }; } @@ -368,24 +635,30 @@ macro_rules! dma_read { #[macro_export] macro_rules! dma_write { ($dma:ident [ $idx:expr ] $($field:tt)*) => {{ - $crate::dma_write!($dma, $idx, $($field)*); + $crate::dma_write!($dma, $idx, $($field)*) }}; ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{ - $crate::dma_write!($($dma).*, $idx, $($field)*); + $crate::dma_write!($($dma).*, $idx, $($field)*) }}; ($dma:expr, $idx: expr, = $val:expr) => { - let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; - // SAFETY: `item_from_index` ensures that `item` is always a valid item. - unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) } + (|| -> ::core::result::Result<_, $crate::error::Error> { + let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; + // SAFETY: `item_from_index` ensures that `item` is always a valid item. + unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) } + ::core::result::Result::Ok(()) + })() }; ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => { - let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; - // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be - // dereferenced. The compiler also further validates the expression on whether `field` - // is a member of `item` when expanded by the macro. - unsafe { - let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*); - $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val) - } + (|| -> ::core::result::Result<_, $crate::error::Error> { + let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; + // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be + // dereferenced. The compiler also further validates the expression on whether `field` + // is a member of `item` when expanded by the macro. + unsafe { + let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*); + $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val) + } + ::core::result::Result::Ok(()) + })() }; } diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index ec9166cedfa7..a8f2675ba7a7 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -6,7 +6,7 @@ //! register using the [`Registration`] class. use crate::error::{Error, Result}; -use crate::{device, of, str::CStr, try_pin_init, types::Opaque, ThisModule}; +use crate::{acpi, device, of, str::CStr, try_pin_init, types::Opaque, ThisModule}; use core::pin::Pin; use pin_init::{pin_data, pinned_drop, PinInit}; @@ -141,36 +141,76 @@ pub trait Adapter { /// The type holding driver private data about each device id supported by the driver. type IdInfo: 'static; + /// The [`acpi::IdTable`] of the corresponding driver + fn acpi_id_table() -> Option<acpi::IdTable<Self::IdInfo>>; + + /// Returns the driver's private data from the matching entry in the [`acpi::IdTable`], if any. + /// + /// If this returns `None`, it means there is no match with an entry in the [`acpi::IdTable`]. + fn acpi_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + #[cfg(not(CONFIG_ACPI))] + { + let _ = dev; + None + } + + #[cfg(CONFIG_ACPI)] + { + let table = Self::acpi_id_table()?; + + // SAFETY: + // - `table` has static lifetime, hence it's valid for read, + // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let raw_id = unsafe { bindings::acpi_match_device(table.as_ptr(), dev.as_raw()) }; + + if raw_id.is_null() { + None + } else { + // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct acpi_device_id` + // and does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*raw_id.cast::<acpi::DeviceId>() }; + + Some(table.info(<acpi::DeviceId as crate::device_id::RawDeviceIdIndex>::index(id))) + } + } + } + /// The [`of::IdTable`] of the corresponding driver. fn of_id_table() -> Option<of::IdTable<Self::IdInfo>>; /// Returns the driver's private data from the matching entry in the [`of::IdTable`], if any. /// /// If this returns `None`, it means there is no match with an entry in the [`of::IdTable`]. - #[cfg(CONFIG_OF)] fn of_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - let table = Self::of_id_table()?; - - // SAFETY: - // - `table` has static lifetime, hence it's valid for read, - // - `dev` is guaranteed to be valid while it's alive, and so is `pdev.as_ref().as_raw()`. - let raw_id = unsafe { bindings::of_match_device(table.as_ptr(), dev.as_raw()) }; - - if raw_id.is_null() { + #[cfg(not(CONFIG_OF))] + { + let _ = dev; None - } else { - // SAFETY: `DeviceId` is a `#[repr(transparent)` wrapper of `struct of_device_id` and - // does not add additional invariants, so it's safe to transmute. - let id = unsafe { &*raw_id.cast::<of::DeviceId>() }; - - Some(table.info(<of::DeviceId as crate::device_id::RawDeviceId>::index(id))) } - } - #[cfg(not(CONFIG_OF))] - #[allow(missing_docs)] - fn of_id_info(_dev: &device::Device) -> Option<&'static Self::IdInfo> { - None + #[cfg(CONFIG_OF)] + { + let table = Self::of_id_table()?; + + // SAFETY: + // - `table` has static lifetime, hence it's valid for read, + // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let raw_id = unsafe { bindings::of_match_device(table.as_ptr(), dev.as_raw()) }; + + if raw_id.is_null() { + None + } else { + // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` + // and does not add additional invariants, so it's safe to transmute. + let id = unsafe { &*raw_id.cast::<of::DeviceId>() }; + + Some( + table.info(<of::DeviceId as crate::device_id::RawDeviceIdIndex>::index( + id, + )), + ) + } + } } /// Returns the driver's private data from the matching entry of any of the ID tables, if any. @@ -178,6 +218,11 @@ pub trait Adapter { /// If this returns `None`, it means that there is no match in any of the ID tables directly /// associated with a [`device::Device`]. fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + let id = Self::acpi_id_info(dev); + if id.is_some() { + return id; + } + let id = Self::of_id_info(dev); if id.is_some() { return id; diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 624d7a4c83ea..3bb7c83966cf 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -66,7 +66,7 @@ impl<T: drm::Driver> Device<T> { open: Some(drm::File::<T::File>::open_callback), postclose: Some(drm::File::<T::File>::postclose_callback), unload: None, - release: None, + release: Some(Self::release), master_set: None, master_drop: None, debugfs_init: None, @@ -83,13 +83,13 @@ impl<T: drm::Driver> Device<T> { major: T::INFO.major, minor: T::INFO.minor, patchlevel: T::INFO.patchlevel, - name: T::INFO.name.as_char_ptr() as *mut _, - desc: T::INFO.desc.as_char_ptr() as *mut _, + name: T::INFO.name.as_char_ptr().cast_mut(), + desc: T::INFO.desc.as_char_ptr().cast_mut(), driver_features: drm::driver::FEAT_GEM, ioctls: T::IOCTLS.as_ptr(), num_ioctls: T::IOCTLS.len() as i32, - fops: &Self::GEM_FOPS as _, + fops: &Self::GEM_FOPS, }; const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(); @@ -135,11 +135,9 @@ impl<T: drm::Driver> Device<T> { /// /// `ptr` must be a valid pointer to a `struct device` embedded in `Self`. unsafe fn from_drm_device(ptr: *const bindings::drm_device) -> *mut Self { - let ptr: *const Opaque<bindings::drm_device> = ptr.cast(); - // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a // `struct drm_device` embedded in `Self`. - unsafe { crate::container_of!(ptr, Self, dev) }.cast_mut() + unsafe { crate::container_of!(Opaque::cast_from(ptr), Self, dev) }.cast_mut() } /// Not intended to be called externally, except via declare_drm_ioctls!() @@ -154,7 +152,7 @@ impl<T: drm::Driver> Device<T> { /// Additionally, callers must ensure that the `struct device`, `ptr` is pointing to, is /// embedded in `Self`. #[doc(hidden)] - pub unsafe fn as_ref<'a>(ptr: *const bindings::drm_device) -> &'a Self { + pub unsafe fn from_raw<'a>(ptr: *const bindings::drm_device) -> &'a Self { // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a // `struct drm_device` embedded in `Self`. let ptr = unsafe { Self::from_drm_device(ptr) }; @@ -162,6 +160,16 @@ impl<T: drm::Driver> Device<T> { // SAFETY: `ptr` is valid by the safety requirements of this function. unsafe { &*ptr.cast() } } + + extern "C" fn release(ptr: *mut bindings::drm_device) { + // SAFETY: `ptr` is a valid pointer to a `struct drm_device` and embedded in `Self`. + let this = unsafe { Self::from_drm_device(ptr) }; + + // SAFETY: + // - When `release` runs it is guaranteed that there is no further access to `this`. + // - `this` is valid for dropping. + unsafe { core::ptr::drop_in_place(this) }; + } } impl<T: drm::Driver> Deref for Device<T> { @@ -190,7 +198,7 @@ impl<T: drm::Driver> AsRef<device::Device> for Device<T> { fn as_ref(&self) -> &device::Device { // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid, // which is guaranteed by the type invariant. - unsafe { device::Device::as_ref((*self.as_raw()).dev) } + unsafe { device::Device::from_raw((*self.as_raw()).dev) } } } diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index acb638086131..fe7e8d06961a 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -5,12 +5,9 @@ //! C header: [`include/linux/drm/drm_drv.h`](srctree/include/linux/drm/drm_drv.h) use crate::{ - bindings, device, - devres::Devres, - drm, + bindings, device, devres, drm, error::{to_result, Result}, prelude::*, - str::CStr, types::ARef, }; use macros::vtable; @@ -130,18 +127,22 @@ impl<T: Driver> Registration<T> { } /// Same as [`Registration::new`}, but transfers ownership of the [`Registration`] to - /// [`Devres`]. + /// [`devres::register`]. pub fn new_foreign_owned( drm: &drm::Device<T>, dev: &device::Device<device::Bound>, flags: usize, - ) -> Result { + ) -> Result + where + T: 'static, + { if drm.as_ref().as_raw() != dev.as_raw() { return Err(EINVAL); } let reg = Registration::<T>::new(drm, flags)?; - Devres::new_foreign_owned(dev, reg, GFP_KERNEL) + + devres::register(dev, reg, GFP_KERNEL) } /// Returns a reference to the `Device` instance for this registration. diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index b9527705e551..e8789c9110d6 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -32,7 +32,7 @@ impl<T: DriverFile> File<T> { /// # Safety /// /// `raw_file` must be a valid pointer to an open `struct drm_file`, opened through `T::open`. - pub unsafe fn as_ref<'a>(ptr: *mut bindings::drm_file) -> &'a File<T> { + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_file) -> &'a File<T> { // SAFETY: `raw_file` is valid by the safety requirements of this function. unsafe { &*ptr.cast() } } @@ -61,10 +61,10 @@ impl<T: DriverFile> File<T> { // SAFETY: A callback from `struct drm_driver::open` guarantees that // - `raw_dev` is valid pointer to a `struct drm_device`, // - the corresponding `struct drm_device` has been registered. - let drm = unsafe { drm::Device::as_ref(raw_dev) }; + let drm = unsafe { drm::Device::from_raw(raw_dev) }; // SAFETY: `raw_file` is a valid pointer to a `struct drm_file`. - let file = unsafe { File::<T>::as_ref(raw_file) }; + let file = unsafe { File::<T>::from_raw(raw_file) }; let inner = match T::open(drm) { Err(e) => { @@ -89,7 +89,7 @@ impl<T: DriverFile> File<T> { raw_file: *mut bindings::drm_file, ) { // SAFETY: This reference won't escape this function - let file = unsafe { File::<T>::as_ref(raw_file) }; + let file = unsafe { File::<T>::from_raw(raw_file) }; // SAFETY: `file.driver_priv` has been created in `open_callback` through `KBox::into_raw`. let _ = unsafe { KBox::from_raw(file.driver_priv()) }; diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 4cd69fa84318..b71821cfb5ea 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -51,7 +51,7 @@ pub trait IntoGEMObject: Sized + super::private::Sealed + AlwaysRefCounted { /// - `self_ptr` must be a valid pointer to `Self`. /// - The caller promises that holding the immutable reference returned by this function does /// not violate rust's data aliasing rules and remains valid throughout the lifetime of `'a`. - unsafe fn as_ref<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self; + unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self; } // SAFETY: All gem objects are refcounted. @@ -86,12 +86,12 @@ extern "C" fn open_callback<T: BaseDriverObject<U>, U: BaseObject>( ) -> core::ffi::c_int { // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`. let file = unsafe { - drm::File::<<<U as IntoGEMObject>::Driver as drm::Driver>::File>::as_ref(raw_file) + drm::File::<<<U as IntoGEMObject>::Driver as drm::Driver>::File>::from_raw(raw_file) }; // SAFETY: `open_callback` is specified in the AllocOps structure for `Object<T>`, ensuring that // `raw_obj` is indeed contained within a `Object<T>`. let obj = unsafe { - <<<U as IntoGEMObject>::Driver as drm::Driver>::Object as IntoGEMObject>::as_ref(raw_obj) + <<<U as IntoGEMObject>::Driver as drm::Driver>::Object as IntoGEMObject>::from_raw(raw_obj) }; match T::open(obj, file) { @@ -106,12 +106,12 @@ extern "C" fn close_callback<T: BaseDriverObject<U>, U: BaseObject>( ) { // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`. let file = unsafe { - drm::File::<<<U as IntoGEMObject>::Driver as drm::Driver>::File>::as_ref(raw_file) + drm::File::<<<U as IntoGEMObject>::Driver as drm::Driver>::File>::from_raw(raw_file) }; // SAFETY: `close_callback` is specified in the AllocOps structure for `Object<T>`, ensuring // that `raw_obj` is indeed contained within a `Object<T>`. let obj = unsafe { - <<<U as IntoGEMObject>::Driver as drm::Driver>::Object as IntoGEMObject>::as_ref(raw_obj) + <<<U as IntoGEMObject>::Driver as drm::Driver>::Object as IntoGEMObject>::from_raw(raw_obj) }; T::close(obj, file); @@ -124,12 +124,10 @@ impl<T: DriverObject> IntoGEMObject for Object<T> { self.obj.get() } - unsafe fn as_ref<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self { - let self_ptr: *mut Opaque<bindings::drm_gem_object> = self_ptr.cast(); - + unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self { // SAFETY: `obj` is guaranteed to be in an `Object<T>` via the safety contract of this // function - unsafe { &*crate::container_of!(self_ptr, Object<T>, obj) } + unsafe { &*crate::container_of!(Opaque::cast_from(self_ptr), Object<T>, obj) } } } @@ -170,9 +168,9 @@ pub trait BaseObject: IntoGEMObject { // - A `drm::Driver` can only have a single `File` implementation. // - `file` uses the same `drm::Driver` as `Self`. // - Therefore, we're guaranteed that `ptr` must be a gem object embedded within `Self`. - // - And we check if the pointer is null befoe calling as_ref(), ensuring that `ptr` is a + // - And we check if the pointer is null befoe calling from_raw(), ensuring that `ptr` is a // valid pointer to an initialized `Self`. - let obj = unsafe { Self::as_ref(ptr) }; + let obj = unsafe { Self::from_raw(ptr) }; // SAFETY: // - We take ownership of the reference of `drm_gem_object_lookup()`. diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index 445639404fb7..fdec01c37168 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -134,7 +134,7 @@ macro_rules! declare_drm_ioctls { // FIXME: Currently there is nothing enforcing that the types of the // dev/file match the current driver these ioctls are being declared // for, and it's not clear how to enforce this within the type system. - let dev = $crate::drm::device::Device::as_ref(raw_dev); + let dev = $crate::drm::device::Device::from_raw(raw_dev); // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we // asserted above matches the size of this type, and all bit patterns of // UAPI structs must be valid. @@ -142,7 +142,7 @@ macro_rules! declare_drm_ioctls { &*(raw_data as *const $crate::types::Opaque<$crate::uapi::$struct>) }; // SAFETY: This is just the DRM file structure - let file = unsafe { $crate::drm::File::as_ref(raw_file) }; + let file = unsafe { $crate::drm::File::from_raw(raw_file) }; match $func(dev, data, file) { Err(e) => e.to_errno(), diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 3dee3139fcd4..a41de293dcd1 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -6,10 +6,10 @@ use crate::{ alloc::{layout::LayoutError, AllocError}, + fmt, str::CStr, }; -use core::fmt; use core::num::NonZeroI32; use core::num::TryFromIntError; use core::str::Utf8Error; @@ -65,6 +65,7 @@ pub mod code { declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); declare_err!(EOVERFLOW, "Value too large for defined data type."); + declare_err!(ETIMEDOUT, "Connection timed out."); declare_err!(ERESTARTSYS, "Restart the system call."); declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); declare_err!(ERESTARTNOHAND, "Restart if no handler."); @@ -153,7 +154,7 @@ impl Error { /// Returns the error encoded as a pointer. pub fn to_ptr<T>(self) -> *mut T { // SAFETY: `self.0` is a valid error due to its invariant. - unsafe { bindings::ERR_PTR(self.0.get() as _) as *mut _ } + unsafe { bindings::ERR_PTR(self.0.get() as crate::ffi::c_long).cast() } } /// Returns a string representing the error, if one exists. @@ -188,7 +189,7 @@ impl fmt::Debug for Error { Some(name) => f .debug_tuple( // SAFETY: These strings are ASCII-only. - unsafe { core::str::from_utf8_unchecked(name) }, + unsafe { core::str::from_utf8_unchecked(name.to_bytes()) }, ) .finish(), } @@ -219,8 +220,8 @@ impl From<LayoutError> for Error { } } -impl From<core::fmt::Error> for Error { - fn from(_: core::fmt::Error) -> Error { +impl From<fmt::Error> for Error { + fn from(_: fmt::Error) -> Error { code::EINVAL } } diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs index 8a50fcd4c9bb..7a906099993f 100644 --- a/rust/kernel/faux.rs +++ b/rust/kernel/faux.rs @@ -54,7 +54,7 @@ impl AsRef<device::Device> for Registration { fn as_ref(&self) -> &device::Device { // SAFETY: The underlying `device` in `faux_device` is guaranteed by the C API to be // a valid initialized `device`. - unsafe { device::Device::as_ref(addr_of_mut!((*self.as_raw()).dev)) } + unsafe { device::Device::from_raw(addr_of_mut!((*self.as_raw()).dev)) } } } diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs index 2494c96e105f..1abab5b2f052 100644 --- a/rust/kernel/firmware.rs +++ b/rust/kernel/firmware.rs @@ -62,10 +62,11 @@ impl Firmware { fn request_internal(name: &CStr, dev: &Device, func: FwFunc) -> Result<Self> { let mut fw: *mut bindings::firmware = core::ptr::null_mut(); let pfw: *mut *mut bindings::firmware = &mut fw; + let pfw: *mut *const bindings::firmware = pfw.cast(); // SAFETY: `pfw` is a valid pointer to a NULL initialized `bindings::firmware` pointer. // `name` and `dev` are valid as by their type invariants. - let ret = unsafe { func.0(pfw as _, name.as_char_ptr(), dev.as_raw()) }; + let ret = unsafe { func.0(pfw, name.as_char_ptr(), dev.as_raw()) }; if ret != 0 { return Err(Error::from_errno(ret)); } @@ -139,7 +140,7 @@ unsafe impl Sync for Firmware {} /// Typically, such contracts would be enforced by a trait, however traits do not (yet) support /// const functions. /// -/// # Example +/// # Examples /// /// ``` /// # mod module_firmware_test { @@ -181,7 +182,7 @@ unsafe impl Sync for Firmware {} /// module! { /// type: MyModule, /// name: "module_firmware_test", -/// author: "Rust for Linux", +/// authors: ["Rust for Linux"], /// description: "module_firmware! test module", /// license: "GPL", /// } @@ -202,7 +203,7 @@ macro_rules! module_firmware { }; #[link_section = ".modinfo"] - #[used] + #[used(compiler)] static __MODULE_FIRMWARE: [u8; $($builder)*::create(__MODULE_FIRMWARE_PREFIX) .build_length()] = $($builder)*::create(__MODULE_FIRMWARE_PREFIX).build(); }; @@ -261,7 +262,7 @@ impl<const N: usize> ModInfoBuilder<N> { /// Append path components to the [`ModInfoBuilder`] instance. Paths need to be separated /// with [`ModInfoBuilder::new_entry`]. /// - /// # Example + /// # Examples /// /// ``` /// use kernel::firmware::ModInfoBuilder; diff --git a/rust/kernel/fmt.rs b/rust/kernel/fmt.rs new file mode 100644 index 000000000000..0306e8388968 --- /dev/null +++ b/rust/kernel/fmt.rs @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Formatting utilities. +//! +//! This module is intended to be used in place of `core::fmt` in kernel code. + +pub use core::fmt::{Arguments, Debug, Display, Error, Formatter, Result, Write}; diff --git a/rust/kernel/fs/file.rs b/rust/kernel/fs/file.rs index 72d84fb0e266..35fd5db35c46 100644 --- a/rust/kernel/fs/file.rs +++ b/rust/kernel/fs/file.rs @@ -366,7 +366,7 @@ impl core::ops::Deref for File { // // By the type invariants, there are no `fdget_pos` calls that did not take the // `f_pos_lock` mutex. - unsafe { LocalFile::from_raw_file(self as *const File as *const bindings::file) } + unsafe { LocalFile::from_raw_file(core::ptr::from_ref(self).cast()) } } } diff --git a/rust/kernel/generated_arch_reachable_asm.rs.S b/rust/kernel/generated_arch_reachable_asm.rs.S new file mode 100644 index 000000000000..3886a9ad3a99 --- /dev/null +++ b/rust/kernel/generated_arch_reachable_asm.rs.S @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/bug.h> + +// Cut here. + +::kernel::concat_literals!(ARCH_WARN_REACHABLE) diff --git a/rust/kernel/generated_arch_warn_asm.rs.S b/rust/kernel/generated_arch_warn_asm.rs.S new file mode 100644 index 000000000000..409eb4c2d3a1 --- /dev/null +++ b/rust/kernel/generated_arch_warn_asm.rs.S @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/bug.h> + +// Cut here. + +::kernel::concat_literals!(ARCH_WARN_ASM("{file}", "{line}", "{flags}", "{size}")) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 8d228c237954..4949047af8d7 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -29,15 +29,15 @@ //! //! ## General Examples //! -//! ```rust,ignore -//! # #![allow(clippy::disallowed_names)] +//! ```rust +//! # #![expect(clippy::disallowed_names, clippy::undocumented_unsafe_blocks)] //! use kernel::types::Opaque; //! use pin_init::pin_init_from_closure; //! //! // assume we have some `raw_foo` type in C: //! #[repr(C)] //! struct RawFoo([u8; 16]); -//! extern { +//! extern "C" { //! fn init_foo(_: *mut RawFoo); //! } //! @@ -66,25 +66,17 @@ //! }); //! ``` //! -//! ```rust,ignore -//! # #![allow(unreachable_pub, clippy::disallowed_names)] +//! ```rust +//! # #![expect(unreachable_pub, clippy::disallowed_names)] //! use kernel::{prelude::*, types::Opaque}; //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; //! # mod bindings { -//! # #![allow(non_camel_case_types)] +//! # #![expect(non_camel_case_types, clippy::missing_safety_doc)] //! # pub struct foo; //! # pub unsafe fn init_foo(_ptr: *mut foo) {} //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} //! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } //! # } -//! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround. -//! # trait FromErrno { -//! # fn from_errno(errno: core::ffi::c_int) -> Error { -//! # // Dummy error that can be constructed outside the `kernel` crate. -//! # Error::from(core::fmt::Error) -//! # } -//! # } -//! # impl FromErrno for Error {} //! /// # Invariants //! /// //! /// `foo` is always initialized @@ -108,13 +100,13 @@ //! let foo = addr_of_mut!((*slot).foo); //! //! // Initialize the `foo` -//! bindings::init_foo(Opaque::raw_get(foo)); +//! bindings::init_foo(Opaque::cast_into(foo)); //! //! // Try to enable it. -//! let err = bindings::enable_foo(Opaque::raw_get(foo), flags); +//! let err = bindings::enable_foo(Opaque::cast_into(foo), flags); //! if err != 0 { //! // Enabling has failed, first clean up the foo and then return the error. -//! bindings::destroy_foo(Opaque::raw_get(foo)); +//! bindings::destroy_foo(Opaque::cast_into(foo)); //! return Err(Error::from_errno(err)); //! } //! @@ -206,7 +198,7 @@ pub trait InPlaceInit<T>: Sized { /// /// ```rust /// use kernel::error::Error; -/// use pin_init::zeroed; +/// use pin_init::init_zeroed; /// struct BigBuf { /// big: KBox<[u8; 1024 * 1024 * 1024]>, /// small: [u8; 1024 * 1024], @@ -215,7 +207,7 @@ pub trait InPlaceInit<T>: Sized { /// impl BigBuf { /// fn new() -> impl Init<Self, Error> { /// try_init!(Self { -/// big: KBox::init(zeroed(), GFP_KERNEL)?, +/// big: KBox::init(init_zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// }? Error) /// } @@ -231,14 +223,14 @@ macro_rules! try_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? { + ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),*>)? { $($fields)* }? $crate::error::Error) }; ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? { + ::pin_init::try_init!($(&$this in)? $t $(::<$($generics),*>)? { $($fields)* }? $err) }; @@ -264,7 +256,7 @@ macro_rules! try_init { /// ```rust /// # #![feature(new_uninit)] /// use kernel::error::Error; -/// use pin_init::zeroed; +/// use pin_init::init_zeroed; /// #[pin_data] /// struct BigBuf { /// big: KBox<[u8; 1024 * 1024 * 1024]>, @@ -275,7 +267,7 @@ macro_rules! try_init { /// impl BigBuf { /// fn new() -> impl PinInit<Self, Error> { /// try_pin_init!(Self { -/// big: KBox::init(zeroed(), GFP_KERNEL)?, +/// big: KBox::init(init_zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// ptr: core::ptr::null_mut(), /// }? Error) @@ -291,14 +283,14 @@ macro_rules! try_pin_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? { + ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? { $($fields)* }? $crate::error::Error) }; ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),* $(,)?>)? { + ::pin_init::try_pin_init!($(&$this in)? $t $(::<$($generics),*>)? { $($fields)* }? $err) }; diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 72d80a6f131e..03b467722b86 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -5,7 +5,12 @@ //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) use crate::error::{code::EINVAL, Result}; -use crate::{bindings, build_assert}; +use crate::{bindings, build_assert, ffi::c_void}; + +pub mod mem; +pub mod resource; + +pub use resource::Resource; /// Raw representation of an MMIO region. /// @@ -43,7 +48,7 @@ impl<const SIZE: usize> IoRaw<SIZE> { } } -/// IO-mapped memory, starting at the base address @addr and spanning @maxlen bytes. +/// IO-mapped memory region. /// /// The creator (usually a subsystem / bus such as PCI) is responsible for creating the /// mapping, performing an additional region request etc. @@ -56,7 +61,7 @@ impl<const SIZE: usize> IoRaw<SIZE> { /// # Examples /// /// ```no_run -/// # use kernel::{bindings, io::{Io, IoRaw}}; +/// # use kernel::{bindings, ffi::c_void, io::{Io, IoRaw}}; /// # use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. @@ -70,19 +75,19 @@ impl<const SIZE: usize> IoRaw<SIZE> { /// unsafe fn new(paddr: usize) -> Result<Self>{ /// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is /// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as _, SIZE as _) }; +/// let addr = unsafe { bindings::ioremap(paddr as bindings::phys_addr_t, SIZE) }; /// if addr.is_null() { /// return Err(ENOMEM); /// } /// -/// Ok(IoMem(IoRaw::new(addr as _, SIZE)?)) +/// Ok(IoMem(IoRaw::new(addr as usize, SIZE)?)) /// } /// } /// /// impl<const SIZE: usize> Drop for IoMem<SIZE> { /// fn drop(&mut self) { /// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. -/// unsafe { bindings::iounmap(self.0.addr() as _); }; +/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); }; /// } /// } /// @@ -119,7 +124,7 @@ macro_rules! define_read { let addr = self.io_addr_assert::<$type_name>(offset); // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - unsafe { bindings::$c_fn(addr as _) } + unsafe { bindings::$c_fn(addr as *const c_void) } } /// Read IO data from a given offset. @@ -131,7 +136,7 @@ macro_rules! define_read { let addr = self.io_addr::<$type_name>(offset)?; // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - Ok(unsafe { bindings::$c_fn(addr as _) }) + Ok(unsafe { bindings::$c_fn(addr as *const c_void) }) } }; } @@ -148,7 +153,7 @@ macro_rules! define_write { let addr = self.io_addr_assert::<$type_name>(offset); // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - unsafe { bindings::$c_fn(value, addr as _, ) } + unsafe { bindings::$c_fn(value, addr as *mut c_void) } } /// Write IO data from a given offset. @@ -160,7 +165,7 @@ macro_rules! define_write { let addr = self.io_addr::<$type_name>(offset)?; // SAFETY: By the type invariant `addr` is a valid address for MMIO operations. - unsafe { bindings::$c_fn(value, addr as _) } + unsafe { bindings::$c_fn(value, addr as *mut c_void) } Ok(()) } }; diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs new file mode 100644 index 000000000000..6f99510bfc3a --- /dev/null +++ b/rust/kernel/io/mem.rs @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic memory-mapped IO. + +use core::ops::Deref; + +use crate::c_str; +use crate::device::Bound; +use crate::device::Device; +use crate::devres::Devres; +use crate::io; +use crate::io::resource::Region; +use crate::io::resource::Resource; +use crate::io::Io; +use crate::io::IoRaw; +use crate::prelude::*; + +/// An IO request for a specific device and resource. +pub struct IoRequest<'a> { + device: &'a Device<Bound>, + resource: &'a Resource, +} + +impl<'a> IoRequest<'a> { + /// Creates a new [`IoRequest`] instance. + /// + /// # Safety + /// + /// Callers must ensure that `resource` is valid for `device` during the + /// lifetime `'a`. + pub(crate) unsafe fn new(device: &'a Device<Bound>, resource: &'a Resource) -> Self { + IoRequest { device, resource } + } + + /// Maps an [`IoRequest`] where the size is known at compile time. + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + /// + /// # Examples + /// + /// The following example uses a [`kernel::platform::Device`] for + /// illustration purposes. + /// + /// ```no_run + /// use kernel::{bindings, c_str, platform, of, device::Core}; + /// struct SampleDriver; + /// + /// impl platform::Driver for SampleDriver { + /// # type IdInfo = (); + /// + /// fn probe( + /// pdev: &platform::Device<Core>, + /// info: Option<&Self::IdInfo>, + /// ) -> Result<Pin<KBox<Self>>> { + /// let offset = 0; // Some offset. + /// + /// // If the size is known at compile time, use [`Self::iomap_sized`]. + /// // + /// // No runtime checks will apply when reading and writing. + /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; + /// let iomem = request.iomap_sized::<42>(); + /// let iomem = KBox::pin_init(iomem, GFP_KERNEL)?; + /// + /// let io = iomem.access(pdev.as_ref())?; + /// + /// // Read and write a 32-bit value at `offset`. + /// let data = io.read32_relaxed(offset); + /// + /// io.write32_relaxed(data, offset); + /// + /// # Ok(KBox::new(SampleDriver, GFP_KERNEL)?.into()) + /// } + /// } + /// ``` + pub fn iomap_sized<const SIZE: usize>(self) -> impl PinInit<Devres<IoMem<SIZE>>, Error> + 'a { + IoMem::new(self) + } + + /// Same as [`Self::iomap_sized`] but with exclusive access to the + /// underlying region. + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + pub fn iomap_exclusive_sized<const SIZE: usize>( + self, + ) -> impl PinInit<Devres<ExclusiveIoMem<SIZE>>, Error> + 'a { + ExclusiveIoMem::new(self) + } + + /// Maps an [`IoRequest`] where the size is not known at compile time, + /// + /// This uses the [`ioremap()`] C API. + /// + /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device + /// + /// # Examples + /// + /// The following example uses a [`kernel::platform::Device`] for + /// illustration purposes. + /// + /// ```no_run + /// use kernel::{bindings, c_str, platform, of, device::Core}; + /// struct SampleDriver; + /// + /// impl platform::Driver for SampleDriver { + /// # type IdInfo = (); + /// + /// fn probe( + /// pdev: &platform::Device<Core>, + /// info: Option<&Self::IdInfo>, + /// ) -> Result<Pin<KBox<Self>>> { + /// let offset = 0; // Some offset. + /// + /// // Unlike [`Self::iomap_sized`], here the size of the memory region + /// // is not known at compile time, so only the `try_read*` and `try_write*` + /// // family of functions should be used, leading to runtime checks on every + /// // access. + /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; + /// let iomem = request.iomap(); + /// let iomem = KBox::pin_init(iomem, GFP_KERNEL)?; + /// + /// let io = iomem.access(pdev.as_ref())?; + /// + /// let data = io.try_read32_relaxed(offset)?; + /// + /// io.try_write32_relaxed(data, offset)?; + /// + /// # Ok(KBox::new(SampleDriver, GFP_KERNEL)?.into()) + /// } + /// } + /// ``` + pub fn iomap(self) -> impl PinInit<Devres<IoMem<0>>, Error> + 'a { + Self::iomap_sized::<0>(self) + } + + /// Same as [`Self::iomap`] but with exclusive access to the underlying + /// region. + pub fn iomap_exclusive(self) -> impl PinInit<Devres<ExclusiveIoMem<0>>, Error> + 'a { + Self::iomap_exclusive_sized::<0>(self) + } +} + +/// An exclusive memory-mapped IO region. +/// +/// # Invariants +/// +/// - [`ExclusiveIoMem`] has exclusive access to the underlying [`IoMem`]. +pub struct ExclusiveIoMem<const SIZE: usize> { + /// The underlying `IoMem` instance. + iomem: IoMem<SIZE>, + + /// The region abstraction. This represents exclusive access to the + /// range represented by the underlying `iomem`. + /// + /// This field is needed for ownership of the region. + _region: Region, +} + +impl<const SIZE: usize> ExclusiveIoMem<SIZE> { + /// Creates a new `ExclusiveIoMem` instance. + fn ioremap(resource: &Resource) -> Result<Self> { + let start = resource.start(); + let size = resource.size(); + let name = resource.name().unwrap_or(c_str!("")); + + let region = resource + .request_region( + start, + size, + name.to_cstring()?, + io::resource::Flags::IORESOURCE_MEM, + ) + .ok_or(EBUSY)?; + + let iomem = IoMem::ioremap(resource)?; + + let iomem = ExclusiveIoMem { + iomem, + _region: region, + }; + + Ok(iomem) + } + + /// Creates a new `ExclusiveIoMem` instance from a previously acquired [`IoRequest`]. + pub fn new<'a>(io_request: IoRequest<'a>) -> impl PinInit<Devres<Self>, Error> + 'a { + let dev = io_request.device; + let res = io_request.resource; + + Devres::new(dev, Self::ioremap(res)) + } +} + +impl<const SIZE: usize> Deref for ExclusiveIoMem<SIZE> { + type Target = Io<SIZE>; + + fn deref(&self) -> &Self::Target { + &self.iomem + } +} + +/// A generic memory-mapped IO region. +/// +/// Accesses to the underlying region is checked either at compile time, if the +/// region's size is known at that point, or at runtime otherwise. +/// +/// # Invariants +/// +/// [`IoMem`] always holds an [`IoRaw`] instance that holds a valid pointer to the +/// start of the I/O memory mapped region. +pub struct IoMem<const SIZE: usize = 0> { + io: IoRaw<SIZE>, +} + +impl<const SIZE: usize> IoMem<SIZE> { + fn ioremap(resource: &Resource) -> Result<Self> { + // Note: Some ioremap() implementations use types that depend on the CPU + // word width rather than the bus address width. + // + // TODO: Properly address this in the C code to avoid this `try_into`. + let size = resource.size().try_into()?; + if size == 0 { + return Err(EINVAL); + } + + let res_start = resource.start(); + + let addr = if resource + .flags() + .contains(io::resource::Flags::IORESOURCE_MEM_NONPOSTED) + { + // SAFETY: + // - `res_start` and `size` are read from a presumably valid `struct resource`. + // - `size` is known not to be zero at this point. + unsafe { bindings::ioremap_np(res_start, size) } + } else { + // SAFETY: + // - `res_start` and `size` are read from a presumably valid `struct resource`. + // - `size` is known not to be zero at this point. + unsafe { bindings::ioremap(res_start, size) } + }; + + if addr.is_null() { + return Err(ENOMEM); + } + + let io = IoRaw::new(addr as usize, size)?; + let io = IoMem { io }; + + Ok(io) + } + + /// Creates a new `IoMem` instance from a previously acquired [`IoRequest`]. + pub fn new<'a>(io_request: IoRequest<'a>) -> impl PinInit<Devres<Self>, Error> + 'a { + let dev = io_request.device; + let res = io_request.resource; + + Devres::new(dev, Self::ioremap(res)) + } +} + +impl<const SIZE: usize> Drop for IoMem<SIZE> { + fn drop(&mut self) { + // SAFETY: Safe as by the invariant of `Io`. + unsafe { bindings::iounmap(self.io.addr() as *mut c_void) } + } +} + +impl<const SIZE: usize> Deref for IoMem<SIZE> { + type Target = Io<SIZE>; + + fn deref(&self) -> &Self::Target { + // SAFETY: Safe as by the invariant of `IoMem`. + unsafe { Io::from_raw(&self.io) } + } +} diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs new file mode 100644 index 000000000000..bea3ee0ed87b --- /dev/null +++ b/rust/kernel/io/resource.rs @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for [system +//! resources](https://docs.kernel.org/core-api/kernel-api.html#resources-management). +//! +//! C header: [`include/linux/ioport.h`](srctree/include/linux/ioport.h) + +use core::ops::Deref; +use core::ptr::NonNull; + +use crate::prelude::*; +use crate::str::{CStr, CString}; +use crate::types::Opaque; + +/// Resource Size type. +/// +/// This is a type alias to either `u32` or `u64` depending on the config option +/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. +pub type ResourceSize = bindings::phys_addr_t; + +/// A region allocated from a parent [`Resource`]. +/// +/// # Invariants +/// +/// - `self.0` points to a valid `bindings::resource` that was obtained through +/// `bindings::__request_region`. +pub struct Region { + /// The resource returned when the region was requested. + resource: NonNull<bindings::resource>, + /// The name that was passed in when the region was requested. We need to + /// store it for ownership reasons. + _name: CString, +} + +impl Deref for Region { + type Target = Resource; + + fn deref(&self) -> &Self::Target { + // SAFETY: Safe as per the invariant of `Region`. + unsafe { Resource::from_raw(self.resource.as_ptr()) } + } +} + +impl Drop for Region { + fn drop(&mut self) { + let (flags, start, size) = { + let res = &**self; + (res.flags(), res.start(), res.size()) + }; + + let release_fn = if flags.contains(Flags::IORESOURCE_MEM) { + bindings::release_mem_region + } else { + bindings::release_region + }; + + // SAFETY: Safe as per the invariant of `Region`. + unsafe { release_fn(start, size) }; + } +} + +// SAFETY: `Region` only holds a pointer to a C `struct resource`, which is safe to be used from +// any thread. +unsafe impl Send for Region {} + +// SAFETY: `Region` only holds a pointer to a C `struct resource`, references to which are +// safe to be used from any thread. +unsafe impl Sync for Region {} + +/// A resource abstraction. +/// +/// # Invariants +/// +/// [`Resource`] is a transparent wrapper around a valid `bindings::resource`. +#[repr(transparent)] +pub struct Resource(Opaque<bindings::resource>); + +impl Resource { + /// Creates a reference to a [`Resource`] from a valid pointer. + /// + /// # Safety + /// + /// The caller must ensure that for the duration of 'a, the pointer will + /// point at a valid `bindings::resource`. + /// + /// The caller must also ensure that the [`Resource`] is only accessed via the + /// returned reference for the duration of 'a. + pub(crate) const unsafe fn from_raw<'a>(ptr: *mut bindings::resource) -> &'a Self { + // SAFETY: Self is a transparent wrapper around `Opaque<bindings::resource>`. + unsafe { &*ptr.cast() } + } + + /// Requests a resource region. + /// + /// Exclusive access will be given and the region will be marked as busy. + /// Further calls to [`Self::request_region`] will return [`None`] if + /// the region, or a part of it, is already in use. + pub fn request_region( + &self, + start: ResourceSize, + size: ResourceSize, + name: CString, + flags: Flags, + ) -> Option<Region> { + // SAFETY: + // - Safe as per the invariant of `Resource`. + // - `__request_region` will store a reference to the name, but that is + // safe as we own it and it will not be dropped until the `Region` is + // dropped. + let region = unsafe { + bindings::__request_region( + self.0.get(), + start, + size, + name.as_char_ptr(), + flags.0 as c_int, + ) + }; + + Some(Region { + resource: NonNull::new(region)?, + _name: name, + }) + } + + /// Returns the size of the resource. + pub fn size(&self) -> ResourceSize { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + unsafe { bindings::resource_size(inner) } + } + + /// Returns the start address of the resource. + pub fn start(&self) -> ResourceSize { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + unsafe { (*inner).start } + } + + /// Returns the name of the resource. + pub fn name(&self) -> Option<&CStr> { + let inner = self.0.get(); + + // SAFETY: Safe as per the invariants of `Resource`. + let name = unsafe { (*inner).name }; + + if name.is_null() { + return None; + } + + // SAFETY: In the C code, `resource::name` either contains a null + // pointer or points to a valid NUL-terminated C string, and at this + // point we know it is not null, so we can safely convert it to a + // `CStr`. + Some(unsafe { CStr::from_char_ptr(name) }) + } + + /// Returns the flags associated with the resource. + pub fn flags(&self) -> Flags { + let inner = self.0.get(); + // SAFETY: Safe as per the invariants of `Resource`. + let flags = unsafe { (*inner).flags }; + + Flags(flags) + } +} + +// SAFETY: `Resource` only holds a pointer to a C `struct resource`, which is +// safe to be used from any thread. +unsafe impl Send for Resource {} + +// SAFETY: `Resource` only holds a pointer to a C `struct resource`, references +// to which are safe to be used from any thread. +unsafe impl Sync for Resource {} + +/// Resource flags as stored in the C `struct resource::flags` field. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the associated constants such as +/// [`Flags::IORESOURCE_IO`]. +#[derive(Clone, Copy, PartialEq)] +pub struct Flags(c_ulong); + +impl Flags { + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Flags) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for Flags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Flags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Flags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +impl Flags { + /// PCI/ISA I/O ports. + pub const IORESOURCE_IO: Flags = Flags::new(bindings::IORESOURCE_IO); + + /// Resource is software muxed. + pub const IORESOURCE_MUXED: Flags = Flags::new(bindings::IORESOURCE_MUXED); + + /// Resource represents a memory region. + pub const IORESOURCE_MEM: Flags = Flags::new(bindings::IORESOURCE_MEM); + + /// Resource represents a memory region that must be ioremaped using `ioremap_np`. + pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED); + + const fn new(value: u32) -> Self { + crate::build_assert!(value as u64 <= c_ulong::MAX as u64); + Flags(value as c_ulong) + } +} diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 4b8cdcb21e77..41efd87595d6 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -7,7 +7,10 @@ //! Reference: <https://docs.kernel.org/dev-tools/kunit/index.html> use crate::prelude::*; -use core::{ffi::c_void, fmt}; +use core::fmt; + +#[cfg(CONFIG_PRINTK)] +use crate::c_str; /// Prints a KUnit error-level message. /// @@ -19,8 +22,8 @@ pub fn err(args: fmt::Arguments<'_>) { #[cfg(CONFIG_PRINTK)] unsafe { bindings::_printk( - c"\x013%pA".as_ptr() as _, - &args as *const _ as *const c_void, + c_str!("\x013%pA").as_char_ptr(), + core::ptr::from_ref(&args).cast::<c_void>(), ); } } @@ -35,8 +38,8 @@ pub fn info(args: fmt::Arguments<'_>) { #[cfg(CONFIG_PRINTK)] unsafe { bindings::_printk( - c"\x016%pA".as_ptr() as _, - &args as *const _ as *const c_void, + c_str!("\x016%pA").as_char_ptr(), + core::ptr::from_ref(&args).cast::<c_void>(), ); } } @@ -302,7 +305,7 @@ macro_rules! kunit_unsafe_test_suite { is_init: false, }; - #[used] + #[used(compiler)] #[allow(unused_unsafe)] #[cfg_attr(not(target_os = "macos"), link_section = ".kunit_test_suites")] static mut KUNIT_TEST_SUITE_ENTRY: *const ::kernel::bindings::kunit_suite = diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 6b4774b2b1c3..ed53169e795c 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -34,12 +34,19 @@ // Expected to become stable. #![feature(arbitrary_self_types)] // +// To be determined. +#![feature(used_with_arg)] +// // `feature(derive_coerce_pointee)` is expected to become stable. Before Rust // 1.84.0, it did not exist, so enable the predecessor features. #![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(derive_coerce_pointee))] #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(coerce_unsized))] #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(dispatch_from_dyn))] #![cfg_attr(not(CONFIG_RUSTC_HAS_COERCE_POINTEE), feature(unsize))] +// +// `feature(file_with_nul)` is expected to become stable. Before Rust 1.89.0, it did not exist, so +// enable it conditionally. +#![cfg_attr(CONFIG_RUSTC_HAS_FILE_WITH_NUL, feature(file_with_nul))] // Ensure conditional compilation based on the kernel configuration works; // otherwise we may silently break things like initcall handling. @@ -51,11 +58,14 @@ extern crate self as kernel; pub use ffi; +pub mod acpi; pub mod alloc; #[cfg(CONFIG_AUXILIARY_BUS)] pub mod auxiliary; +pub mod bits; #[cfg(CONFIG_BLOCK)] pub mod block; +pub mod bug; #[doc(hidden)] pub mod build_assert; pub mod clk; @@ -77,6 +87,7 @@ pub mod error; pub mod faux; #[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)] pub mod firmware; +pub mod fmt; pub mod fs; pub mod init; pub mod io; @@ -100,6 +111,7 @@ pub mod platform; pub mod prelude; pub mod print; pub mod rbtree; +pub mod regulator; pub mod revocable; pub mod security; pub mod seq_file; @@ -204,6 +216,13 @@ fn panic(info: &core::panic::PanicInfo<'_>) -> ! { /// Produces a pointer to an object from a pointer to one of its fields. /// +/// If you encounter a type mismatch due to the [`Opaque`] type, then use [`Opaque::cast_into`] or +/// [`Opaque::cast_from`] to resolve the mismatch. +/// +/// [`Opaque`]: crate::types::Opaque +/// [`Opaque::cast_into`]: crate::types::Opaque::cast_into +/// [`Opaque::cast_from`]: crate::types::Opaque::cast_from +/// /// # Safety /// /// The pointer passed to this macro, and the pointer returned by this macro, must both be in @@ -274,3 +293,47 @@ macro_rules! asm { ::core::arch::asm!( $($asm)*, $($rest)* ) }; } + +/// Gets the C string file name of a [`Location`]. +/// +/// If `file_with_nul()` is not available, returns a string that warns about it. +/// +/// [`Location`]: core::panic::Location +/// +/// # Examples +/// +/// ``` +/// # use kernel::file_from_location; +/// +/// #[track_caller] +/// fn foo() { +/// let caller = core::panic::Location::caller(); +/// +/// // Output: +/// // - A path like "rust/kernel/example.rs" if file_with_nul() is available. +/// // - "<Location::file_with_nul() not supported>" otherwise. +/// let caller_file = file_from_location(caller); +/// +/// // Prints out the message with caller's file name. +/// pr_info!("foo() called in file {caller_file:?}\n"); +/// +/// # if cfg!(CONFIG_RUSTC_HAS_FILE_WITH_NUL) { +/// # assert_eq!(Ok(caller.file()), caller_file.to_str()); +/// # } +/// } +/// +/// # foo(); +/// ``` +#[inline] +pub fn file_from_location<'a>(loc: &'a core::panic::Location<'a>) -> &'a core::ffi::CStr { + #[cfg(CONFIG_RUSTC_HAS_FILE_WITH_NUL)] + { + loc.file_with_nul() + } + + #[cfg(not(CONFIG_RUSTC_HAS_FILE_WITH_NUL))] + { + let _ = loc; + c"<Location::file_with_nul() not supported>" + } +} diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index c391c30b80f8..44e5219cfcbc 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -57,14 +57,11 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; /// } /// } /// -/// impl_has_list_links! { -/// impl HasListLinks<0> for BasicItem { self.links } -/// } /// impl_list_arc_safe! { /// impl ListArcSafe<0> for BasicItem { untracked; } /// } /// impl_list_item! { -/// impl ListItem<0> for BasicItem { using ListLinks; } +/// impl ListItem<0> for BasicItem { using ListLinks { self.links }; } /// } /// /// // Create a new empty list. @@ -82,9 +79,9 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; /// // [15, 10, 30] /// { /// let mut iter = list.iter(); -/// assert_eq!(iter.next().unwrap().value, 15); -/// assert_eq!(iter.next().unwrap().value, 10); -/// assert_eq!(iter.next().unwrap().value, 30); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 15); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 10); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 30); /// assert!(iter.next().is_none()); /// /// // Verify the length of the list. @@ -93,9 +90,9 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; /// /// // Pop the items from the list using `pop_back()` and verify the content. /// { -/// assert_eq!(list.pop_back().unwrap().value, 30); -/// assert_eq!(list.pop_back().unwrap().value, 10); -/// assert_eq!(list.pop_back().unwrap().value, 15); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value, 30); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value, 10); +/// assert_eq!(list.pop_back().ok_or(EINVAL)?.value, 15); /// } /// /// // Insert 3 elements using `push_front()`. @@ -107,9 +104,9 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; /// // [30, 10, 15] /// { /// let mut iter = list.iter(); -/// assert_eq!(iter.next().unwrap().value, 30); -/// assert_eq!(iter.next().unwrap().value, 10); -/// assert_eq!(iter.next().unwrap().value, 15); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 30); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 10); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 15); /// assert!(iter.next().is_none()); /// /// // Verify the length of the list. @@ -118,8 +115,8 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; /// /// // Pop the items from the list using `pop_front()` and verify the content. /// { -/// assert_eq!(list.pop_front().unwrap().value, 30); -/// assert_eq!(list.pop_front().unwrap().value, 10); +/// assert_eq!(list.pop_front().ok_or(EINVAL)?.value, 30); +/// assert_eq!(list.pop_front().ok_or(EINVAL)?.value, 10); /// } /// /// // Push `list2` to `list` through `push_all_back()`. @@ -135,9 +132,9 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; /// // list: [15, 25, 35] /// // list2: [] /// let mut iter = list.iter(); -/// assert_eq!(iter.next().unwrap().value, 15); -/// assert_eq!(iter.next().unwrap().value, 25); -/// assert_eq!(iter.next().unwrap().value, 35); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 15); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 25); +/// assert_eq!(iter.next().ok_or(EINVAL)?.value, 35); /// assert!(iter.next().is_none()); /// assert!(list2.is_empty()); /// } @@ -284,7 +281,7 @@ impl<const ID: u64> ListLinks<ID> { #[inline] unsafe fn fields(me: *mut Self) -> *mut ListLinksFields { // SAFETY: The caller promises that the pointer is valid. - unsafe { Opaque::raw_get(ptr::addr_of!((*me).inner)) } + unsafe { Opaque::cast_into(ptr::addr_of!((*me).inner)) } } /// # Safety @@ -320,9 +317,6 @@ unsafe impl<T: ?Sized + Send, const ID: u64> Send for ListLinksSelfPtr<T, ID> {} unsafe impl<T: ?Sized + Sync, const ID: u64> Sync for ListLinksSelfPtr<T, ID> {} impl<T: ?Sized, const ID: u64> ListLinksSelfPtr<T, ID> { - /// The offset from the [`ListLinks`] to the self pointer field. - pub const LIST_LINKS_SELF_PTR_OFFSET: usize = core::mem::offset_of!(Self, self_ptr); - /// Creates a new initializer for this type. pub fn new() -> impl PinInit<Self> { // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will @@ -337,6 +331,16 @@ impl<T: ?Sized, const ID: u64> ListLinksSelfPtr<T, ID> { self_ptr: Opaque::uninit(), } } + + /// Returns a pointer to the self pointer. + /// + /// # Safety + /// + /// The provided pointer must point at a valid struct of type `Self`. + pub unsafe fn raw_get_self_ptr(me: *const Self) -> *const Opaque<*const T> { + // SAFETY: The caller promises that the pointer is valid. + unsafe { ptr::addr_of!((*me).self_ptr) } + } } impl<T: ?Sized + ListItem<ID>, const ID: u64> List<T, ID> { @@ -711,14 +715,11 @@ impl<'a, T: ?Sized + ListItem<ID>, const ID: u64> Iterator for Iter<'a, T, ID> { /// } /// } /// -/// kernel::list::impl_has_list_links! { -/// impl HasListLinks<0> for ListItem { self.links } -/// } /// kernel::list::impl_list_arc_safe! { /// impl ListArcSafe<0> for ListItem { untracked; } /// } /// kernel::list::impl_list_item! { -/// impl ListItem<0> for ListItem { using ListLinks; } +/// impl ListItem<0> for ListItem { using ListLinks { self.links }; } /// } /// /// // Use a cursor to remove the first element with the given value. @@ -809,11 +810,11 @@ impl<'a, T: ?Sized + ListItem<ID>, const ID: u64> Iterator for Iter<'a, T, ID> { /// merge_sorted(&mut list, list2); /// /// let mut items = list.into_iter(); -/// assert_eq!(items.next().unwrap().value, 10); -/// assert_eq!(items.next().unwrap().value, 11); -/// assert_eq!(items.next().unwrap().value, 12); -/// assert_eq!(items.next().unwrap().value, 13); -/// assert_eq!(items.next().unwrap().value, 14); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 10); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 11); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 12); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 13); +/// assert_eq!(items.next().ok_or(EINVAL)?.value, 14); /// assert!(items.next().is_none()); /// # Result::<(), Error>::Ok(()) /// ``` diff --git a/rust/kernel/list/impl_list_item_mod.rs b/rust/kernel/list/impl_list_item_mod.rs index a0438537cee1..202bc6f97c13 100644 --- a/rust/kernel/list/impl_list_item_mod.rs +++ b/rust/kernel/list/impl_list_item_mod.rs @@ -4,60 +4,48 @@ //! Helpers for implementing list traits safely. -use crate::list::ListLinks; - -/// Declares that this type has a `ListLinks<ID>` field at a fixed offset. +/// Declares that this type has a [`ListLinks<ID>`] field. /// -/// This trait is only used to help implement `ListItem` safely. If `ListItem` is implemented +/// This trait is only used to help implement [`ListItem`] safely. If [`ListItem`] is implemented /// manually, then this trait is not needed. Use the [`impl_has_list_links!`] macro to implement /// this trait. /// /// # Safety /// -/// All values of this type must have a `ListLinks<ID>` field at the given offset. +/// The methods on this trait must have exactly the behavior that the definitions given below have. /// -/// The behavior of `raw_get_list_links` must not be changed. +/// [`ListLinks<ID>`]: crate::list::ListLinks +/// [`ListItem`]: crate::list::ListItem pub unsafe trait HasListLinks<const ID: u64 = 0> { - /// The offset of the `ListLinks` field. - const OFFSET: usize; - - /// Returns a pointer to the [`ListLinks<T, ID>`] field. + /// Returns a pointer to the [`ListLinks<ID>`] field. /// /// # Safety /// /// The provided pointer must point at a valid struct of type `Self`. /// - /// [`ListLinks<T, ID>`]: ListLinks - // We don't really need this method, but it's necessary for the implementation of - // `impl_has_list_links!` to be correct. - #[inline] - unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut ListLinks<ID> { - // SAFETY: The caller promises that the pointer is valid. The implementer promises that the - // `OFFSET` constant is correct. - unsafe { (ptr as *mut u8).add(Self::OFFSET) as *mut ListLinks<ID> } - } + /// [`ListLinks<ID>`]: crate::list::ListLinks + unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut crate::list::ListLinks<ID>; } /// Implements the [`HasListLinks`] trait for the given type. #[macro_export] macro_rules! impl_has_list_links { - ($(impl$(<$($implarg:ident),*>)? + ($(impl$({$($generics:tt)*})? HasListLinks$(<$id:tt>)? - for $self:ident $(<$($selfarg:ty),*>)? + for $self:ty { self$(.$field:ident)* } )*) => {$( // SAFETY: The implementation of `raw_get_list_links` only compiles if the field has the // right type. - // - // The behavior of `raw_get_list_links` is not changed since the `addr_of_mut!` macro is - // equivalent to the pointer offset operation in the trait definition. - unsafe impl$(<$($implarg),*>)? $crate::list::HasListLinks$(<$id>)? for - $self $(<$($selfarg),*>)? - { - const OFFSET: usize = ::core::mem::offset_of!(Self, $($field).*) as usize; - + unsafe impl$(<$($generics)*>)? $crate::list::HasListLinks$(<$id>)? for $self { #[inline] unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { + // Statically ensure that `$(.field)*` doesn't follow any pointers. + // + // Cannot be `const` because `$self` may contain generics and E0401 says constants + // "can't use {`Self`,generic parameters} from outer item". + if false { let _: usize = ::core::mem::offset_of!(Self, $($field).*); } + // SAFETY: The caller promises that the pointer is not dangling. We know that this // expression doesn't follow any pointers, as the `offset_of!` invocation above // would otherwise not compile. @@ -68,12 +56,16 @@ macro_rules! impl_has_list_links { } pub use impl_has_list_links; -/// Declares that the `ListLinks<ID>` field in this struct is inside a `ListLinksSelfPtr<T, ID>`. +/// Declares that the [`ListLinks<ID>`] field in this struct is inside a +/// [`ListLinksSelfPtr<T, ID>`]. /// /// # Safety /// -/// The `ListLinks<ID>` field of this struct at the offset `HasListLinks<ID>::OFFSET` must be -/// inside a `ListLinksSelfPtr<T, ID>`. +/// The [`ListLinks<ID>`] field of this struct at [`HasListLinks<ID>::raw_get_list_links`] must be +/// inside a [`ListLinksSelfPtr<T, ID>`]. +/// +/// [`ListLinks<ID>`]: crate::list::ListLinks +/// [`ListLinksSelfPtr<T, ID>`]: crate::list::ListLinksSelfPtr pub unsafe trait HasSelfPtr<T: ?Sized, const ID: u64 = 0> where Self: HasListLinks<ID>, @@ -83,27 +75,21 @@ where /// Implements the [`HasListLinks`] and [`HasSelfPtr`] traits for the given type. #[macro_export] macro_rules! impl_has_list_links_self_ptr { - ($(impl$({$($implarg:tt)*})? + ($(impl$({$($generics:tt)*})? HasSelfPtr<$item_type:ty $(, $id:tt)?> - for $self:ident $(<$($selfarg:ty),*>)? - { self.$field:ident } + for $self:ty + { self$(.$field:ident)* } )*) => {$( // SAFETY: The implementation of `raw_get_list_links` only compiles if the field has the // right type. - unsafe impl$(<$($implarg)*>)? $crate::list::HasSelfPtr<$item_type $(, $id)?> for - $self $(<$($selfarg),*>)? - {} - - unsafe impl$(<$($implarg)*>)? $crate::list::HasListLinks$(<$id>)? for - $self $(<$($selfarg),*>)? - { - const OFFSET: usize = ::core::mem::offset_of!(Self, $field) as usize; + unsafe impl$(<$($generics)*>)? $crate::list::HasSelfPtr<$item_type $(, $id)?> for $self {} + unsafe impl$(<$($generics)*>)? $crate::list::HasListLinks$(<$id>)? for $self { #[inline] unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { // SAFETY: The caller promises that the pointer is not dangling. let ptr: *mut $crate::list::ListLinksSelfPtr<$item_type $(, $id)?> = - unsafe { ::core::ptr::addr_of_mut!((*ptr).$field) }; + unsafe { ::core::ptr::addr_of_mut!((*ptr)$(.$field)*) }; ptr.cast() } } @@ -117,15 +103,95 @@ pub use impl_has_list_links_self_ptr; /// implement that trait. /// /// [`ListItem`]: crate::list::ListItem +/// +/// # Examples +/// +/// ``` +/// #[pin_data] +/// struct SimpleListItem { +/// value: u32, +/// #[pin] +/// links: kernel::list::ListLinks, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl ListArcSafe<0> for SimpleListItem { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl ListItem<0> for SimpleListItem { using ListLinks { self.links }; } +/// } +/// +/// struct ListLinksHolder { +/// inner: kernel::list::ListLinks, +/// } +/// +/// #[pin_data] +/// struct ComplexListItem<T, U> { +/// value: Result<T, U>, +/// #[pin] +/// links: ListLinksHolder, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl{T, U} ListArcSafe<0> for ComplexListItem<T, U> { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl{T, U} ListItem<0> for ComplexListItem<T, U> { using ListLinks { self.links.inner }; } +/// } +/// ``` +/// +/// ``` +/// #[pin_data] +/// struct SimpleListItem { +/// value: u32, +/// #[pin] +/// links: kernel::list::ListLinksSelfPtr<SimpleListItem>, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl ListArcSafe<0> for SimpleListItem { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl ListItem<0> for SimpleListItem { using ListLinksSelfPtr { self.links }; } +/// } +/// +/// struct ListLinksSelfPtrHolder<T, U> { +/// inner: kernel::list::ListLinksSelfPtr<ComplexListItem<T, U>>, +/// } +/// +/// #[pin_data] +/// struct ComplexListItem<T, U> { +/// value: Result<T, U>, +/// #[pin] +/// links: ListLinksSelfPtrHolder<T, U>, +/// } +/// +/// kernel::list::impl_list_arc_safe! { +/// impl{T, U} ListArcSafe<0> for ComplexListItem<T, U> { untracked; } +/// } +/// +/// kernel::list::impl_list_item! { +/// impl{T, U} ListItem<0> for ComplexListItem<T, U> { +/// using ListLinksSelfPtr { self.links.inner }; +/// } +/// } +/// ``` #[macro_export] macro_rules! impl_list_item { ( - $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $t:ty { - using ListLinks; + $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $self:ty { + using ListLinks { self$(.$field:ident)* }; })* ) => {$( + $crate::list::impl_has_list_links! { + impl$({$($generics)*})? HasListLinks<$num> for $self { self$(.$field)* } + } + // SAFETY: See GUARANTEES comment on each method. - unsafe impl$(<$($generics)*>)? $crate::list::ListItem<$num> for $t { + unsafe impl$(<$($generics)*>)? $crate::list::ListItem<$num> for $self { // GUARANTEES: // * This returns the same pointer as `prepare_to_insert` because `prepare_to_insert` // is implemented in terms of `view_links`. @@ -139,20 +205,19 @@ macro_rules! impl_list_item { } // GUARANTEES: - // * `me` originates from the most recent call to `prepare_to_insert`, which just added - // `offset` to the pointer passed to `prepare_to_insert`. This method subtracts - // `offset` from `me` so it returns the pointer originally passed to - // `prepare_to_insert`. + // * `me` originates from the most recent call to `prepare_to_insert`, which calls + // `raw_get_list_link`, which is implemented using `addr_of_mut!((*self)$(.$field)*)`. + // This method uses `container_of` to perform the inverse operation, so it returns the + // pointer originally passed to `prepare_to_insert`. // * The pointer remains valid until the next call to `post_remove` because the caller // of the most recent call to `prepare_to_insert` promised to retain ownership of the // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot // be destroyed while a `ListArc` reference exists. unsafe fn view_value(me: *mut $crate::list::ListLinks<$num>) -> *const Self { - let offset = <Self as $crate::list::HasListLinks<$num>>::OFFSET; // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it - // points at the field at offset `offset` in a value of type `Self`. Thus, - // subtracting `offset` from `me` is still in-bounds of the allocation. - unsafe { (me as *const u8).sub(offset) as *const Self } + // points at the field `$field` in a value of type `Self`. Thus, reversing that + // operation is still in-bounds of the allocation. + $crate::container_of!(me, Self, $($field).*) } // GUARANTEES: @@ -169,27 +234,30 @@ macro_rules! impl_list_item { } // GUARANTEES: - // * `me` originates from the most recent call to `prepare_to_insert`, which just added - // `offset` to the pointer passed to `prepare_to_insert`. This method subtracts - // `offset` from `me` so it returns the pointer originally passed to - // `prepare_to_insert`. + // * `me` originates from the most recent call to `prepare_to_insert`, which calls + // `raw_get_list_link`, which is implemented using `addr_of_mut!((*self)$(.$field)*)`. + // This method uses `container_of` to perform the inverse operation, so it returns the + // pointer originally passed to `prepare_to_insert`. unsafe fn post_remove(me: *mut $crate::list::ListLinks<$num>) -> *const Self { - let offset = <Self as $crate::list::HasListLinks<$num>>::OFFSET; // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it - // points at the field at offset `offset` in a value of type `Self`. Thus, - // subtracting `offset` from `me` is still in-bounds of the allocation. - unsafe { (me as *const u8).sub(offset) as *const Self } + // points at the field `$field` in a value of type `Self`. Thus, reversing that + // operation is still in-bounds of the allocation. + $crate::container_of!(me, Self, $($field).*) } } )*}; ( - $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $t:ty { - using ListLinksSelfPtr; + $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $self:ty { + using ListLinksSelfPtr { self$(.$field:ident)* }; })* ) => {$( + $crate::list::impl_has_list_links_self_ptr! { + impl$({$($generics)*})? HasSelfPtr<$self> for $self { self$(.$field)* } + } + // SAFETY: See GUARANTEES comment on each method. - unsafe impl$(<$($generics)*>)? $crate::list::ListItem<$num> for $t { + unsafe impl$(<$($generics)*>)? $crate::list::ListItem<$num> for $self { // GUARANTEES: // This implementation of `ListItem` will not give out exclusive access to the same // `ListLinks` several times because calls to `prepare_to_insert` and `post_remove` @@ -202,14 +270,16 @@ macro_rules! impl_list_item { // SAFETY: The caller promises that `me` points at a valid value of type `Self`. let links_field = unsafe { <Self as $crate::list::ListItem<$num>>::view_links(me) }; - let spoff = $crate::list::ListLinksSelfPtr::<Self, $num>::LIST_LINKS_SELF_PTR_OFFSET; - // Goes via the offset as the field is private. - // - // SAFETY: The constant is equal to `offset_of!(ListLinksSelfPtr, self_ptr)`, so - // the pointer stays in bounds of the allocation. - let self_ptr = unsafe { (links_field as *const u8).add(spoff) } - as *const $crate::types::Opaque<*const Self>; - let cell_inner = $crate::types::Opaque::raw_get(self_ptr); + let container = $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr<Self, $num>, inner + ); + + // SAFETY: By the same reasoning above, `links_field` is a valid pointer. + let self_ptr = unsafe { + $crate::list::ListLinksSelfPtr::raw_get_self_ptr(container) + }; + + let cell_inner = $crate::types::Opaque::cast_into(self_ptr); // SAFETY: This value is not accessed in any other places than `prepare_to_insert`, // `post_remove`, or `view_value`. By the safety requirements of those methods, @@ -228,7 +298,9 @@ macro_rules! impl_list_item { // this value is not in a list. unsafe fn view_links(me: *const Self) -> *mut $crate::list::ListLinks<$num> { // SAFETY: The caller promises that `me` points at a valid value of type `Self`. - unsafe { <Self as HasListLinks<$num>>::raw_get_list_links(me.cast_mut()) } + unsafe { + <Self as $crate::list::HasListLinks<$num>>::raw_get_list_links(me.cast_mut()) + } } // This function is also used as the implementation of `post_remove`, so the caller @@ -247,12 +319,17 @@ macro_rules! impl_list_item { // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot // be destroyed while a `ListArc` reference exists. unsafe fn view_value(links_field: *mut $crate::list::ListLinks<$num>) -> *const Self { - let spoff = $crate::list::ListLinksSelfPtr::<Self, $num>::LIST_LINKS_SELF_PTR_OFFSET; - // SAFETY: The constant is equal to `offset_of!(ListLinksSelfPtr, self_ptr)`, so - // the pointer stays in bounds of the allocation. - let self_ptr = unsafe { (links_field as *const u8).add(spoff) } - as *const ::core::cell::UnsafeCell<*const Self>; - let cell_inner = ::core::cell::UnsafeCell::raw_get(self_ptr); + let container = $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr<Self, $num>, inner + ); + + // SAFETY: By the same reasoning above, `links_field` is a valid pointer. + let self_ptr = unsafe { + $crate::list::ListLinksSelfPtr::raw_get_self_ptr(container) + }; + + let cell_inner = $crate::types::Opaque::cast_into(self_ptr); + // SAFETY: This is not a data race, because the only function that writes to this // value is `prepare_to_insert`, but by the safety requirements the // `prepare_to_insert` method may not be called in parallel with `view_value` or diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs index 939278bc7b03..6373fe183b27 100644 --- a/rust/kernel/miscdevice.rs +++ b/rust/kernel/miscdevice.rs @@ -17,7 +17,6 @@ use crate::{ mm::virt::VmaNew, prelude::*, seq_file::SeqFile, - str::CStr, types::{ForeignOwnable, Opaque}, }; use core::{marker::PhantomData, mem::MaybeUninit, pin::Pin}; @@ -34,7 +33,7 @@ impl MiscDeviceOptions { pub const fn into_raw<T: MiscDevice>(self) -> bindings::miscdevice { // SAFETY: All zeros is valid for this C type. let mut result: bindings::miscdevice = unsafe { MaybeUninit::zeroed().assume_init() }; - result.minor = bindings::MISC_DYNAMIC_MINOR as _; + result.minor = bindings::MISC_DYNAMIC_MINOR as ffi::c_int; result.name = self.name.as_char_ptr(); result.fops = MiscdeviceVTable::<T>::build(); result @@ -45,7 +44,13 @@ impl MiscDeviceOptions { /// /// # Invariants /// -/// `inner` is a registered misc device. +/// - `inner` contains a `struct miscdevice` that is registered using +/// `misc_register()`. +/// - This registration remains valid for the entire lifetime of the +/// [`MiscDeviceRegistration`] instance. +/// - Deregistration occurs exactly once in [`Drop`] via `misc_deregister()`. +/// - `inner` wraps a valid, pinned `miscdevice` created using +/// [`MiscDeviceOptions::into_raw`]. #[repr(transparent)] #[pin_data(PinnedDrop)] pub struct MiscDeviceRegistration<T> { @@ -92,7 +97,7 @@ impl<T: MiscDevice> MiscDeviceRegistration<T> { // function tells the borrow-checker that the `&Device` reference must not outlive the // `&MiscDeviceRegistration<T>` used to obtain it, so the last use of the reference must be // before the underlying `struct miscdevice` is destroyed. - unsafe { Device::as_ref((*self.as_raw()).this_device) } + unsafe { Device::from_raw((*self.as_raw()).this_device) } } } @@ -217,7 +222,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { // type. // // SAFETY: The open call of a file can access the private data. - unsafe { (*raw_file).private_data = ptr.into_foreign().cast() }; + unsafe { (*raw_file).private_data = ptr.into_foreign() }; 0 } @@ -228,7 +233,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { /// must be associated with a `MiscDeviceRegistration<T>`. unsafe extern "C" fn release(_inode: *mut bindings::inode, file: *mut bindings::file) -> c_int { // SAFETY: The release call of a file owns the private data. - let private = unsafe { (*file).private_data }.cast(); + let private = unsafe { (*file).private_data }; // SAFETY: The release call of a file owns the private data. let ptr = unsafe { <T::Ptr as ForeignOwnable>::from_foreign(private) }; @@ -272,7 +277,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { /// `file` must be a valid file that is associated with a `MiscDeviceRegistration<T>`. unsafe extern "C" fn ioctl(file: *mut bindings::file, cmd: c_uint, arg: c_ulong) -> c_long { // SAFETY: The ioctl call of a file can access the private data. - let private = unsafe { (*file).private_data }.cast(); + let private = unsafe { (*file).private_data }; // SAFETY: Ioctl calls can borrow the private data of the file. let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; @@ -297,7 +302,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { arg: c_ulong, ) -> c_long { // SAFETY: The compat ioctl call of a file can access the private data. - let private = unsafe { (*file).private_data }.cast(); + let private = unsafe { (*file).private_data }; // SAFETY: Ioctl calls can borrow the private data of the file. let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; @@ -318,7 +323,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { /// - `seq_file` must be a valid `struct seq_file` that we can write to. unsafe extern "C" fn show_fdinfo(seq_file: *mut bindings::seq_file, file: *mut bindings::file) { // SAFETY: The release call of a file owns the private data. - let private = unsafe { (*file).private_data }.cast(); + let private = unsafe { (*file).private_data }; // SAFETY: Ioctl calls can borrow the private data of the file. let device = unsafe { <T::Ptr as ForeignOwnable>::borrow(private) }; // SAFETY: diff --git a/rust/kernel/mm/virt.rs b/rust/kernel/mm/virt.rs index 31803674aecc..6086ca981b06 100644 --- a/rust/kernel/mm/virt.rs +++ b/rust/kernel/mm/virt.rs @@ -392,80 +392,80 @@ pub mod flags { use crate::bindings; /// No flags are set. - pub const NONE: vm_flags_t = bindings::VM_NONE as _; + pub const NONE: vm_flags_t = bindings::VM_NONE as vm_flags_t; /// Mapping allows reads. - pub const READ: vm_flags_t = bindings::VM_READ as _; + pub const READ: vm_flags_t = bindings::VM_READ as vm_flags_t; /// Mapping allows writes. - pub const WRITE: vm_flags_t = bindings::VM_WRITE as _; + pub const WRITE: vm_flags_t = bindings::VM_WRITE as vm_flags_t; /// Mapping allows execution. - pub const EXEC: vm_flags_t = bindings::VM_EXEC as _; + pub const EXEC: vm_flags_t = bindings::VM_EXEC as vm_flags_t; /// Mapping is shared. - pub const SHARED: vm_flags_t = bindings::VM_SHARED as _; + pub const SHARED: vm_flags_t = bindings::VM_SHARED as vm_flags_t; /// Mapping may be updated to allow reads. - pub const MAYREAD: vm_flags_t = bindings::VM_MAYREAD as _; + pub const MAYREAD: vm_flags_t = bindings::VM_MAYREAD as vm_flags_t; /// Mapping may be updated to allow writes. - pub const MAYWRITE: vm_flags_t = bindings::VM_MAYWRITE as _; + pub const MAYWRITE: vm_flags_t = bindings::VM_MAYWRITE as vm_flags_t; /// Mapping may be updated to allow execution. - pub const MAYEXEC: vm_flags_t = bindings::VM_MAYEXEC as _; + pub const MAYEXEC: vm_flags_t = bindings::VM_MAYEXEC as vm_flags_t; /// Mapping may be updated to be shared. - pub const MAYSHARE: vm_flags_t = bindings::VM_MAYSHARE as _; + pub const MAYSHARE: vm_flags_t = bindings::VM_MAYSHARE as vm_flags_t; /// Page-ranges managed without `struct page`, just pure PFN. - pub const PFNMAP: vm_flags_t = bindings::VM_PFNMAP as _; + pub const PFNMAP: vm_flags_t = bindings::VM_PFNMAP as vm_flags_t; /// Memory mapped I/O or similar. - pub const IO: vm_flags_t = bindings::VM_IO as _; + pub const IO: vm_flags_t = bindings::VM_IO as vm_flags_t; /// Do not copy this vma on fork. - pub const DONTCOPY: vm_flags_t = bindings::VM_DONTCOPY as _; + pub const DONTCOPY: vm_flags_t = bindings::VM_DONTCOPY as vm_flags_t; /// Cannot expand with mremap(). - pub const DONTEXPAND: vm_flags_t = bindings::VM_DONTEXPAND as _; + pub const DONTEXPAND: vm_flags_t = bindings::VM_DONTEXPAND as vm_flags_t; /// Lock the pages covered when they are faulted in. - pub const LOCKONFAULT: vm_flags_t = bindings::VM_LOCKONFAULT as _; + pub const LOCKONFAULT: vm_flags_t = bindings::VM_LOCKONFAULT as vm_flags_t; /// Is a VM accounted object. - pub const ACCOUNT: vm_flags_t = bindings::VM_ACCOUNT as _; + pub const ACCOUNT: vm_flags_t = bindings::VM_ACCOUNT as vm_flags_t; /// Should the VM suppress accounting. - pub const NORESERVE: vm_flags_t = bindings::VM_NORESERVE as _; + pub const NORESERVE: vm_flags_t = bindings::VM_NORESERVE as vm_flags_t; /// Huge TLB Page VM. - pub const HUGETLB: vm_flags_t = bindings::VM_HUGETLB as _; + pub const HUGETLB: vm_flags_t = bindings::VM_HUGETLB as vm_flags_t; /// Synchronous page faults. (DAX-specific) - pub const SYNC: vm_flags_t = bindings::VM_SYNC as _; + pub const SYNC: vm_flags_t = bindings::VM_SYNC as vm_flags_t; /// Architecture-specific flag. - pub const ARCH_1: vm_flags_t = bindings::VM_ARCH_1 as _; + pub const ARCH_1: vm_flags_t = bindings::VM_ARCH_1 as vm_flags_t; /// Wipe VMA contents in child on fork. - pub const WIPEONFORK: vm_flags_t = bindings::VM_WIPEONFORK as _; + pub const WIPEONFORK: vm_flags_t = bindings::VM_WIPEONFORK as vm_flags_t; /// Do not include in the core dump. - pub const DONTDUMP: vm_flags_t = bindings::VM_DONTDUMP as _; + pub const DONTDUMP: vm_flags_t = bindings::VM_DONTDUMP as vm_flags_t; /// Not soft dirty clean area. - pub const SOFTDIRTY: vm_flags_t = bindings::VM_SOFTDIRTY as _; + pub const SOFTDIRTY: vm_flags_t = bindings::VM_SOFTDIRTY as vm_flags_t; /// Can contain `struct page` and pure PFN pages. - pub const MIXEDMAP: vm_flags_t = bindings::VM_MIXEDMAP as _; + pub const MIXEDMAP: vm_flags_t = bindings::VM_MIXEDMAP as vm_flags_t; /// MADV_HUGEPAGE marked this vma. - pub const HUGEPAGE: vm_flags_t = bindings::VM_HUGEPAGE as _; + pub const HUGEPAGE: vm_flags_t = bindings::VM_HUGEPAGE as vm_flags_t; /// MADV_NOHUGEPAGE marked this vma. - pub const NOHUGEPAGE: vm_flags_t = bindings::VM_NOHUGEPAGE as _; + pub const NOHUGEPAGE: vm_flags_t = bindings::VM_NOHUGEPAGE as vm_flags_t; /// KSM may merge identical pages. - pub const MERGEABLE: vm_flags_t = bindings::VM_MERGEABLE as _; + pub const MERGEABLE: vm_flags_t = bindings::VM_MERGEABLE as vm_flags_t; } diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 32ea43ece646..7de5cc7a0eee 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -6,7 +6,7 @@ //! //! C headers: [`include/linux/phy.h`](srctree/include/linux/phy.h). -use crate::{error::*, prelude::*, types::Opaque}; +use crate::{device_id::RawDeviceId, error::*, prelude::*, types::Opaque}; use core::{marker::PhantomData, ptr::addr_of_mut}; pub mod reg; @@ -142,7 +142,7 @@ impl Device { // SAFETY: The struct invariant ensures that we may access // this field without additional synchronization. let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; - bit_field.get(13, 1) == bindings::AUTONEG_ENABLE as u64 + bit_field.get(13, 1) == u64::from(bindings::AUTONEG_ENABLE) } /// Gets the current auto-negotiation state. @@ -163,20 +163,20 @@ impl Device { let phydev = self.0.get(); // SAFETY: The struct invariant ensures that we may access // this field without additional synchronization. - unsafe { (*phydev).speed = speed as i32 }; + unsafe { (*phydev).speed = speed as c_int }; } /// Sets duplex mode. pub fn set_duplex(&mut self, mode: DuplexMode) { let phydev = self.0.get(); let v = match mode { - DuplexMode::Full => bindings::DUPLEX_FULL as i32, - DuplexMode::Half => bindings::DUPLEX_HALF as i32, - DuplexMode::Unknown => bindings::DUPLEX_UNKNOWN as i32, + DuplexMode::Full => bindings::DUPLEX_FULL, + DuplexMode::Half => bindings::DUPLEX_HALF, + DuplexMode::Unknown => bindings::DUPLEX_UNKNOWN, }; // SAFETY: The struct invariant ensures that we may access // this field without additional synchronization. - unsafe { (*phydev).duplex = v }; + unsafe { (*phydev).duplex = v as c_int }; } /// Reads a PHY register. @@ -285,7 +285,7 @@ impl AsRef<kernel::device::Device> for Device { fn as_ref(&self) -> &kernel::device::Device { let phydev = self.0.get(); // SAFETY: The struct invariant ensures that `mdio.dev` is valid. - unsafe { kernel::device::Device::as_ref(addr_of_mut!((*phydev).mdio.dev)) } + unsafe { kernel::device::Device::from_raw(addr_of_mut!((*phydev).mdio.dev)) } } } @@ -312,9 +312,7 @@ impl<T: Driver> Adapter<T> { /// # Safety /// /// `phydev` must be passed by the corresponding callback in `phy_driver`. - unsafe extern "C" fn soft_reset_callback( - phydev: *mut bindings::phy_device, - ) -> crate::ffi::c_int { + unsafe extern "C" fn soft_reset_callback(phydev: *mut bindings::phy_device) -> c_int { from_result(|| { // SAFETY: This callback is called only in contexts // where we hold `phy_device->lock`, so the accessors on @@ -328,7 +326,7 @@ impl<T: Driver> Adapter<T> { /// # Safety /// /// `phydev` must be passed by the corresponding callback in `phy_driver`. - unsafe extern "C" fn probe_callback(phydev: *mut bindings::phy_device) -> crate::ffi::c_int { + unsafe extern "C" fn probe_callback(phydev: *mut bindings::phy_device) -> c_int { from_result(|| { // SAFETY: This callback is called only in contexts // where we can exclusively access `phy_device` because @@ -343,9 +341,7 @@ impl<T: Driver> Adapter<T> { /// # Safety /// /// `phydev` must be passed by the corresponding callback in `phy_driver`. - unsafe extern "C" fn get_features_callback( - phydev: *mut bindings::phy_device, - ) -> crate::ffi::c_int { + unsafe extern "C" fn get_features_callback(phydev: *mut bindings::phy_device) -> c_int { from_result(|| { // SAFETY: This callback is called only in contexts // where we hold `phy_device->lock`, so the accessors on @@ -359,7 +355,7 @@ impl<T: Driver> Adapter<T> { /// # Safety /// /// `phydev` must be passed by the corresponding callback in `phy_driver`. - unsafe extern "C" fn suspend_callback(phydev: *mut bindings::phy_device) -> crate::ffi::c_int { + unsafe extern "C" fn suspend_callback(phydev: *mut bindings::phy_device) -> c_int { from_result(|| { // SAFETY: The C core code ensures that the accessors on // `Device` are okay to call even though `phy_device->lock` @@ -373,7 +369,7 @@ impl<T: Driver> Adapter<T> { /// # Safety /// /// `phydev` must be passed by the corresponding callback in `phy_driver`. - unsafe extern "C" fn resume_callback(phydev: *mut bindings::phy_device) -> crate::ffi::c_int { + unsafe extern "C" fn resume_callback(phydev: *mut bindings::phy_device) -> c_int { from_result(|| { // SAFETY: The C core code ensures that the accessors on // `Device` are okay to call even though `phy_device->lock` @@ -387,9 +383,7 @@ impl<T: Driver> Adapter<T> { /// # Safety /// /// `phydev` must be passed by the corresponding callback in `phy_driver`. - unsafe extern "C" fn config_aneg_callback( - phydev: *mut bindings::phy_device, - ) -> crate::ffi::c_int { + unsafe extern "C" fn config_aneg_callback(phydev: *mut bindings::phy_device) -> c_int { from_result(|| { // SAFETY: This callback is called only in contexts // where we hold `phy_device->lock`, so the accessors on @@ -403,9 +397,7 @@ impl<T: Driver> Adapter<T> { /// # Safety /// /// `phydev` must be passed by the corresponding callback in `phy_driver`. - unsafe extern "C" fn read_status_callback( - phydev: *mut bindings::phy_device, - ) -> crate::ffi::c_int { + unsafe extern "C" fn read_status_callback(phydev: *mut bindings::phy_device) -> c_int { from_result(|| { // SAFETY: This callback is called only in contexts // where we hold `phy_device->lock`, so the accessors on @@ -422,12 +414,12 @@ impl<T: Driver> Adapter<T> { unsafe extern "C" fn match_phy_device_callback( phydev: *mut bindings::phy_device, _phydrv: *const bindings::phy_driver, - ) -> crate::ffi::c_int { + ) -> c_int { // SAFETY: This callback is called only in contexts // where we hold `phy_device->lock`, so the accessors on // `Device` are okay to call. let dev = unsafe { Device::from_raw(phydev) }; - T::match_phy_device(dev) as i32 + T::match_phy_device(dev).into() } /// # Safety @@ -507,7 +499,7 @@ pub const fn create_phy_driver<T: Driver>() -> DriverVTable { DriverVTable(Opaque::new(bindings::phy_driver { name: T::NAME.as_char_ptr().cast_mut(), flags: T::FLAGS, - phy_id: T::PHY_DEVICE_ID.id, + phy_id: T::PHY_DEVICE_ID.id(), phy_id_mask: T::PHY_DEVICE_ID.mask_as_int(), soft_reset: if T::HAS_SOFT_RESET { Some(Adapter::<T>::soft_reset_callback) @@ -691,42 +683,41 @@ impl Drop for Registration { /// /// Represents the kernel's `struct mdio_device_id`. This is used to find an appropriate /// PHY driver. -pub struct DeviceId { - id: u32, - mask: DeviceMask, -} +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct DeviceId(bindings::mdio_device_id); impl DeviceId { /// Creates a new instance with the exact match mask. pub const fn new_with_exact_mask(id: u32) -> Self { - DeviceId { - id, - mask: DeviceMask::Exact, - } + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Exact.as_int(), + }) } /// Creates a new instance with the model match mask. pub const fn new_with_model_mask(id: u32) -> Self { - DeviceId { - id, - mask: DeviceMask::Model, - } + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Model.as_int(), + }) } /// Creates a new instance with the vendor match mask. pub const fn new_with_vendor_mask(id: u32) -> Self { - DeviceId { - id, - mask: DeviceMask::Vendor, - } + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Vendor.as_int(), + }) } /// Creates a new instance with a custom match mask. pub const fn new_with_custom_mask(id: u32, mask: u32) -> Self { - DeviceId { - id, - mask: DeviceMask::Custom(mask), - } + Self(bindings::mdio_device_id { + phy_id: id, + phy_id_mask: DeviceMask::Custom(mask).as_int(), + }) } /// Creates a new instance from [`Driver`]. @@ -734,21 +725,29 @@ impl DeviceId { T::PHY_DEVICE_ID } - /// Get a `mask` as u32. + /// Get the MDIO device's PHY ID. + pub const fn id(&self) -> u32 { + self.0.phy_id + } + + /// Get the MDIO device's match mask. pub const fn mask_as_int(&self) -> u32 { - self.mask.as_int() + self.0.phy_id_mask } // macro use only #[doc(hidden)] pub const fn mdio_device_id(&self) -> bindings::mdio_device_id { - bindings::mdio_device_id { - phy_id: self.id, - phy_id_mask: self.mask.as_int(), - } + self.0 } } +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct mdio_device_id` +// and does not add additional invariants, so it's safe to transmute to `RawType`. +unsafe impl RawDeviceId for DeviceId { + type RawType = bindings::mdio_device_id; +} + enum DeviceMask { Exact, Model, @@ -849,19 +848,18 @@ impl DeviceMask { /// } /// }; /// -/// const _DEVICE_TABLE: [::kernel::bindings::mdio_device_id; 2] = [ -/// ::kernel::bindings::mdio_device_id { -/// phy_id: 0x00000001, -/// phy_id_mask: 0xffffffff, -/// }, -/// ::kernel::bindings::mdio_device_id { -/// phy_id: 0, -/// phy_id_mask: 0, -/// }, -/// ]; -/// #[cfg(MODULE)] -/// #[no_mangle] -/// static __mod_device_table__mdio__phydev: [::kernel::bindings::mdio_device_id; 2] = _DEVICE_TABLE; +/// const N: usize = 1; +/// +/// const TABLE: ::kernel::device_id::IdArray<::kernel::net::phy::DeviceId, (), N> = +/// ::kernel::device_id::IdArray::new_without_index([ +/// ::kernel::net::phy::DeviceId( +/// ::kernel::bindings::mdio_device_id { +/// phy_id: 0x00000001, +/// phy_id_mask: 0xffffffff, +/// }), +/// ]); +/// +/// ::kernel::module_device_table!("mdio", phydev, TABLE); /// ``` #[macro_export] macro_rules! module_phy_driver { @@ -872,20 +870,12 @@ macro_rules! module_phy_driver { }; (@device_table [$($dev:expr),+]) => { - // SAFETY: C will not read off the end of this constant since the last element is zero. - const _DEVICE_TABLE: [$crate::bindings::mdio_device_id; - $crate::module_phy_driver!(@count_devices $($dev),+) + 1] = [ - $($dev.mdio_device_id()),+, - $crate::bindings::mdio_device_id { - phy_id: 0, - phy_id_mask: 0 - } - ]; + const N: usize = $crate::module_phy_driver!(@count_devices $($dev),+); + + const TABLE: $crate::device_id::IdArray<$crate::net::phy::DeviceId, (), N> = + $crate::device_id::IdArray::new_without_index([ $(($dev,())),+, ]); - #[cfg(MODULE)] - #[no_mangle] - static __mod_device_table__mdio__phydev: [$crate::bindings::mdio_device_id; - $crate::module_phy_driver!(@count_devices $($dev),+) + 1] = _DEVICE_TABLE; + $crate::module_device_table!("mdio", phydev, TABLE); }; (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => { diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index 04f2d8ef29cb..b76b35265df2 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -2,7 +2,11 @@ //! Device Tree / Open Firmware abstractions. -use crate::{bindings, device_id::RawDeviceId, prelude::*}; +use crate::{ + bindings, + device_id::{RawDeviceId, RawDeviceIdIndex}, + prelude::*, +}; /// IdTable type for OF drivers. pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; @@ -12,17 +16,18 @@ pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; #[derive(Clone, Copy)] pub struct DeviceId(bindings::of_device_id); -// SAFETY: -// * `DeviceId` is a `#[repr(transparent)` wrapper of `struct of_device_id` and does not add -// additional invariants, so it's safe to transmute to `RawType`. -// * `DRIVER_DATA_OFFSET` is the offset to the `data` field. +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` and +// does not add additional invariants, so it's safe to transmute to `RawType`. unsafe impl RawDeviceId for DeviceId { type RawType = bindings::of_device_id; +} +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `data` field. +unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::of_device_id, data); fn index(&self) -> usize { - self.0.data as _ + self.0.data as usize } } @@ -34,10 +39,10 @@ impl DeviceId { // SAFETY: FFI type is valid to be zero-initialized. let mut of: bindings::of_device_id = unsafe { core::mem::zeroed() }; - // TODO: Use `clone_from_slice` once the corresponding types do match. + // TODO: Use `copy_from_slice` once stabilized for `const`. let mut i = 0; while i < src.len() { - of.compatible[i] = src[i] as _; + of.compatible[i] = src[i]; i += 1; } diff --git a/rust/kernel/opp.rs b/rust/kernel/opp.rs index a566fc3e7dcb..08126035d2c6 100644 --- a/rust/kernel/opp.rs +++ b/rust/kernel/opp.rs @@ -92,7 +92,7 @@ fn to_c_str_array(names: &[CString]) -> Result<KVec<*const u8>> { let mut list = KVec::with_capacity(names.len() + 1, GFP_KERNEL)?; for name in names.iter() { - list.push(name.as_ptr() as _, GFP_KERNEL)?; + list.push(name.as_ptr().cast(), GFP_KERNEL)?; } list.push(ptr::null(), GFP_KERNEL)?; @@ -103,7 +103,7 @@ fn to_c_str_array(names: &[CString]) -> Result<KVec<*const u8>> { /// /// Represents voltage in microvolts, wrapping a [`c_ulong`] value. /// -/// ## Examples +/// # Examples /// /// ``` /// use kernel::opp::MicroVolt; @@ -128,7 +128,7 @@ impl From<MicroVolt> for c_ulong { /// /// Represents power in microwatts, wrapping a [`c_ulong`] value. /// -/// ## Examples +/// # Examples /// /// ``` /// use kernel::opp::MicroWatt; @@ -153,7 +153,7 @@ impl From<MicroWatt> for c_ulong { /// /// The associated [`OPP`] is automatically removed when the [`Token`] is dropped. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to create an [`OPP`] dynamically. /// @@ -202,7 +202,7 @@ impl Drop for Token { /// Rust abstraction for the C `struct dev_pm_opp_data`, used to define operating performance /// points (OPPs) dynamically. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to create an [`OPP`] with [`Data`]. /// @@ -254,7 +254,7 @@ impl Data { /// [`OPP`] search options. /// -/// ## Examples +/// # Examples /// /// Defines how to search for an [`OPP`] in a [`Table`] relative to a frequency. /// @@ -326,7 +326,7 @@ impl Drop for ConfigToken { /// /// Rust abstraction for the C `struct dev_pm_opp_config`. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to set OPP property-name configuration for a [`Device`]. /// @@ -345,7 +345,7 @@ impl Drop for ConfigToken { /// impl ConfigOps for Driver {} /// /// fn configure(dev: &ARef<Device>) -> Result<ConfigToken> { -/// let name = CString::try_from_fmt(fmt!("{}", "slow"))?; +/// let name = CString::try_from_fmt(fmt!("slow"))?; /// /// // The OPP configuration is cleared once the [`ConfigToken`] goes out of scope. /// Config::<Driver>::new() @@ -514,9 +514,9 @@ impl<T: ConfigOps + Default> Config<T> { dev: *mut bindings::device, opp_table: *mut bindings::opp_table, opp: *mut bindings::dev_pm_opp, - _data: *mut kernel::ffi::c_void, + _data: *mut c_void, scaling_down: bool, - ) -> kernel::ffi::c_int { + ) -> c_int { from_result(|| { // SAFETY: 'dev' is guaranteed by the C code to be valid. let dev = unsafe { Device::get_device(dev) }; @@ -540,8 +540,8 @@ impl<T: ConfigOps + Default> Config<T> { old_opp: *mut bindings::dev_pm_opp, new_opp: *mut bindings::dev_pm_opp, regulators: *mut *mut bindings::regulator, - count: kernel::ffi::c_uint, - ) -> kernel::ffi::c_int { + count: c_uint, + ) -> c_int { from_result(|| { // SAFETY: 'dev' is guaranteed by the C code to be valid. let dev = unsafe { Device::get_device(dev) }; @@ -569,7 +569,7 @@ impl<T: ConfigOps + Default> Config<T> { /// /// Instances of this type are reference-counted. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to get OPP [`Table`] for a [`Cpumask`] and set its /// frequency. @@ -1011,7 +1011,7 @@ impl Drop for Table { /// /// A reference to the [`OPP`], &[`OPP`], isn't refcounted by the Rust code. /// -/// ## Examples +/// # Examples /// /// The following example demonstrates how to get [`OPP`] corresponding to a frequency value and /// configure the device with it. diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 8435f8132e38..887ee611b553 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -5,16 +5,15 @@ //! C header: [`include/linux/pci.h`](srctree/include/linux/pci.h) use crate::{ - alloc::flags::*, bindings, container_of, device, - device_id::RawDeviceId, + device_id::{RawDeviceId, RawDeviceIdIndex}, devres::Devres, driver, - error::{to_result, Result}, + error::{from_result, to_result, Result}, io::Io, io::IoRaw, str::CStr, - types::{ARef, ForeignOwnable, Opaque}, + types::{ARef, Opaque}, ThisModule, }; use core::{ @@ -66,41 +65,40 @@ impl<T: Driver + 'static> Adapter<T> { // `struct pci_dev`. // // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. - let pdev = unsafe { &*pdev.cast::<Device<device::Core>>() }; + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; - // SAFETY: `DeviceId` is a `#[repr(transparent)` wrapper of `struct pci_device_id` and + // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct pci_device_id` and // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::<DeviceId>() }; let info = T::ID_TABLE.info(id.index()); - match T::probe(pdev, info) { - Ok(data) => { - // Let the `struct pci_dev` own a reference of the driver's private data. - // SAFETY: By the type invariant `pdev.as_raw` returns a valid pointer to a - // `struct pci_dev`. - unsafe { bindings::pci_set_drvdata(pdev.as_raw(), data.into_foreign() as _) }; - } - Err(err) => return Error::to_errno(err), - } + from_result(|| { + let data = T::probe(pdev, info)?; - 0 + pdev.as_ref().set_drvdata(data); + Ok(0) + }) } extern "C" fn remove_callback(pdev: *mut bindings::pci_dev) { // SAFETY: The PCI bus only ever calls the remove callback with a valid pointer to a // `struct pci_dev`. - let ptr = unsafe { bindings::pci_get_drvdata(pdev) }.cast(); + // + // INVARIANT: `pdev` is valid for the duration of `remove_callback()`. + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to - // `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized - // `KBox<T>` pointer created through `KBox::into_foreign`. - let _ = unsafe { KBox::<T>::from_foreign(ptr) }; + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<T>>`. + let data = unsafe { pdev.as_ref().drvdata_obtain::<Pin<KBox<T>>>() }; + + T::unbind(pdev, data.as_ref()); } } /// Declares a kernel module that exposes a single PCI driver. /// -/// # Example +/// # Examples /// ///```ignore /// kernel::module_pci_driver! { @@ -161,17 +159,18 @@ impl DeviceId { } } -// SAFETY: -// * `DeviceId` is a `#[repr(transparent)` wrapper of `pci_device_id` and does not add -// additional invariants, so it's safe to transmute to `RawType`. -// * `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +// SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `pci_device_id` and does not add +// additional invariants, so it's safe to transmute to `RawType`. unsafe impl RawDeviceId for DeviceId { type RawType = bindings::pci_device_id; +} +// SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. +unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::pci_device_id, driver_data); fn index(&self) -> usize { - self.0.driver_data as _ + self.0.driver_data } } @@ -194,7 +193,7 @@ macro_rules! pci_device_table { /// The PCI driver trait. /// -/// # Example +/// # Examples /// ///``` /// # use kernel::{bindings, device::Core, pci}; @@ -206,7 +205,10 @@ macro_rules! pci_device_table { /// MODULE_PCI_TABLE, /// <MyDriver as pci::Driver>::IdInfo, /// [ -/// (pci::DeviceId::from_id(bindings::PCI_VENDOR_ID_REDHAT, bindings::PCI_ANY_ID as _), ()) +/// ( +/// pci::DeviceId::from_id(bindings::PCI_VENDOR_ID_REDHAT, bindings::PCI_ANY_ID as u32), +/// (), +/// ) /// ] /// ); /// @@ -241,6 +243,20 @@ pub trait Driver: Send { /// Called when a new platform device is added or discovered. /// Implementers should attempt to initialize the device here. fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> Result<Pin<KBox<Self>>>; + + /// Platform driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Self::drop`. + fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) { + let _ = (dev, this); + } } /// The PCI device representation. @@ -251,7 +267,8 @@ pub trait Driver: Send { /// /// # Invariants /// -/// A [`Device`] instance represents a valid `struct device` created by the C portion of the kernel. +/// A [`Device`] instance represents a valid `struct pci_dev` created by the C portion of the +/// kernel. #[repr(transparent)] pub struct Device<Ctx: device::DeviceContext = device::Normal>( Opaque<bindings::pci_dev>, @@ -330,7 +347,7 @@ impl<const SIZE: usize> Bar<SIZE> { // `ioptr` is valid by the safety requirements. // `num` is valid by the safety requirements. unsafe { - bindings::pci_iounmap(pdev.as_raw(), ioptr as _); + bindings::pci_iounmap(pdev.as_raw(), ioptr as *mut kernel::ffi::c_void); bindings::pci_release_region(pdev.as_raw(), num); } } @@ -398,19 +415,20 @@ impl Device { impl Device<device::Bound> { /// Mapps an entire PCI-BAR after performing a region-request on it. I/O operation bound checks /// can be performed on compile time for offsets (plus the requested type size) < SIZE. - pub fn iomap_region_sized<const SIZE: usize>( - &self, + pub fn iomap_region_sized<'a, const SIZE: usize>( + &'a self, bar: u32, - name: &CStr, - ) -> Result<Devres<Bar<SIZE>>> { - let bar = Bar::<SIZE>::new(self, bar, name)?; - let devres = Devres::new(self.as_ref(), bar, GFP_KERNEL)?; - - Ok(devres) + name: &'a CStr, + ) -> impl PinInit<Devres<Bar<SIZE>>, Error> + 'a { + Devres::new(self.as_ref(), Bar::<SIZE>::new(self, bar, name)) } /// Mapps an entire PCI-BAR after performing a region-request on it. - pub fn iomap_region(&self, bar: u32, name: &CStr) -> Result<Devres<Bar>> { + pub fn iomap_region<'a>( + &'a self, + bar: u32, + name: &'a CStr, + ) -> impl PinInit<Devres<Bar>, Error> + 'a { self.iomap_region_sized::<0>(bar, name) } } @@ -434,6 +452,8 @@ impl Device<device::Core> { kernel::impl_device_context_deref!(unsafe { Device }); kernel::impl_device_context_into_aref!(Device); +impl crate::dma::Device for Device<device::Core> {} + // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::types::AlwaysRefCounted for Device { fn inc_ref(&self) { @@ -454,7 +474,7 @@ impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> { let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; // SAFETY: `dev` points to a valid `struct device`. - unsafe { device::Device::as_ref(dev) } + unsafe { device::Device::from_raw(dev) } } } diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 5b21fa517e55..8f028c76f9fa 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -5,12 +5,14 @@ //! C header: [`include/linux/platform_device.h`](srctree/include/linux/platform_device.h) use crate::{ - bindings, container_of, device, driver, - error::{to_result, Result}, + acpi, bindings, container_of, + device::{self, Bound}, + driver, + error::{from_result, to_result, Result}, + io::{mem::IoRequest, Resource}, of, prelude::*, - str::CStr, - types::{ForeignOwnable, Opaque}, + types::Opaque, ThisModule, }; @@ -37,12 +39,18 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { None => core::ptr::null(), }; + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + // SAFETY: It's safe to set the fields of `struct platform_driver` on initialization. unsafe { (*pdrv.get()).driver.name = name.as_char_ptr(); (*pdrv.get()).probe = Some(Self::probe_callback); (*pdrv.get()).remove = Some(Self::remove_callback); (*pdrv.get()).driver.of_match_table = of_table; + (*pdrv.get()).driver.acpi_match_table = acpi_table; } // SAFETY: `pdrv` is guaranteed to be a valid `RegType`. @@ -61,30 +69,30 @@ impl<T: Driver + 'static> Adapter<T> { // `struct platform_device`. // // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. - let pdev = unsafe { &*pdev.cast::<Device<device::Core>>() }; - + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; let info = <Self as driver::Adapter>::id_info(pdev.as_ref()); - match T::probe(pdev, info) { - Ok(data) => { - // Let the `struct platform_device` own a reference of the driver's private data. - // SAFETY: By the type invariant `pdev.as_raw` returns a valid pointer to a - // `struct platform_device`. - unsafe { bindings::platform_set_drvdata(pdev.as_raw(), data.into_foreign() as _) }; - } - Err(err) => return Error::to_errno(err), - } - 0 + from_result(|| { + let data = T::probe(pdev, info)?; + + pdev.as_ref().set_drvdata(data); + Ok(0) + }) } extern "C" fn remove_callback(pdev: *mut bindings::platform_device) { - // SAFETY: `pdev` is a valid pointer to a `struct platform_device`. - let ptr = unsafe { bindings::platform_get_drvdata(pdev) }.cast(); + // SAFETY: The platform bus only ever calls the remove callback with a valid pointer to a + // `struct platform_device`. + // + // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. + let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to - // `probe_callback`, hence it's guaranteed that `ptr` points to a valid and initialized - // `KBox<T>` pointer created through `KBox::into_foreign`. - let _ = unsafe { KBox::<T>::from_foreign(ptr) }; + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<T>>`. + let data = unsafe { pdev.as_ref().drvdata_obtain::<Pin<KBox<T>>>() }; + + T::unbind(pdev, data.as_ref()); } } @@ -94,6 +102,10 @@ impl<T: Driver + 'static> driver::Adapter for Adapter<T> { fn of_id_table() -> Option<of::IdTable<Self::IdInfo>> { T::OF_ID_TABLE } + + fn acpi_id_table() -> Option<acpi::IdTable<Self::IdInfo>> { + T::ACPI_ID_TABLE + } } /// Declares a kernel module that exposes a single platform driver. @@ -120,10 +132,10 @@ macro_rules! module_platform_driver { /// /// Drivers must implement this trait in order to get a platform driver registered. /// -/// # Example +/// # Examples /// ///``` -/// # use kernel::{bindings, c_str, device::Core, of, platform}; +/// # use kernel::{acpi, bindings, c_str, device::Core, of, platform}; /// /// struct MyDriver; /// @@ -136,9 +148,19 @@ macro_rules! module_platform_driver { /// ] /// ); /// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// MODULE_ACPI_TABLE, +/// <MyDriver as platform::Driver>::IdInfo, +/// [ +/// (acpi::DeviceId::new(c_str!("LNUXBEEF")), ()) +/// ] +/// ); +/// /// impl platform::Driver for MyDriver { /// type IdInfo = (); /// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = Some(&ACPI_TABLE); /// /// fn probe( /// _pdev: &platform::Device<Core>, @@ -158,7 +180,10 @@ pub trait Driver: Send { type IdInfo: 'static; /// The table of OF device ids supported by the driver. - const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>>; + const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = None; /// Platform driver probe. /// @@ -166,6 +191,20 @@ pub trait Driver: Send { /// Implementers should attempt to initialize the device here. fn probe(dev: &Device<device::Core>, id_info: Option<&Self::IdInfo>) -> Result<Pin<KBox<Self>>>; + + /// Platform driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O + /// operations to gracefully tear down the device. + /// + /// Otherwise, release operations for driver resources should be performed in `Self::drop`. + fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) { + let _ = (dev, this); + } } /// The platform device representation. @@ -188,6 +227,61 @@ impl<Ctx: device::DeviceContext> Device<Ctx> { fn as_raw(&self) -> *mut bindings::platform_device { self.0.get() } + + /// Returns the resource at `index`, if any. + pub fn resource_by_index(&self, index: u32) -> Option<&Resource> { + // SAFETY: `self.as_raw()` returns a valid pointer to a `struct platform_device`. + let resource = unsafe { + bindings::platform_get_resource(self.as_raw(), bindings::IORESOURCE_MEM, index) + }; + + if resource.is_null() { + return None; + } + + // SAFETY: `resource` is a valid pointer to a `struct resource` as + // returned by `platform_get_resource`. + Some(unsafe { Resource::from_raw(resource) }) + } + + /// Returns the resource with a given `name`, if any. + pub fn resource_by_name(&self, name: &CStr) -> Option<&Resource> { + // SAFETY: `self.as_raw()` returns a valid pointer to a `struct + // platform_device` and `name` points to a valid C string. + let resource = unsafe { + bindings::platform_get_resource_byname( + self.as_raw(), + bindings::IORESOURCE_MEM, + name.as_char_ptr(), + ) + }; + + if resource.is_null() { + return None; + } + + // SAFETY: `resource` is a valid pointer to a `struct resource` as + // returned by `platform_get_resource`. + Some(unsafe { Resource::from_raw(resource) }) + } +} + +impl Device<Bound> { + /// Returns an `IoRequest` for the resource at `index`, if any. + pub fn io_request_by_index(&self, index: u32) -> Option<IoRequest<'_>> { + self.resource_by_index(index) + // SAFETY: `resource` is a valid resource for `&self` during the + // lifetime of the `IoRequest`. + .map(|resource| unsafe { IoRequest::new(self.as_ref(), resource) }) + } + + /// Returns an `IoRequest` for the resource with a given `name`, if any. + pub fn io_request_by_name(&self, name: &CStr) -> Option<IoRequest<'_>> { + self.resource_by_name(name) + // SAFETY: `resource` is a valid resource for `&self` during the + // lifetime of the `IoRequest`. + .map(|resource| unsafe { IoRequest::new(self.as_ref(), resource) }) + } } // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic @@ -195,6 +289,8 @@ impl<Ctx: device::DeviceContext> Device<Ctx> { kernel::impl_device_context_deref!(unsafe { Device }); kernel::impl_device_context_into_aref!(Device); +impl crate::dma::Device for Device<device::Core> {} + // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::types::AlwaysRefCounted for Device { fn inc_ref(&self) { @@ -215,7 +311,7 @@ impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> { let dev = unsafe { addr_of_mut!((*self.as_raw()).dev) }; // SAFETY: `dev` points to a valid `struct device`. - unsafe { device::Device::as_ref(dev) } + unsafe { device::Device::from_raw(dev) } } } diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 2f30a398dddd..25fe97aafd02 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -31,9 +31,9 @@ pub use super::{build_assert, build_error}; // `super::std_vendor` is hidden, which makes the macro inline for some reason. #[doc(no_inline)] pub use super::dbg; -pub use super::fmt; pub use super::{dev_alert, dev_crit, dev_dbg, dev_emerg, dev_err, dev_info, dev_notice, dev_warn}; pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn}; +pub use core::format_args as fmt; pub use super::{try_init, try_pin_init}; @@ -46,3 +46,5 @@ pub use super::{str::CStr, ThisModule}; pub use super::init::InPlaceInit; pub use super::current; + +pub use super::uaccess::UserPtr; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs index 9783d960a97a..2d743d78d220 100644 --- a/rust/kernel/print.rs +++ b/rust/kernel/print.rs @@ -8,10 +8,10 @@ use crate::{ ffi::{c_char, c_void}, + fmt, prelude::*, str::RawFormatter, }; -use core::fmt; // Called from `vsprintf` with format specifier `%pA`. #[expect(clippy::missing_safety_doc)] @@ -25,7 +25,7 @@ unsafe extern "C" fn rust_fmt_argument( // SAFETY: The C contract guarantees that `buf` is valid if it's less than `end`. let mut w = unsafe { RawFormatter::from_ptrs(buf.cast(), end.cast()) }; // SAFETY: TODO. - let _ = w.write_fmt(unsafe { *(ptr as *const fmt::Arguments<'_>) }); + let _ = w.write_fmt(unsafe { *ptr.cast::<fmt::Arguments<'_>>() }); w.pos().cast() } @@ -109,7 +109,7 @@ pub unsafe fn call_printk( bindings::_printk( format_string.as_ptr(), module_name.as_ptr(), - &args as *const _ as *const c_void, + core::ptr::from_ref(&args).cast::<c_void>(), ); } } @@ -129,7 +129,7 @@ pub fn call_printk_cont(args: fmt::Arguments<'_>) { unsafe { bindings::_printk( format_strings::CONT.as_ptr(), - &args as *const _ as *const c_void, + core::ptr::from_ref(&args).cast::<c_void>(), ); } } @@ -149,7 +149,7 @@ macro_rules! print_macro ( // takes borrows on the arguments, but does not extend the scope of temporaries. // Therefore, a `match` expression is used to keep them around, since // the scrutinee is kept until the end of the `match`. - match format_args!($($arg)+) { + match $crate::prelude::fmt!($($arg)+) { // SAFETY: This hidden macro should only be called by the documented // printing macros which ensure the format string is one of the fixed // ones. All `__LOG_PREFIX`s are null-terminated as they are generated @@ -168,7 +168,7 @@ macro_rules! print_macro ( // The `CONT` case. ($format_string:path, true, $($arg:tt)+) => ( $crate::print::call_printk_cont( - format_args!($($arg)+), + $crate::prelude::fmt!($($arg)+), ); ); ); diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index 8d978c896747..b8fe6be6fcc4 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -191,6 +191,12 @@ impl<K, V> RBTree<K, V> { } } + /// Returns true if this tree is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.root.rb_node.is_null() + } + /// Returns an iterator over the tree nodes, sorted by key. pub fn iter(&self) -> Iter<'_, K, V> { Iter { @@ -769,23 +775,14 @@ impl<'a, K, V> Cursor<'a, K, V> { // the tree cannot change. By the tree invariant, all nodes are valid. unsafe { bindings::rb_erase(&mut (*this).links, addr_of_mut!(self.tree.root)) }; - let current = match (prev, next) { - (_, Some(next)) => next, - (Some(prev), None) => prev, - (None, None) => { - return (None, node); - } - }; + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self.tree`. + let cursor = next.or(prev).map(|current| Self { + current, + tree: self.tree, + }); - ( - // INVARIANT: - // - `current` is a valid node in the [`RBTree`] pointed to by `self.tree`. - Some(Self { - current, - tree: self.tree, - }), - node, - ) + (cursor, node) } /// Remove the previous node, returning it if it exists. diff --git a/rust/kernel/regulator.rs b/rust/kernel/regulator.rs new file mode 100644 index 000000000000..65f3a125348f --- /dev/null +++ b/rust/kernel/regulator.rs @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Regulator abstractions, providing a standard kernel interface to control +//! voltage and current regulators. +//! +//! The intention is to allow systems to dynamically control regulator power +//! output in order to save power and prolong battery life. This applies to both +//! voltage regulators (where voltage output is controllable) and current sinks +//! (where current limit is controllable). +//! +//! C header: [`include/linux/regulator/consumer.h`](srctree/include/linux/regulator/consumer.h) +//! +//! Regulators are modeled in Rust with a collection of states. Each state may +//! enforce a given invariant, and they may convert between each other where applicable. +//! +//! See [Voltage and current regulator API](https://docs.kernel.org/driver-api/regulator.html) +//! for more information. + +use crate::{ + bindings, + device::Device, + error::{from_err_ptr, to_result, Result}, + prelude::*, +}; + +use core::{marker::PhantomData, mem::ManuallyDrop, ptr::NonNull}; + +mod private { + pub trait Sealed {} + + impl Sealed for super::Enabled {} + impl Sealed for super::Disabled {} + impl Sealed for super::Dynamic {} +} + +/// A trait representing the different states a [`Regulator`] can be in. +pub trait RegulatorState: private::Sealed + 'static { + /// Whether the regulator should be disabled when dropped. + const DISABLE_ON_DROP: bool; +} + +/// A state where the [`Regulator`] is known to be enabled. +/// +/// The `enable` reference count held by this state is decremented when it is +/// dropped. +pub struct Enabled; + +/// A state where this [`Regulator`] handle has not specifically asked for the +/// underlying regulator to be enabled. This means that this reference does not +/// own an `enable` reference count, but the regulator may still be on. +pub struct Disabled; + +/// A state that models the C API. The [`Regulator`] can be either enabled or +/// disabled, and the user is in control of the reference count. This is also +/// the default state. +/// +/// Use [`Regulator::is_enabled`] to check the regulator's current state. +pub struct Dynamic; + +impl RegulatorState for Enabled { + const DISABLE_ON_DROP: bool = true; +} + +impl RegulatorState for Disabled { + const DISABLE_ON_DROP: bool = false; +} + +impl RegulatorState for Dynamic { + const DISABLE_ON_DROP: bool = false; +} + +/// A trait that abstracts the ability to check if a [`Regulator`] is enabled. +pub trait IsEnabled: RegulatorState {} +impl IsEnabled for Disabled {} +impl IsEnabled for Dynamic {} + +/// An error that can occur when trying to convert a [`Regulator`] between states. +pub struct Error<State: RegulatorState> { + /// The error that occurred. + pub error: kernel::error::Error, + + /// The regulator that caused the error, so that the operation may be retried. + pub regulator: Regulator<State>, +} + +/// A `struct regulator` abstraction. +/// +/// # Examples +/// +/// ## Enabling a regulator +/// +/// This example uses [`Regulator<Enabled>`], which is suitable for drivers that +/// enable a regulator at probe time and leave them on until the device is +/// removed or otherwise shutdown. +/// +/// These users can store [`Regulator<Enabled>`] directly in their driver's +/// private data struct. +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::c_str; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Voltage, Regulator, Disabled, Enabled}; +/// fn enable(dev: &Device, min_voltage: Voltage, max_voltage: Voltage) -> Result { +/// // Obtain a reference to a (fictitious) regulator. +/// let regulator: Regulator<Disabled> = Regulator::<Disabled>::get(dev, c_str!("vcc"))?; +/// +/// // The voltage can be set before enabling the regulator if needed, e.g.: +/// regulator.set_voltage(min_voltage, max_voltage)?; +/// +/// // The same applies for `get_voltage()`, i.e.: +/// let voltage: Voltage = regulator.get_voltage()?; +/// +/// // Enables the regulator, consuming the previous value. +/// // +/// // From now on, the regulator is known to be enabled because of the type +/// // `Enabled`. +/// // +/// // If this operation fails, the `Error` will contain the regulator +/// // reference, so that the operation may be retried. +/// let regulator: Regulator<Enabled> = +/// regulator.try_into_enabled().map_err(|error| error.error)?; +/// +/// // The voltage can also be set after enabling the regulator, e.g.: +/// regulator.set_voltage(min_voltage, max_voltage)?; +/// +/// // The same applies for `get_voltage()`, i.e.: +/// let voltage: Voltage = regulator.get_voltage()?; +/// +/// // Dropping an enabled regulator will disable it. The refcount will be +/// // decremented. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// A more concise shortcut is available for enabling a regulator. This is +/// equivalent to `regulator_get_enable()`: +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::c_str; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Voltage, Regulator, Enabled}; +/// fn enable(dev: &Device) -> Result { +/// // Obtain a reference to a (fictitious) regulator and enable it. +/// let regulator: Regulator<Enabled> = Regulator::<Enabled>::get(dev, c_str!("vcc"))?; +/// +/// // Dropping an enabled regulator will disable it. The refcount will be +/// // decremented. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// ## Disabling a regulator +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Regulator, Enabled, Disabled}; +/// fn disable(dev: &Device, regulator: Regulator<Enabled>) -> Result { +/// // We can also disable an enabled regulator without reliquinshing our +/// // refcount: +/// // +/// // If this operation fails, the `Error` will contain the regulator +/// // reference, so that the operation may be retried. +/// let regulator: Regulator<Disabled> = +/// regulator.try_into_disabled().map_err(|error| error.error)?; +/// +/// // The refcount will be decremented when `regulator` is dropped. +/// drop(regulator); +/// +/// // ... +/// +/// Ok(()) +/// } +/// ``` +/// +/// ## Using [`Regulator<Dynamic>`] +/// +/// This example mimics the behavior of the C API, where the user is in +/// control of the enabled reference count. This is useful for drivers that +/// might call enable and disable to manage the `enable` reference count at +/// runtime, perhaps as a result of `open()` and `close()` calls or whatever +/// other driver-specific or subsystem-specific hooks. +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::c_str; +/// # use kernel::device::Device; +/// # use kernel::regulator::{Regulator, Dynamic}; +/// struct PrivateData { +/// regulator: Regulator<Dynamic>, +/// } +/// +/// // A fictictious probe function that obtains a regulator and sets it up. +/// fn probe(dev: &Device) -> Result<PrivateData> { +/// // Obtain a reference to a (fictitious) regulator. +/// let mut regulator = Regulator::<Dynamic>::get(dev, c_str!("vcc"))?; +/// +/// Ok(PrivateData { regulator }) +/// } +/// +/// // A fictictious function that indicates that the device is going to be used. +/// fn open(dev: &Device, data: &mut PrivateData) -> Result { +/// // Increase the `enabled` reference count. +/// data.regulator.enable()?; +/// +/// Ok(()) +/// } +/// +/// fn close(dev: &Device, data: &mut PrivateData) -> Result { +/// // Decrease the `enabled` reference count. +/// data.regulator.disable()?; +/// +/// Ok(()) +/// } +/// +/// fn remove(dev: &Device, data: PrivateData) -> Result { +/// // `PrivateData` is dropped here, which will drop the +/// // `Regulator<Dynamic>` in turn. +/// // +/// // The reference that was obtained by `regulator_get()` will be +/// // released, but it is up to the user to make sure that the number of calls +/// // to `enable()` and `disabled()` are balanced before this point. +/// Ok(()) +/// } +/// ``` +/// +/// # Invariants +/// +/// - `inner` is a non-null wrapper over a pointer to a `struct +/// regulator` obtained from [`regulator_get()`]. +/// +/// [`regulator_get()`]: https://docs.kernel.org/driver-api/regulator.html#c.regulator_get +pub struct Regulator<State = Dynamic> +where + State: RegulatorState, +{ + inner: NonNull<bindings::regulator>, + _phantom: PhantomData<State>, +} + +impl<T: RegulatorState> Regulator<T> { + /// Sets the voltage for the regulator. + /// + /// This can be used to ensure that the device powers up cleanly. + pub fn set_voltage(&self, min_voltage: Voltage, max_voltage: Voltage) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { + bindings::regulator_set_voltage( + self.inner.as_ptr(), + min_voltage.as_microvolts(), + max_voltage.as_microvolts(), + ) + }) + } + + /// Gets the current voltage of the regulator. + pub fn get_voltage(&self) -> Result<Voltage> { + // SAFETY: Safe as per the type invariants of `Regulator`. + let voltage = unsafe { bindings::regulator_get_voltage(self.inner.as_ptr()) }; + if voltage < 0 { + Err(kernel::error::Error::from_errno(voltage)) + } else { + Ok(Voltage::from_microvolts(voltage)) + } + } + + fn get_internal(dev: &Device, name: &CStr) -> Result<Regulator<T>> { + // SAFETY: It is safe to call `regulator_get()`, on a device pointer + // received from the C code. + let inner = from_err_ptr(unsafe { bindings::regulator_get(dev.as_raw(), name.as_ptr()) })?; + + // SAFETY: We can safely trust `inner` to be a pointer to a valid + // regulator if `ERR_PTR` was not returned. + let inner = unsafe { NonNull::new_unchecked(inner) }; + + Ok(Self { + inner, + _phantom: PhantomData, + }) + } + + fn enable_internal(&mut self) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { bindings::regulator_enable(self.inner.as_ptr()) }) + } + + fn disable_internal(&mut self) -> Result { + // SAFETY: Safe as per the type invariants of `Regulator`. + to_result(unsafe { bindings::regulator_disable(self.inner.as_ptr()) }) + } +} + +impl Regulator<Disabled> { + /// Obtains a [`Regulator`] instance from the system. + pub fn get(dev: &Device, name: &CStr) -> Result<Self> { + Regulator::get_internal(dev, name) + } + + /// Attempts to convert the regulator to an enabled state. + pub fn try_into_enabled(self) -> Result<Regulator<Enabled>, Error<Disabled>> { + // We will be transferring the ownership of our `regulator_get()` count to + // `Regulator<Enabled>`. + let mut regulator = ManuallyDrop::new(self); + + regulator + .enable_internal() + .map(|()| Regulator { + inner: regulator.inner, + _phantom: PhantomData, + }) + .map_err(|error| Error { + error, + regulator: ManuallyDrop::into_inner(regulator), + }) + } +} + +impl Regulator<Enabled> { + /// Obtains a [`Regulator`] instance from the system and enables it. + /// + /// This is equivalent to calling `regulator_get_enable()` in the C API. + pub fn get(dev: &Device, name: &CStr) -> Result<Self> { + Regulator::<Disabled>::get_internal(dev, name)? + .try_into_enabled() + .map_err(|error| error.error) + } + + /// Attempts to convert the regulator to a disabled state. + pub fn try_into_disabled(self) -> Result<Regulator<Disabled>, Error<Enabled>> { + // We will be transferring the ownership of our `regulator_get()` count + // to `Regulator<Disabled>`. + let mut regulator = ManuallyDrop::new(self); + + regulator + .disable_internal() + .map(|()| Regulator { + inner: regulator.inner, + _phantom: PhantomData, + }) + .map_err(|error| Error { + error, + regulator: ManuallyDrop::into_inner(regulator), + }) + } +} + +impl Regulator<Dynamic> { + /// Obtains a [`Regulator`] instance from the system. The current state of + /// the regulator is unknown and it is up to the user to manage the enabled + /// reference count. + /// + /// This closely mimics the behavior of the C API and can be used to + /// dynamically manage the enabled reference count at runtime. + pub fn get(dev: &Device, name: &CStr) -> Result<Self> { + Regulator::get_internal(dev, name) + } + + /// Increases the `enabled` reference count. + pub fn enable(&mut self) -> Result { + self.enable_internal() + } + + /// Decreases the `enabled` reference count. + pub fn disable(&mut self) -> Result { + self.disable_internal() + } +} + +impl<T: IsEnabled> Regulator<T> { + /// Checks if the regulator is enabled. + pub fn is_enabled(&self) -> bool { + // SAFETY: Safe as per the type invariants of `Regulator`. + unsafe { bindings::regulator_is_enabled(self.inner.as_ptr()) != 0 } + } +} + +impl<T: RegulatorState> Drop for Regulator<T> { + fn drop(&mut self) { + if T::DISABLE_ON_DROP { + // SAFETY: By the type invariants, we know that `self` owns a + // reference on the enabled refcount, so it is safe to relinquish it + // now. + unsafe { bindings::regulator_disable(self.inner.as_ptr()) }; + } + // SAFETY: By the type invariants, we know that `self` owns a reference, + // so it is safe to relinquish it now. + unsafe { bindings::regulator_put(self.inner.as_ptr()) }; + } +} + +/// A voltage. +/// +/// This type represents a voltage value in microvolts. +#[repr(transparent)] +#[derive(Copy, Clone, PartialEq, Eq)] +pub struct Voltage(i32); + +impl Voltage { + /// Creates a new `Voltage` from a value in microvolts. + pub fn from_microvolts(uv: i32) -> Self { + Self(uv) + } + + /// Returns the value of the voltage in microvolts as an [`i32`]. + pub fn as_microvolts(self) -> i32 { + self.0 + } +} diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index db4aa46bb121..0f4ae673256d 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -5,6 +5,8 @@ //! The [`Revocable`] type wraps other types and allows access to them to be revoked. The existence //! of a [`RevocableGuard`] ensures that objects remain valid. +use pin_init::Wrapper; + use crate::{bindings, prelude::*, sync::rcu, types::Opaque}; use core::{ marker::PhantomData, @@ -80,11 +82,11 @@ unsafe impl<T: Sync + Send> Sync for Revocable<T> {} impl<T> Revocable<T> { /// Creates a new revocable instance of the given data. - pub fn new(data: impl PinInit<T>) -> impl PinInit<Self> { - pin_init!(Self { + pub fn new<E>(data: impl PinInit<T, E>) -> impl PinInit<Self, E> { + try_pin_init!(Self { is_available: AtomicBool::new(true), data <- Opaque::pin_init(data), - }) + }? E) } /// Tries to access the revocable wrapped object. @@ -154,8 +156,10 @@ impl<T> Revocable<T> { /// # Safety /// /// Callers must ensure that there are no more concurrent users of the revocable object. - unsafe fn revoke_internal<const SYNC: bool>(&self) { - if self.is_available.swap(false, Ordering::Relaxed) { + unsafe fn revoke_internal<const SYNC: bool>(&self) -> bool { + let revoke = self.is_available.swap(false, Ordering::Relaxed); + + if revoke { if SYNC { // SAFETY: Just an FFI call, there are no further requirements. unsafe { bindings::synchronize_rcu() }; @@ -165,6 +169,8 @@ impl<T> Revocable<T> { // `compare_exchange` above that takes `is_available` from `true` to `false`. unsafe { drop_in_place(self.data.get()) }; } + + revoke } /// Revokes access to and drops the wrapped object. @@ -172,10 +178,13 @@ impl<T> Revocable<T> { /// Access to the object is revoked immediately to new callers of [`Revocable::try_access`], /// expecting that there are no concurrent users of the object. /// + /// Returns `true` if `&self` has been revoked with this call, `false` if it was revoked + /// already. + /// /// # Safety /// /// Callers must ensure that there are no more concurrent users of the revocable object. - pub unsafe fn revoke_nosync(&self) { + pub unsafe fn revoke_nosync(&self) -> bool { // SAFETY: By the safety requirement of this function, the caller ensures that nobody is // accessing the data anymore and hence we don't have to wait for the grace period to // finish. @@ -189,7 +198,10 @@ impl<T> Revocable<T> { /// If there are concurrent users of the object (i.e., ones that called /// [`Revocable::try_access`] beforehand and still haven't dropped the returned guard), this /// function waits for the concurrent access to complete before dropping the wrapped object. - pub fn revoke(&self) { + /// + /// Returns `true` if `&self` has been revoked with this call, `false` if it was revoked + /// already. + pub fn revoke(&self) -> bool { // SAFETY: By passing `true` we ask `revoke_internal` to wait for the grace period to // finish. unsafe { self.revoke_internal::<true>() } @@ -221,6 +233,10 @@ impl<T> PinnedDrop for Revocable<T> { /// /// The RCU read-side lock is held while the guard is alive. pub struct RevocableGuard<'a, T> { + // This can't use the `&'a T` type because references that appear in function arguments must + // not become dangling during the execution of the function, which can happen if the + // `RevocableGuard` is passed as a function argument and then dropped during execution of the + // function. data_ref: *const T, _rcu_guard: rcu::Guard, _p: PhantomData<&'a ()>, diff --git a/rust/kernel/seq_file.rs b/rust/kernel/seq_file.rs index 7a9403eb6e5b..8f199b1a3bb1 100644 --- a/rust/kernel/seq_file.rs +++ b/rust/kernel/seq_file.rs @@ -37,7 +37,7 @@ impl SeqFile { bindings::seq_printf( self.inner.get(), c_str!("%pA").as_char_ptr(), - &args as *const _ as *const crate::ffi::c_void, + core::ptr::from_ref(&args).cast::<crate::ffi::c_void>(), ); } } diff --git a/rust/kernel/sizes.rs b/rust/kernel/sizes.rs index 834c343e4170..661e680d9330 100644 --- a/rust/kernel/sizes.rs +++ b/rust/kernel/sizes.rs @@ -24,3 +24,27 @@ pub const SZ_128K: usize = bindings::SZ_128K as usize; pub const SZ_256K: usize = bindings::SZ_256K as usize; /// 0x00080000 pub const SZ_512K: usize = bindings::SZ_512K as usize; +/// 0x00100000 +pub const SZ_1M: usize = bindings::SZ_1M as usize; +/// 0x00200000 +pub const SZ_2M: usize = bindings::SZ_2M as usize; +/// 0x00400000 +pub const SZ_4M: usize = bindings::SZ_4M as usize; +/// 0x00800000 +pub const SZ_8M: usize = bindings::SZ_8M as usize; +/// 0x01000000 +pub const SZ_16M: usize = bindings::SZ_16M as usize; +/// 0x02000000 +pub const SZ_32M: usize = bindings::SZ_32M as usize; +/// 0x04000000 +pub const SZ_64M: usize = bindings::SZ_64M as usize; +/// 0x08000000 +pub const SZ_128M: usize = bindings::SZ_128M as usize; +/// 0x10000000 +pub const SZ_256M: usize = bindings::SZ_256M as usize; +/// 0x20000000 +pub const SZ_512M: usize = bindings::SZ_512M as usize; +/// 0x40000000 +pub const SZ_1G: usize = bindings::SZ_1G as usize; +/// 0x80000000 +pub const SZ_2G: usize = bindings::SZ_2G as usize; diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index a927db8e079c..6c892550c0ba 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -3,7 +3,7 @@ //! String representations. use crate::alloc::{flags::*, AllocError, KVec}; -use core::fmt::{self, Write}; +use crate::fmt::{self, Write}; use core::ops::{self, Deref, DerefMut, Index}; use crate::prelude::*; @@ -29,7 +29,7 @@ impl BStr { #[inline] pub const fn from_bytes(bytes: &[u8]) -> &Self { // SAFETY: `BStr` is transparent to `[u8]`. - unsafe { &*(bytes as *const [u8] as *const BStr) } + unsafe { &*(core::ptr::from_ref(bytes) as *const BStr) } } /// Strip a prefix from `self`. Delegates to [`slice::strip_prefix`]. @@ -54,14 +54,14 @@ impl fmt::Display for BStr { /// Formats printable ASCII characters, escaping the rest. /// /// ``` - /// # use kernel::{fmt, b_str, str::{BStr, CString}}; + /// # use kernel::{prelude::fmt, b_str, str::{BStr, CString}}; /// let ascii = b_str!("Hello, BStr!"); - /// let s = CString::try_from_fmt(fmt!("{}", ascii))?; - /// assert_eq!(s.as_bytes(), "Hello, BStr!".as_bytes()); + /// let s = CString::try_from_fmt(fmt!("{ascii}"))?; + /// assert_eq!(s.to_bytes(), "Hello, BStr!".as_bytes()); /// /// let non_ascii = b_str!("🦀"); - /// let s = CString::try_from_fmt(fmt!("{}", non_ascii))?; - /// assert_eq!(s.as_bytes(), "\\xf0\\x9f\\xa6\\x80".as_bytes()); + /// let s = CString::try_from_fmt(fmt!("{non_ascii}"))?; + /// assert_eq!(s.to_bytes(), "\\xf0\\x9f\\xa6\\x80".as_bytes()); /// # Ok::<(), kernel::error::Error>(()) /// ``` fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -85,15 +85,15 @@ impl fmt::Debug for BStr { /// escaping the rest. /// /// ``` - /// # use kernel::{fmt, b_str, str::{BStr, CString}}; + /// # use kernel::{prelude::fmt, b_str, str::{BStr, CString}}; /// // Embedded double quotes are escaped. /// let ascii = b_str!("Hello, \"BStr\"!"); - /// let s = CString::try_from_fmt(fmt!("{:?}", ascii))?; - /// assert_eq!(s.as_bytes(), "\"Hello, \\\"BStr\\\"!\"".as_bytes()); + /// let s = CString::try_from_fmt(fmt!("{ascii:?}"))?; + /// assert_eq!(s.to_bytes(), "\"Hello, \\\"BStr\\\"!\"".as_bytes()); /// /// let non_ascii = b_str!("😺"); - /// let s = CString::try_from_fmt(fmt!("{:?}", non_ascii))?; - /// assert_eq!(s.as_bytes(), "\"\\xf0\\x9f\\x98\\xba\"".as_bytes()); + /// let s = CString::try_from_fmt(fmt!("{non_ascii:?}"))?; + /// assert_eq!(s.to_bytes(), "\"\\xf0\\x9f\\x98\\xba\"".as_bytes()); /// # Ok::<(), kernel::error::Error>(()) /// ``` fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -175,6 +175,15 @@ macro_rules! b_str { }}; } +/// Returns a C pointer to the string. +// It is a free function rather than a method on an extension trait because: +// +// - error[E0379]: functions in trait impls cannot be declared const +#[inline] +pub const fn as_char_ptr_in_const_context(c_str: &CStr) -> *const c_char { + c_str.0.as_ptr() +} + /// Possible errors when using conversion functions in [`CStr`]. #[derive(Debug, Clone, Copy)] pub enum CStrConvertError { @@ -232,12 +241,12 @@ impl CStr { /// last at least `'a`. When `CStr` is alive, the memory pointed by `ptr` /// must not be mutated. #[inline] - pub unsafe fn from_char_ptr<'a>(ptr: *const crate::ffi::c_char) -> &'a Self { + pub unsafe fn from_char_ptr<'a>(ptr: *const c_char) -> &'a Self { // SAFETY: The safety precondition guarantees `ptr` is a valid pointer // to a `NUL`-terminated C string. let len = unsafe { bindings::strlen(ptr) } + 1; // SAFETY: Lifetime guaranteed by the safety precondition. - let bytes = unsafe { core::slice::from_raw_parts(ptr as _, len) }; + let bytes = unsafe { core::slice::from_raw_parts(ptr.cast(), len) }; // SAFETY: As `len` is returned by `strlen`, `bytes` does not contain interior `NUL`. // As we have added 1 to `len`, the last byte is known to be `NUL`. unsafe { Self::from_bytes_with_nul_unchecked(bytes) } @@ -290,27 +299,49 @@ impl CStr { #[inline] pub unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut CStr { // SAFETY: Properties of `bytes` guaranteed by the safety precondition. - unsafe { &mut *(bytes as *mut [u8] as *mut CStr) } + unsafe { &mut *(core::ptr::from_mut(bytes) as *mut CStr) } } /// Returns a C pointer to the string. + /// + /// Using this function in a const context is deprecated in favor of + /// [`as_char_ptr_in_const_context`] in preparation for replacing `CStr` with `core::ffi::CStr` + /// which does not have this method. #[inline] - pub const fn as_char_ptr(&self) -> *const crate::ffi::c_char { - self.0.as_ptr() + pub const fn as_char_ptr(&self) -> *const c_char { + as_char_ptr_in_const_context(self) } /// Convert the string to a byte slice without the trailing `NUL` byte. #[inline] - pub fn as_bytes(&self) -> &[u8] { + pub fn to_bytes(&self) -> &[u8] { &self.0[..self.len()] } + /// Convert the string to a byte slice without the trailing `NUL` byte. + /// + /// This function is deprecated in favor of [`Self::to_bytes`] in preparation for replacing + /// `CStr` with `core::ffi::CStr` which does not have this method. + #[inline] + pub fn as_bytes(&self) -> &[u8] { + self.to_bytes() + } + /// Convert the string to a byte slice containing the trailing `NUL` byte. #[inline] - pub const fn as_bytes_with_nul(&self) -> &[u8] { + pub const fn to_bytes_with_nul(&self) -> &[u8] { &self.0 } + /// Convert the string to a byte slice containing the trailing `NUL` byte. + /// + /// This function is deprecated in favor of [`Self::to_bytes_with_nul`] in preparation for + /// replacing `CStr` with `core::ffi::CStr` which does not have this method. + #[inline] + pub const fn as_bytes_with_nul(&self) -> &[u8] { + self.to_bytes_with_nul() + } + /// Yields a [`&str`] slice if the [`CStr`] contains valid UTF-8. /// /// If the contents of the [`CStr`] are valid UTF-8 data, this @@ -429,20 +460,20 @@ impl fmt::Display for CStr { /// /// ``` /// # use kernel::c_str; - /// # use kernel::fmt; + /// # use kernel::prelude::fmt; /// # use kernel::str::CStr; /// # use kernel::str::CString; /// let penguin = c_str!("🐧"); - /// let s = CString::try_from_fmt(fmt!("{}", penguin))?; - /// assert_eq!(s.as_bytes_with_nul(), "\\xf0\\x9f\\x90\\xa7\0".as_bytes()); + /// let s = CString::try_from_fmt(fmt!("{penguin}"))?; + /// assert_eq!(s.to_bytes_with_nul(), "\\xf0\\x9f\\x90\\xa7\0".as_bytes()); /// /// let ascii = c_str!("so \"cool\""); - /// let s = CString::try_from_fmt(fmt!("{}", ascii))?; - /// assert_eq!(s.as_bytes_with_nul(), "so \"cool\"\0".as_bytes()); + /// let s = CString::try_from_fmt(fmt!("{ascii}"))?; + /// assert_eq!(s.to_bytes_with_nul(), "so \"cool\"\0".as_bytes()); /// # Ok::<(), kernel::error::Error>(()) /// ``` fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - for &c in self.as_bytes() { + for &c in self.to_bytes() { if (0x20..0x7f).contains(&c) { // Printable character. f.write_char(c as char)?; @@ -459,16 +490,16 @@ impl fmt::Debug for CStr { /// /// ``` /// # use kernel::c_str; - /// # use kernel::fmt; + /// # use kernel::prelude::fmt; /// # use kernel::str::CStr; /// # use kernel::str::CString; /// let penguin = c_str!("🐧"); - /// let s = CString::try_from_fmt(fmt!("{:?}", penguin))?; + /// let s = CString::try_from_fmt(fmt!("{penguin:?}"))?; /// assert_eq!(s.as_bytes_with_nul(), "\"\\xf0\\x9f\\x90\\xa7\"\0".as_bytes()); /// /// // Embedded double quotes are escaped. /// let ascii = c_str!("so \"cool\""); - /// let s = CString::try_from_fmt(fmt!("{:?}", ascii))?; + /// let s = CString::try_from_fmt(fmt!("{ascii:?}"))?; /// assert_eq!(s.as_bytes_with_nul(), "\"so \\\"cool\\\"\"\0".as_bytes()); /// # Ok::<(), kernel::error::Error>(()) /// ``` @@ -578,7 +609,7 @@ mod tests { macro_rules! format { ($($f:tt)*) => ({ - CString::try_from_fmt(::kernel::fmt!($($f)*))?.to_str()? + CString::try_from_fmt(fmt!($($f)*))?.to_str()? }) } @@ -728,9 +759,9 @@ impl RawFormatter { pub(crate) unsafe fn from_ptrs(pos: *mut u8, end: *mut u8) -> Self { // INVARIANT: The safety requirements guarantee the type invariants. Self { - beg: pos as _, - pos: pos as _, - end: end as _, + beg: pos as usize, + pos: pos as usize, + end: end as usize, } } @@ -755,7 +786,7 @@ impl RawFormatter { /// /// N.B. It may point to invalid memory. pub(crate) fn pos(&self) -> *mut u8 { - self.pos as _ + self.pos as *mut u8 } /// Returns the number of bytes written to the formatter. @@ -840,14 +871,14 @@ impl fmt::Write for Formatter { /// # Examples /// /// ``` -/// use kernel::{str::CString, fmt}; +/// use kernel::{str::CString, prelude::fmt}; /// /// let s = CString::try_from_fmt(fmt!("{}{}{}", "abc", 10, 20))?; -/// assert_eq!(s.as_bytes_with_nul(), "abc1020\0".as_bytes()); +/// assert_eq!(s.to_bytes_with_nul(), "abc1020\0".as_bytes()); /// /// let tmp = "testing"; /// let s = CString::try_from_fmt(fmt!("{tmp}{}", 123))?; -/// assert_eq!(s.as_bytes_with_nul(), "testing123\0".as_bytes()); +/// assert_eq!(s.to_bytes_with_nul(), "testing123\0".as_bytes()); /// /// // This fails because it has an embedded `NUL` byte. /// let s = CString::try_from_fmt(fmt!("a\0b{}", 123)); @@ -917,7 +948,7 @@ impl<'a> TryFrom<&'a CStr> for CString { fn try_from(cstr: &'a CStr) -> Result<CString, AllocError> { let mut buf = KVec::new(); - buf.extend_from_slice(cstr.as_bytes_with_nul(), GFP_KERNEL)?; + buf.extend_from_slice(cstr.to_bytes_with_nul(), GFP_KERNEL)?; // INVARIANT: The `CStr` and `CString` types have the same invariants for // the string data, and we copied it over without changes. @@ -930,9 +961,3 @@ impl fmt::Debug for CString { fmt::Debug::fmt(&**self, f) } } - -/// A convenience alias for [`core::format_args`]. -#[macro_export] -macro_rules! fmt { - ($($f:tt)*) => ( ::core::format_args!($($f)*) ) -} diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 36a719015583..00f9b558a3ad 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -10,6 +10,8 @@ use crate::types::Opaque; use pin_init; mod arc; +pub mod aref; +pub mod completion; mod condvar; pub mod lock; mod locked_by; @@ -17,6 +19,7 @@ pub mod poll; pub mod rcu; pub use arc::{Arc, ArcBorrow, UniqueArc}; +pub use completion::Completion; pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; @@ -39,7 +42,7 @@ impl LockClassKey { /// Initializes a dynamically allocated lock class key. In the common case of using a /// statically allocated lock class key, the static_lock_class! macro should be used instead. /// - /// # Example + /// # Examples /// ``` /// # use kernel::c_str; /// # use kernel::alloc::KBox; @@ -93,8 +96,11 @@ impl PinnedDrop for LockClassKey { macro_rules! static_lock_class { () => {{ static CLASS: $crate::sync::LockClassKey = - // SAFETY: lockdep expects uninitialized memory when it's handed a statically allocated - // lock_class_key + // Lockdep expects uninitialized memory when it's handed a statically allocated `struct + // lock_class_key`. + // + // SAFETY: `LockClassKey` transparently wraps `Opaque` which permits uninitialized + // memory. unsafe { ::core::mem::MaybeUninit::uninit().assume_init() }; $crate::prelude::Pin::static_ref(&CLASS) }}; diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index c7af0aa48a0a..63a66761d0c7 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -19,12 +19,14 @@ use crate::{ alloc::{AllocError, Flags, KBox}, bindings, + ffi::c_void, init::InPlaceInit, try_init, types::{ForeignOwnable, Opaque}, }; use core::{ alloc::Layout, + borrow::{Borrow, BorrowMut}, fmt, marker::PhantomData, mem::{ManuallyDrop, MaybeUninit}, @@ -140,10 +142,9 @@ pub struct Arc<T: ?Sized> { _p: PhantomData<ArcInner<T>>, } -#[doc(hidden)] #[pin_data] #[repr(C)] -pub struct ArcInner<T: ?Sized> { +struct ArcInner<T: ?Sized> { refcount: Opaque<bindings::refcount_t>, data: T, } @@ -372,20 +373,22 @@ impl<T: ?Sized> Arc<T> { } } -// SAFETY: The `into_foreign` function returns a pointer that is well-aligned. +// SAFETY: The pointer returned by `into_foreign` comes from a well aligned +// pointer to `ArcInner<T>`. unsafe impl<T: 'static> ForeignOwnable for Arc<T> { - type PointedTo = ArcInner<T>; + const FOREIGN_ALIGN: usize = core::mem::align_of::<ArcInner<T>>(); + type Borrowed<'a> = ArcBorrow<'a, T>; type BorrowedMut<'a> = Self::Borrowed<'a>; - fn into_foreign(self) -> *mut Self::PointedTo { - ManuallyDrop::new(self).ptr.as_ptr() + fn into_foreign(self) -> *mut c_void { + ManuallyDrop::new(self).ptr.as_ptr().cast() } - unsafe fn from_foreign(ptr: *mut Self::PointedTo) -> Self { + unsafe fn from_foreign(ptr: *mut c_void) -> Self { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. - let inner = unsafe { NonNull::new_unchecked(ptr) }; + let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) }; // SAFETY: By the safety requirement of this function, we know that `ptr` came from // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and @@ -393,20 +396,20 @@ unsafe impl<T: 'static> ForeignOwnable for Arc<T> { unsafe { Self::from_inner(inner) } } - unsafe fn borrow<'a>(ptr: *mut Self::PointedTo) -> ArcBorrow<'a, T> { + unsafe fn borrow<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. - let inner = unsafe { NonNull::new_unchecked(ptr) }; + let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) }; // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive // for the lifetime of the returned value. unsafe { ArcBorrow::new(inner) } } - unsafe fn borrow_mut<'a>(ptr: *mut Self::PointedTo) -> ArcBorrow<'a, T> { + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> ArcBorrow<'a, T> { // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety // requirements for `borrow`. - unsafe { Self::borrow(ptr) } + unsafe { <Self as ForeignOwnable>::borrow(ptr) } } } @@ -426,6 +429,31 @@ impl<T: ?Sized> AsRef<T> for Arc<T> { } } +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// # use kernel::sync::Arc; +/// struct Foo<B: Borrow<u32>>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Shared instance. +/// let arc = Arc::new(1, GFP_KERNEL)?; +/// let shared = Foo(arc.clone()); +/// +/// let i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&i); +/// # Ok::<(), Error>(()) +/// ``` +impl<T: ?Sized> Borrow<T> for Arc<T> { + fn borrow(&self) -> &T { + self.deref() + } +} + impl<T: ?Sized> Clone for Arc<T> { fn clone(&self) -> Self { // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is @@ -834,6 +862,56 @@ impl<T: ?Sized> DerefMut for UniqueArc<T> { } } +/// # Examples +/// +/// ``` +/// # use core::borrow::Borrow; +/// # use kernel::sync::UniqueArc; +/// struct Foo<B: Borrow<u32>>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `UniqueArc`. +/// let arc = UniqueArc::new(1, GFP_KERNEL)?; +/// let shared = Foo(arc); +/// +/// let i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&i); +/// # Ok::<(), Error>(()) +/// ``` +impl<T: ?Sized> Borrow<T> for UniqueArc<T> { + fn borrow(&self) -> &T { + self.deref() + } +} + +/// # Examples +/// +/// ``` +/// # use core::borrow::BorrowMut; +/// # use kernel::sync::UniqueArc; +/// struct Foo<B: BorrowMut<u32>>(B); +/// +/// // Owned instance. +/// let owned = Foo(1); +/// +/// // Owned instance using `UniqueArc`. +/// let arc = UniqueArc::new(1, GFP_KERNEL)?; +/// let shared = Foo(arc); +/// +/// let mut i = 1; +/// // Borrowed from `i`. +/// let borrowed = Foo(&mut i); +/// # Ok::<(), Error>(()) +/// ``` +impl<T: ?Sized> BorrowMut<T> for UniqueArc<T> { + fn borrow_mut(&mut self) -> &mut T { + self.deref_mut() + } +} + impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(self.deref(), f) diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs new file mode 100644 index 000000000000..dbd77bb68617 --- /dev/null +++ b/rust/kernel/sync/aref.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Internal reference counting support. + +use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; + +/// Types that are _always_ reference counted. +/// +/// It allows such types to define their own custom ref increment and decrement functions. +/// Additionally, it allows users to convert from a shared reference `&T` to an owned reference +/// [`ARef<T>`]. +/// +/// This is usually implemented by wrappers to existing structures on the C side of the code. For +/// Rust code, the recommendation is to use [`Arc`](crate::sync::Arc) to create reference-counted +/// instances of a type. +/// +/// # Safety +/// +/// Implementers must ensure that increments to the reference count keep the object alive in memory +/// at least until matching decrements are performed. +/// +/// Implementers must also ensure that all instances are reference-counted. (Otherwise they +/// won't be able to honour the requirement that [`AlwaysRefCounted::inc_ref`] keep the object +/// alive.) +pub unsafe trait AlwaysRefCounted { + /// Increments the reference count on the object. + fn inc_ref(&self); + + /// Decrements the reference count on the object. + /// + /// Frees the object when the count reaches zero. + /// + /// # Safety + /// + /// Callers must ensure that there was a previous matching increment to the reference count, + /// and that the object is no longer used after its reference count is decremented (as it may + /// result in the object being freed), unless the caller owns another increment on the refcount + /// (e.g., it calls [`AlwaysRefCounted::inc_ref`] twice, then calls + /// [`AlwaysRefCounted::dec_ref`] once). + unsafe fn dec_ref(obj: NonNull<Self>); +} + +/// An owned reference to an always-reference-counted object. +/// +/// The object's reference count is automatically decremented when an instance of [`ARef`] is +/// dropped. It is also automatically incremented when a new instance is created via +/// [`ARef::clone`]. +/// +/// # Invariants +/// +/// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`ARef`] instance. In +/// particular, the [`ARef`] instance owns an increment on the underlying object's reference count. +pub struct ARef<T: AlwaysRefCounted> { + ptr: NonNull<T>, + _p: PhantomData<T>, +} + +// SAFETY: It is safe to send `ARef<T>` to another thread when the underlying `T` is `Sync` because +// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs +// `T` to be `Send` because any thread that has an `ARef<T>` may ultimately access `T` using a +// mutable reference, for example, when the reference count reaches zero and `T` is dropped. +unsafe impl<T: AlwaysRefCounted + Sync + Send> Send for ARef<T> {} + +// SAFETY: It is safe to send `&ARef<T>` to another thread when the underlying `T` is `Sync` +// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, +// it needs `T` to be `Send` because any thread that has a `&ARef<T>` may clone it and get an +// `ARef<T>` on that thread, so the thread may ultimately access `T` using a mutable reference, for +// example, when the reference count reaches zero and `T` is dropped. +unsafe impl<T: AlwaysRefCounted + Sync + Send> Sync for ARef<T> {} + +impl<T: AlwaysRefCounted> ARef<T> { + /// Creates a new instance of [`ARef`]. + /// + /// It takes over an increment of the reference count on the underlying object. + /// + /// # Safety + /// + /// Callers must ensure that the reference count was incremented at least once, and that they + /// are properly relinquishing one increment. That is, if there is only one increment, callers + /// must not use the underlying object anymore -- it is only safe to do so via the newly + /// created [`ARef`]. + pub unsafe fn from_raw(ptr: NonNull<T>) -> Self { + // INVARIANT: The safety requirements guarantee that the new instance now owns the + // increment on the refcount. + Self { + ptr, + _p: PhantomData, + } + } + + /// Consumes the `ARef`, returning a raw pointer. + /// + /// This function does not change the refcount. After calling this function, the caller is + /// responsible for the refcount previously managed by the `ARef`. + /// + /// # Examples + /// + /// ``` + /// use core::ptr::NonNull; + /// use kernel::types::{ARef, AlwaysRefCounted}; + /// + /// struct Empty {} + /// + /// # // SAFETY: TODO. + /// unsafe impl AlwaysRefCounted for Empty { + /// fn inc_ref(&self) {} + /// unsafe fn dec_ref(_obj: NonNull<Self>) {} + /// } + /// + /// let mut data = Empty {}; + /// let ptr = NonNull::<Empty>::new(&mut data).unwrap(); + /// # // SAFETY: TODO. + /// let data_ref: ARef<Empty> = unsafe { ARef::from_raw(ptr) }; + /// let raw_ptr: NonNull<Empty> = ARef::into_raw(data_ref); + /// + /// assert_eq!(ptr, raw_ptr); + /// ``` + pub fn into_raw(me: Self) -> NonNull<T> { + ManuallyDrop::new(me).ptr + } +} + +impl<T: AlwaysRefCounted> Clone for ARef<T> { + fn clone(&self) -> Self { + self.inc_ref(); + // SAFETY: We just incremented the refcount above. + unsafe { Self::from_raw(self.ptr) } + } +} + +impl<T: AlwaysRefCounted> Deref for ARef<T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: The type invariants guarantee that the object is valid. + unsafe { self.ptr.as_ref() } + } +} + +impl<T: AlwaysRefCounted> From<&T> for ARef<T> { + fn from(b: &T) -> Self { + b.inc_ref(); + // SAFETY: We just incremented the refcount above. + unsafe { Self::from_raw(NonNull::from(b)) } + } +} + +impl<T: AlwaysRefCounted> Drop for ARef<T> { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that the `ARef` owns the reference we're about to + // decrement. + unsafe { T::dec_ref(self.ptr) }; + } +} diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs new file mode 100644 index 000000000000..c50012a940a3 --- /dev/null +++ b/rust/kernel/sync/completion.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Completion support. +//! +//! Reference: <https://docs.kernel.org/scheduler/completion.html> +//! +//! C header: [`include/linux/completion.h`](srctree/include/linux/completion.h) + +use crate::{bindings, prelude::*, types::Opaque}; + +/// Synchronization primitive to signal when a certain task has been completed. +/// +/// The [`Completion`] synchronization primitive signals when a certain task has been completed by +/// waking up other tasks that have been queued up to wait for the [`Completion`] to be completed. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::{Arc, Completion}; +/// use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +/// +/// #[pin_data] +/// struct MyTask { +/// #[pin] +/// work: Work<MyTask>, +/// #[pin] +/// done: Completion, +/// } +/// +/// impl_has_work! { +/// impl HasWork<Self> for MyTask { self.work } +/// } +/// +/// impl MyTask { +/// fn new() -> Result<Arc<Self>> { +/// let this = Arc::pin_init(pin_init!(MyTask { +/// work <- new_work!("MyTask::work"), +/// done <- Completion::new(), +/// }), GFP_KERNEL)?; +/// +/// let _ = workqueue::system().enqueue(this.clone()); +/// +/// Ok(this) +/// } +/// +/// fn wait_for_completion(&self) { +/// self.done.wait_for_completion(); +/// +/// pr_info!("Completion: task complete\n"); +/// } +/// } +/// +/// impl WorkItem for MyTask { +/// type Pointer = Arc<MyTask>; +/// +/// fn run(this: Arc<MyTask>) { +/// // process this task +/// this.done.complete_all(); +/// } +/// } +/// +/// let task = MyTask::new()?; +/// task.wait_for_completion(); +/// # Ok::<(), Error>(()) +/// ``` +#[pin_data] +pub struct Completion { + #[pin] + inner: Opaque<bindings::completion>, +} + +// SAFETY: `Completion` is safe to be send to any task. +unsafe impl Send for Completion {} + +// SAFETY: `Completion` is safe to be accessed concurrently. +unsafe impl Sync for Completion {} + +impl Completion { + /// Create an initializer for a new [`Completion`]. + pub fn new() -> impl PinInit<Self> { + pin_init!(Self { + inner <- Opaque::ffi_init(|slot: *mut bindings::completion| { + // SAFETY: `slot` is a valid pointer to an uninitialized `struct completion`. + unsafe { bindings::init_completion(slot) }; + }), + }) + } + + fn as_raw(&self) -> *mut bindings::completion { + self.inner.get() + } + + /// Signal all tasks waiting on this completion. + /// + /// This method wakes up all tasks waiting on this completion; after this operation the + /// completion is permanently done, i.e. signals all current and future waiters. + pub fn complete_all(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. + unsafe { bindings::complete_all(self.as_raw()) }; + } + + /// Wait for completion of a task. + /// + /// This method waits for the completion of a task; it is not interruptible and there is no + /// timeout. + /// + /// See also [`Completion::complete_all`]. + pub fn wait_for_completion(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. + unsafe { bindings::wait_for_completion(self.as_raw()) }; + } +} diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index caebf03f553b..c6ec64295c9f 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -216,6 +216,7 @@ impl CondVar { /// This method behaves like `notify_one`, except that it hints to the scheduler that the /// current thread is about to go to sleep, so it should schedule the target thread on the same /// CPU. + #[inline] pub fn notify_sync(&self) { // SAFETY: `wait_queue_head` points to valid memory. unsafe { bindings::__wake_up_sync(self.wait_queue_head.get(), TASK_NORMAL) }; @@ -225,6 +226,7 @@ impl CondVar { /// /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost /// completely (as opposed to automatically waking up the next waiter). + #[inline] pub fn notify_one(&self) { self.notify(1); } @@ -233,6 +235,7 @@ impl CondVar { /// /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost /// completely (as opposed to automatically waking up the next waiter). + #[inline] pub fn notify_all(&self) { self.notify(0); } diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index e82fa5be289c..27202beef90c 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -175,6 +175,8 @@ impl<T: ?Sized, B: Backend> Lock<T, B> { /// Tries to acquire the lock. /// /// Returns a guard that can be used to access the data protected by the lock if successful. + // `Option<T>` is not `#[must_use]` even if `T` is, thus the attribute is needed here. + #[must_use = "if unused, the lock will be immediately unlocked"] pub fn try_lock(&self) -> Option<Guard<'_, T, B>> { // SAFETY: The constructor of the type calls `init`, so the existence of the object proves // that `init` was called. diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index d7e6e59e124b..0ec985d560c8 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -9,9 +9,8 @@ use crate::{ fs::File, prelude::*, sync::{CondVar, LockClassKey}, - types::Opaque, }; -use core::ops::Deref; +use core::{marker::PhantomData, ops::Deref}; /// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -23,58 +22,43 @@ macro_rules! new_poll_condvar { }; } -/// Wraps the kernel's `struct poll_table`. +/// Wraps the kernel's `poll_table`. /// /// # Invariants /// -/// This struct contains a valid `struct poll_table`. -/// -/// For a `struct poll_table` to be valid, its `_qproc` function must follow the safety -/// requirements of `_qproc` functions: -/// -/// * The `_qproc` function is given permission to enqueue a waiter to the provided `poll_table` -/// during the call. Once the waiter is removed and an rcu grace period has passed, it must no -/// longer access the `wait_queue_head`. +/// The pointer must be null or reference a valid `poll_table`. #[repr(transparent)] -pub struct PollTable(Opaque<bindings::poll_table>); +pub struct PollTable<'a> { + table: *mut bindings::poll_table, + _lifetime: PhantomData<&'a bindings::poll_table>, +} -impl PollTable { - /// Creates a reference to a [`PollTable`] from a valid pointer. +impl<'a> PollTable<'a> { + /// Creates a [`PollTable`] from a valid pointer. /// /// # Safety /// - /// The caller must ensure that for the duration of `'a`, the pointer will point at a valid poll - /// table (as defined in the type invariants). - /// - /// The caller must also ensure that the `poll_table` is only accessed via the returned - /// reference for the duration of `'a`. - pub unsafe fn from_ptr<'a>(ptr: *mut bindings::poll_table) -> &'a mut PollTable { - // SAFETY: The safety requirements guarantee the validity of the dereference, while the - // `PollTable` type being transparent makes the cast ok. - unsafe { &mut *ptr.cast() } - } - - fn get_qproc(&self) -> bindings::poll_queue_proc { - let ptr = self.0.get(); - // SAFETY: The `ptr` is valid because it originates from a reference, and the `_qproc` - // field is not modified concurrently with this call since we have an immutable reference. - unsafe { (*ptr)._qproc } + /// The pointer must be null or reference a valid `poll_table` for the duration of `'a`. + pub unsafe fn from_raw(table: *mut bindings::poll_table) -> Self { + // INVARIANTS: The safety requirements are the same as the struct invariants. + PollTable { + table, + _lifetime: PhantomData, + } } /// Register this [`PollTable`] with the provided [`PollCondVar`], so that it can be notified /// using the condition variable. - pub fn register_wait(&mut self, file: &File, cv: &PollCondVar) { - if let Some(qproc) = self.get_qproc() { - // SAFETY: The pointers to `file` and `self` need to be valid for the duration of this - // call to `qproc`, which they are because they are references. - // - // The `cv.wait_queue_head` pointer must be valid until an rcu grace period after the - // waiter is removed. The `PollCondVar` is pinned, so before `cv.wait_queue_head` can - // be destroyed, the destructor must run. That destructor first removes all waiters, - // and then waits for an rcu grace period. Therefore, `cv.wait_queue_head` is valid for - // long enough. - unsafe { qproc(file.as_ptr() as _, cv.wait_queue_head.get(), self.0.get()) }; - } + pub fn register_wait(&self, file: &File, cv: &PollCondVar) { + // SAFETY: + // * `file.as_ptr()` references a valid file for the duration of this call. + // * `self.table` is null or references a valid poll_table for the duration of this call. + // * Since `PollCondVar` is pinned, its destructor is guaranteed to run before the memory + // containing `cv.wait_queue_head` is invalidated. Since the destructor clears all + // waiters and then waits for an rcu grace period, it's guaranteed that + // `cv.wait_queue_head` remains valid for at least an rcu grace period after the removal + // of the last waiter. + unsafe { bindings::poll_wait(file.as_ptr(), cv.wait_queue_head.get(), self.table) } } } @@ -107,6 +91,7 @@ impl Deref for PollCondVar { #[pinned_drop] impl PinnedDrop for PollCondVar { + #[inline] fn drop(self: Pin<&mut Self>) { // Clear anything registered using `register_wait`. // diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 927413d85484..7d0935bc325c 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -173,6 +173,7 @@ impl Task { /// Callers must ensure that the returned object is only used to access a [`CurrentTask`] /// within the task context that was active when this function was called. For more details, /// see the invariants section for [`CurrentTask`]. + #[inline] pub unsafe fn current() -> impl Deref<Target = CurrentTask> { struct TaskRef { task: *const CurrentTask, @@ -222,24 +223,28 @@ impl Task { } /// Returns the UID of the given task. + #[inline] pub fn uid(&self) -> Kuid { // SAFETY: It's always safe to call `task_uid` on a valid task. Kuid::from_raw(unsafe { bindings::task_uid(self.as_ptr()) }) } /// Returns the effective UID of the given task. + #[inline] pub fn euid(&self) -> Kuid { // SAFETY: It's always safe to call `task_euid` on a valid task. Kuid::from_raw(unsafe { bindings::task_euid(self.as_ptr()) }) } /// Determines whether the given task has pending signals. + #[inline] pub fn signal_pending(&self) -> bool { // SAFETY: It's always safe to call `signal_pending` on a valid task. unsafe { bindings::signal_pending(self.as_ptr()) != 0 } } /// Returns task's pid namespace with elevated reference count + #[inline] pub fn get_pid_ns(&self) -> Option<ARef<PidNamespace>> { // SAFETY: By the type invariant, we know that `self.0` is valid. let ptr = unsafe { bindings::task_get_pid_ns(self.as_ptr()) }; @@ -255,6 +260,7 @@ impl Task { /// Returns the given task's pid in the provided pid namespace. #[doc(alias = "task_tgid_nr_ns")] + #[inline] pub fn tgid_nr_ns(&self, pidns: Option<&PidNamespace>) -> Pid { let pidns = match pidns { Some(pidns) => pidns.as_ptr(), @@ -268,6 +274,7 @@ impl Task { } /// Wakes up the task. + #[inline] pub fn wake_up(&self) { // SAFETY: It's always safe to call `wake_up_process` on a valid task, even if the task // running. @@ -341,11 +348,13 @@ impl CurrentTask { // SAFETY: The type invariants guarantee that `Task` is always refcounted. unsafe impl crate::types::AlwaysRefCounted for Task { + #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference means that the refcount is nonzero. unsafe { bindings::get_task_struct(self.as_ptr()) }; } + #[inline] unsafe fn dec_ref(obj: ptr::NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is nonzero. unsafe { bindings::put_task_struct(obj.cast().as_ptr()) } @@ -391,3 +400,27 @@ impl PartialEq for Kuid { } impl Eq for Kuid {} + +/// Annotation for functions that can sleep. +/// +/// Equivalent to the C side [`might_sleep()`], this function serves as +/// a debugging aid and a potential scheduling point. +/// +/// This function can only be used in a nonatomic context. +/// +/// [`might_sleep()`]: https://docs.kernel.org/driver-api/basics.html#c.might_sleep +#[track_caller] +#[inline] +pub fn might_sleep() { + #[cfg(CONFIG_DEBUG_ATOMIC_SLEEP)] + { + let loc = core::panic::Location::caller(); + let file = kernel::file_from_location(loc); + + // SAFETY: `file.as_ptr()` is valid for reading and guaranteed to be nul-terminated. + unsafe { crate::bindings::__might_sleep(file.as_ptr().cast(), loc.line() as i32) } + } + + // SAFETY: Always safe to call. + unsafe { crate::bindings::might_resched() } +} diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index a8089a98da9e..64c8dcf548d6 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -24,6 +24,9 @@ //! C header: [`include/linux/jiffies.h`](srctree/include/linux/jiffies.h). //! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h). +use core::marker::PhantomData; + +pub mod delay; pub mod hrtimer; /// The number of nanoseconds per microsecond. @@ -49,26 +52,141 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { unsafe { bindings::__msecs_to_jiffies(msecs) } } +/// Trait for clock sources. +/// +/// Selection of the clock source depends on the use case. In some cases the usage of a +/// particular clock is mandatory, e.g. in network protocols, filesystems. In other +/// cases the user of the clock has to decide which clock is best suited for the +/// purpose. In most scenarios clock [`Monotonic`] is the best choice as it +/// provides a accurate monotonic notion of time (leap second smearing ignored). +pub trait ClockSource { + /// The kernel clock ID associated with this clock source. + /// + /// This constant corresponds to the C side `clockid_t` value. + const ID: bindings::clockid_t; + + /// Get the current time from the clock source. + /// + /// The function must return a value in the range from 0 to `KTIME_MAX`. + fn ktime_get() -> bindings::ktime_t; +} + +/// A monotonically increasing clock. +/// +/// A nonsettable system-wide clock that represents monotonic time since as +/// described by POSIX, "some unspecified point in the past". On Linux, that +/// point corresponds to the number of seconds that the system has been +/// running since it was booted. +/// +/// The CLOCK_MONOTONIC clock is not affected by discontinuous jumps in the +/// CLOCK_REAL (e.g., if the system administrator manually changes the +/// clock), but is affected by frequency adjustments. This clock does not +/// count time that the system is suspended. +pub struct Monotonic; + +impl ClockSource for Monotonic { + const ID: bindings::clockid_t = bindings::CLOCK_MONOTONIC as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get()` outside of NMI context. + unsafe { bindings::ktime_get() } + } +} + +/// A settable system-wide clock that measures real (i.e., wall-clock) time. +/// +/// Setting this clock requires appropriate privileges. This clock is +/// affected by discontinuous jumps in the system time (e.g., if the system +/// administrator manually changes the clock), and by frequency adjustments +/// performed by NTP and similar applications via adjtime(3), adjtimex(2), +/// clock_adjtime(2), and ntp_adjtime(3). This clock normally counts the +/// number of seconds since 1970-01-01 00:00:00 Coordinated Universal Time +/// (UTC) except that it ignores leap seconds; near a leap second it may be +/// adjusted by leap second smearing to stay roughly in sync with UTC. Leap +/// second smearing applies frequency adjustments to the clock to speed up +/// or slow down the clock to account for the leap second without +/// discontinuities in the clock. If leap second smearing is not applied, +/// the clock will experience discontinuity around leap second adjustment. +pub struct RealTime; + +impl ClockSource for RealTime { + const ID: bindings::clockid_t = bindings::CLOCK_REALTIME as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get_real()` outside of NMI context. + unsafe { bindings::ktime_get_real() } + } +} + +/// A monotonic that ticks while system is suspended. +/// +/// A nonsettable system-wide clock that is identical to CLOCK_MONOTONIC, +/// except that it also includes any time that the system is suspended. This +/// allows applications to get a suspend-aware monotonic clock without +/// having to deal with the complications of CLOCK_REALTIME, which may have +/// discontinuities if the time is changed using settimeofday(2) or similar. +pub struct BootTime; + +impl ClockSource for BootTime { + const ID: bindings::clockid_t = bindings::CLOCK_BOOTTIME as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get_boottime()` outside of NMI context. + unsafe { bindings::ktime_get_boottime() } + } +} + +/// International Atomic Time. +/// +/// A system-wide clock derived from wall-clock time but counting leap seconds. +/// +/// This clock is coupled to CLOCK_REALTIME and will be set when CLOCK_REALTIME is +/// set, or when the offset to CLOCK_REALTIME is changed via adjtimex(2). This +/// usually happens during boot and **should** not happen during normal operations. +/// However, if NTP or another application adjusts CLOCK_REALTIME by leap second +/// smearing, this clock will not be precise during leap second smearing. +/// +/// The acronym TAI refers to International Atomic Time. +pub struct Tai; + +impl ClockSource for Tai { + const ID: bindings::clockid_t = bindings::CLOCK_TAI as bindings::clockid_t; + + fn ktime_get() -> bindings::ktime_t { + // SAFETY: It is always safe to call `ktime_get_tai()` outside of NMI context. + unsafe { bindings::ktime_get_clocktai() } + } +} + /// A specific point in time. /// /// # Invariants /// /// The `inner` value is in the range from 0 to `KTIME_MAX`. #[repr(transparent)] -#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord)] -pub struct Instant { +#[derive(PartialEq, PartialOrd, Eq, Ord)] +pub struct Instant<C: ClockSource> { inner: bindings::ktime_t, + _c: PhantomData<C>, +} + +impl<C: ClockSource> Clone for Instant<C> { + fn clone(&self) -> Self { + *self + } } -impl Instant { - /// Get the current time using `CLOCK_MONOTONIC`. +impl<C: ClockSource> Copy for Instant<C> {} + +impl<C: ClockSource> Instant<C> { + /// Get the current time from the clock source. #[inline] pub fn now() -> Self { - // INVARIANT: The `ktime_get()` function returns a value in the range + // INVARIANT: The `ClockSource::ktime_get()` function returns a value in the range // from 0 to `KTIME_MAX`. Self { - // SAFETY: It is always safe to call `ktime_get()` outside of NMI context. - inner: unsafe { bindings::ktime_get() }, + inner: C::ktime_get(), + _c: PhantomData, } } @@ -77,86 +195,25 @@ impl Instant { pub fn elapsed(&self) -> Delta { Self::now() - *self } + + #[inline] + pub(crate) fn as_nanos(&self) -> i64 { + self.inner + } } -impl core::ops::Sub for Instant { +impl<C: ClockSource> core::ops::Sub for Instant<C> { type Output = Delta; // By the type invariant, it never overflows. #[inline] - fn sub(self, other: Instant) -> Delta { + fn sub(self, other: Instant<C>) -> Delta { Delta { nanos: self.inner - other.inner, } } } -/// An identifier for a clock. Used when specifying clock sources. -/// -/// -/// Selection of the clock depends on the use case. In some cases the usage of a -/// particular clock is mandatory, e.g. in network protocols, filesystems.In other -/// cases the user of the clock has to decide which clock is best suited for the -/// purpose. In most scenarios clock [`ClockId::Monotonic`] is the best choice as it -/// provides a accurate monotonic notion of time (leap second smearing ignored). -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -#[repr(u32)] -pub enum ClockId { - /// A settable system-wide clock that measures real (i.e., wall-clock) time. - /// - /// Setting this clock requires appropriate privileges. This clock is - /// affected by discontinuous jumps in the system time (e.g., if the system - /// administrator manually changes the clock), and by frequency adjustments - /// performed by NTP and similar applications via adjtime(3), adjtimex(2), - /// clock_adjtime(2), and ntp_adjtime(3). This clock normally counts the - /// number of seconds since 1970-01-01 00:00:00 Coordinated Universal Time - /// (UTC) except that it ignores leap seconds; near a leap second it may be - /// adjusted by leap second smearing to stay roughly in sync with UTC. Leap - /// second smearing applies frequency adjustments to the clock to speed up - /// or slow down the clock to account for the leap second without - /// discontinuities in the clock. If leap second smearing is not applied, - /// the clock will experience discontinuity around leap second adjustment. - RealTime = bindings::CLOCK_REALTIME, - /// A monotonically increasing clock. - /// - /// A nonsettable system-wide clock that represents monotonic time since—as - /// described by POSIX—"some unspecified point in the past". On Linux, that - /// point corresponds to the number of seconds that the system has been - /// running since it was booted. - /// - /// The CLOCK_MONOTONIC clock is not affected by discontinuous jumps in the - /// CLOCK_REAL (e.g., if the system administrator manually changes the - /// clock), but is affected by frequency adjustments. This clock does not - /// count time that the system is suspended. - Monotonic = bindings::CLOCK_MONOTONIC, - /// A monotonic that ticks while system is suspended. - /// - /// A nonsettable system-wide clock that is identical to CLOCK_MONOTONIC, - /// except that it also includes any time that the system is suspended. This - /// allows applications to get a suspend-aware monotonic clock without - /// having to deal with the complications of CLOCK_REALTIME, which may have - /// discontinuities if the time is changed using settimeofday(2) or similar. - BootTime = bindings::CLOCK_BOOTTIME, - /// International Atomic Time. - /// - /// A system-wide clock derived from wall-clock time but counting leap seconds. - /// - /// This clock is coupled to CLOCK_REALTIME and will be set when CLOCK_REALTIME is - /// set, or when the offset to CLOCK_REALTIME is changed via adjtimex(2). This - /// usually happens during boot and **should** not happen during normal operations. - /// However, if NTP or another application adjusts CLOCK_REALTIME by leap second - /// smearing, this clock will not be precise during leap second smearing. - /// - /// The acronym TAI refers to International Atomic Time. - TAI = bindings::CLOCK_TAI, -} - -impl ClockId { - fn into_c(self) -> bindings::clockid_t { - self as bindings::clockid_t - } -} - /// A span of time. /// /// This struct represents a span of time, with its value stored as nanoseconds. @@ -228,13 +285,31 @@ impl Delta { /// Return the smallest number of microseconds greater than or equal /// to the value in the [`Delta`]. #[inline] - pub const fn as_micros_ceil(self) -> i64 { - self.as_nanos().saturating_add(NSEC_PER_USEC - 1) / NSEC_PER_USEC + pub fn as_micros_ceil(self) -> i64 { + #[cfg(CONFIG_64BIT)] + { + self.as_nanos().saturating_add(NSEC_PER_USEC - 1) / NSEC_PER_USEC + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_us()` with any value. + unsafe { + bindings::ktime_to_us(self.as_nanos().saturating_add(NSEC_PER_USEC - 1)) + } } /// Return the number of milliseconds in the [`Delta`]. #[inline] - pub const fn as_millis(self) -> i64 { - self.as_nanos() / NSEC_PER_MSEC + pub fn as_millis(self) -> i64 { + #[cfg(CONFIG_64BIT)] + { + self.as_nanos() / NSEC_PER_MSEC + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_ms()` with any value. + unsafe { + bindings::ktime_to_ms(self.as_nanos()) + } } } diff --git a/rust/kernel/time/delay.rs b/rust/kernel/time/delay.rs new file mode 100644 index 000000000000..eb8838da62bc --- /dev/null +++ b/rust/kernel/time/delay.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Delay and sleep primitives. +//! +//! This module contains the kernel APIs related to delay and sleep that +//! have been ported or wrapped for usage by Rust code in the kernel. +//! +//! C header: [`include/linux/delay.h`](srctree/include/linux/delay.h). + +use super::Delta; +use crate::prelude::*; + +/// Sleeps for a given duration at least. +/// +/// Equivalent to the C side [`fsleep()`], flexible sleep function, +/// which automatically chooses the best sleep method based on a duration. +/// +/// `delta` must be within `[0, i32::MAX]` microseconds; +/// otherwise, it is erroneous behavior. That is, it is considered a bug +/// to call this function with an out-of-range value, in which case the function +/// will sleep for at least the maximum value in the range and may warn +/// in the future. +/// +/// The behavior above differs from the C side [`fsleep()`] for which out-of-range +/// values mean "infinite timeout" instead. +/// +/// This function can only be used in a nonatomic context. +/// +/// [`fsleep()`]: https://docs.kernel.org/timers/delay_sleep_functions.html#c.fsleep +pub fn fsleep(delta: Delta) { + // The maximum value is set to `i32::MAX` microseconds to prevent integer + // overflow inside fsleep, which could lead to unintentional infinite sleep. + const MAX_DELTA: Delta = Delta::from_micros(i32::MAX as i64); + + let delta = if (Delta::ZERO..=MAX_DELTA).contains(&delta) { + delta + } else { + // TODO: Add WARN_ONCE() when it's supported. + MAX_DELTA + }; + + // SAFETY: It is always safe to call `fsleep()` with any duration. + unsafe { + // Convert the duration to microseconds and round up to preserve + // the guarantee; `fsleep()` sleeps for at least the provided duration, + // but that it may sleep for longer under some circumstances. + bindings::fsleep(delta.as_micros_ceil() as c_ulong) + } +} diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs index 9df3dcd2fa39..144e3b57cc78 100644 --- a/rust/kernel/time/hrtimer.rs +++ b/rust/kernel/time/hrtimer.rs @@ -67,27 +67,11 @@ //! A `restart` operation on a timer in the **stopped** state is equivalent to a //! `start` operation. -use super::ClockId; +use super::{ClockSource, Delta, Instant}; use crate::{prelude::*, types::Opaque}; use core::marker::PhantomData; use pin_init::PinInit; -/// A Rust wrapper around a `ktime_t`. -// NOTE: Ktime is going to be removed when hrtimer is converted to Instant/Delta. -#[repr(transparent)] -#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord)] -pub struct Ktime { - inner: bindings::ktime_t, -} - -impl Ktime { - /// Returns the number of nanoseconds. - #[inline] - pub fn to_ns(self) -> i64 { - self.inner - } -} - /// A timer backed by a C `struct hrtimer`. /// /// # Invariants @@ -98,7 +82,6 @@ impl Ktime { pub struct HrTimer<T> { #[pin] timer: Opaque<bindings::hrtimer>, - mode: HrTimerMode, _t: PhantomData<T>, } @@ -112,9 +95,10 @@ unsafe impl<T> Sync for HrTimer<T> {} impl<T> HrTimer<T> { /// Return an initializer for a new timer instance. - pub fn new(mode: HrTimerMode, clock: ClockId) -> impl PinInit<Self> + pub fn new() -> impl PinInit<Self> where T: HrTimerCallback, + T: HasHrTimer<T>, { pin_init!(Self { // INVARIANT: We initialize `timer` with `hrtimer_setup` below. @@ -126,12 +110,11 @@ impl<T> HrTimer<T> { bindings::hrtimer_setup( place, Some(T::Pointer::run), - clock.into_c(), - mode.into_c(), + <<T as HasHrTimer<T>>::TimerMode as HrTimerMode>::Clock::ID, + <T as HasHrTimer<T>>::TimerMode::C_MODE, ); } }), - mode: mode, _t: PhantomData, }) } @@ -148,7 +131,7 @@ impl<T> HrTimer<T> { // SAFETY: The field projection to `timer` does not go out of bounds, // because the caller of this function promises that `this` points to an // allocation of at least the size of `Self`. - unsafe { Opaque::raw_get(core::ptr::addr_of!((*this).timer)) } + unsafe { Opaque::cast_into(core::ptr::addr_of!((*this).timer)) } } /// Cancel an initialized and potentially running timer. @@ -193,6 +176,11 @@ impl<T> HrTimer<T> { /// exist. A timer can be manipulated through any of the handles, and a handle /// may represent a cancelled timer. pub trait HrTimerPointer: Sync + Sized { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + /// A handle representing a started or restarted timer. /// /// If the timer is running or if the timer callback is executing when the @@ -205,7 +193,7 @@ pub trait HrTimerPointer: Sync + Sized { /// Start the timer with expiry after `expires` time units. If the timer was /// already running, it is restarted with the new expiry time. - fn start(self, expires: Ktime) -> Self::TimerHandle; + fn start(self, expires: <Self::TimerMode as HrTimerMode>::Expires) -> Self::TimerHandle; } /// Unsafe version of [`HrTimerPointer`] for situations where leaking the @@ -220,6 +208,11 @@ pub trait HrTimerPointer: Sync + Sized { /// [`UnsafeHrTimerPointer`] outlives any associated [`HrTimerPointer::TimerHandle`] /// instances. pub unsafe trait UnsafeHrTimerPointer: Sync + Sized { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + /// A handle representing a running timer. /// /// # Safety @@ -236,7 +229,7 @@ pub unsafe trait UnsafeHrTimerPointer: Sync + Sized { /// /// Caller promises keep the timer structure alive until the timer is dead. /// Caller can ensure this by not leaking the returned [`Self::TimerHandle`]. - unsafe fn start(self, expires: Ktime) -> Self::TimerHandle; + unsafe fn start(self, expires: <Self::TimerMode as HrTimerMode>::Expires) -> Self::TimerHandle; } /// A trait for stack allocated timers. @@ -246,9 +239,14 @@ pub unsafe trait UnsafeHrTimerPointer: Sync + Sized { /// Implementers must ensure that `start_scoped` does not return until the /// timer is dead and the timer handler is not running. pub unsafe trait ScopedHrTimerPointer { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + /// Start the timer to run after `expires` time units and immediately /// after call `f`. When `f` returns, the timer is cancelled. - fn start_scoped<T, F>(self, expires: Ktime, f: F) -> T + fn start_scoped<T, F>(self, expires: <Self::TimerMode as HrTimerMode>::Expires, f: F) -> T where F: FnOnce() -> T; } @@ -260,7 +258,13 @@ unsafe impl<T> ScopedHrTimerPointer for T where T: UnsafeHrTimerPointer, { - fn start_scoped<U, F>(self, expires: Ktime, f: F) -> U + type TimerMode = T::TimerMode; + + fn start_scoped<U, F>( + self, + expires: <<T as UnsafeHrTimerPointer>::TimerMode as HrTimerMode>::Expires, + f: F, + ) -> U where F: FnOnce() -> U, { @@ -335,6 +339,11 @@ pub unsafe trait HrTimerHandle { /// their documentation. All the methods of this trait must operate on the same /// field. pub unsafe trait HasHrTimer<T> { + /// The operational mode associated with this timer. + /// + /// This defines how the expiration value is interpreted. + type TimerMode: HrTimerMode; + /// Return a pointer to the [`HrTimer`] within `Self`. /// /// This function is useful to get access to the value without creating @@ -382,14 +391,14 @@ pub unsafe trait HasHrTimer<T> { /// - `this` must point to a valid `Self`. /// - Caller must ensure that the pointee of `this` lives until the timer /// fires or is canceled. - unsafe fn start(this: *const Self, expires: Ktime) { + unsafe fn start(this: *const Self, expires: <Self::TimerMode as HrTimerMode>::Expires) { // SAFETY: By function safety requirement, `this` is a valid `Self`. unsafe { bindings::hrtimer_start_range_ns( Self::c_timer_ptr(this).cast_mut(), - expires.to_ns(), + expires.as_nanos(), 0, - (*Self::raw_get_timer(this)).mode.into_c(), + <Self::TimerMode as HrTimerMode>::C_MODE, ); } } @@ -411,80 +420,171 @@ impl HrTimerRestart { } } -/// Operational mode of [`HrTimer`]. -// NOTE: Some of these have the same encoding on the C side, so we keep -// `repr(Rust)` and convert elsewhere. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum HrTimerMode { - /// Timer expires at the given expiration time. - Absolute, - /// Timer expires after the given expiration time interpreted as a duration from now. - Relative, - /// Timer does not move between CPU cores. - Pinned, - /// Timer handler is executed in soft irq context. - Soft, - /// Timer handler is executed in hard irq context. - Hard, - /// Timer expires at the given expiration time. - /// Timer does not move between CPU cores. - AbsolutePinned, - /// Timer expires after the given expiration time interpreted as a duration from now. - /// Timer does not move between CPU cores. - RelativePinned, - /// Timer expires at the given expiration time. - /// Timer handler is executed in soft irq context. - AbsoluteSoft, - /// Timer expires after the given expiration time interpreted as a duration from now. - /// Timer handler is executed in soft irq context. - RelativeSoft, - /// Timer expires at the given expiration time. - /// Timer does not move between CPU cores. - /// Timer handler is executed in soft irq context. - AbsolutePinnedSoft, - /// Timer expires after the given expiration time interpreted as a duration from now. - /// Timer does not move between CPU cores. - /// Timer handler is executed in soft irq context. - RelativePinnedSoft, - /// Timer expires at the given expiration time. - /// Timer handler is executed in hard irq context. - AbsoluteHard, - /// Timer expires after the given expiration time interpreted as a duration from now. - /// Timer handler is executed in hard irq context. - RelativeHard, - /// Timer expires at the given expiration time. - /// Timer does not move between CPU cores. - /// Timer handler is executed in hard irq context. - AbsolutePinnedHard, - /// Timer expires after the given expiration time interpreted as a duration from now. - /// Timer does not move between CPU cores. - /// Timer handler is executed in hard irq context. - RelativePinnedHard, +/// Time representations that can be used as expiration values in [`HrTimer`]. +pub trait HrTimerExpires { + /// Converts the expiration time into a nanosecond representation. + /// + /// This value corresponds to a raw ktime_t value, suitable for passing to kernel + /// timer functions. The interpretation (absolute vs relative) depends on the + /// associated [HrTimerMode] in use. + fn as_nanos(&self) -> i64; } -impl HrTimerMode { - fn into_c(self) -> bindings::hrtimer_mode { - use bindings::*; - match self { - HrTimerMode::Absolute => hrtimer_mode_HRTIMER_MODE_ABS, - HrTimerMode::Relative => hrtimer_mode_HRTIMER_MODE_REL, - HrTimerMode::Pinned => hrtimer_mode_HRTIMER_MODE_PINNED, - HrTimerMode::Soft => hrtimer_mode_HRTIMER_MODE_SOFT, - HrTimerMode::Hard => hrtimer_mode_HRTIMER_MODE_HARD, - HrTimerMode::AbsolutePinned => hrtimer_mode_HRTIMER_MODE_ABS_PINNED, - HrTimerMode::RelativePinned => hrtimer_mode_HRTIMER_MODE_REL_PINNED, - HrTimerMode::AbsoluteSoft => hrtimer_mode_HRTIMER_MODE_ABS_SOFT, - HrTimerMode::RelativeSoft => hrtimer_mode_HRTIMER_MODE_REL_SOFT, - HrTimerMode::AbsolutePinnedSoft => hrtimer_mode_HRTIMER_MODE_ABS_PINNED_SOFT, - HrTimerMode::RelativePinnedSoft => hrtimer_mode_HRTIMER_MODE_REL_PINNED_SOFT, - HrTimerMode::AbsoluteHard => hrtimer_mode_HRTIMER_MODE_ABS_HARD, - HrTimerMode::RelativeHard => hrtimer_mode_HRTIMER_MODE_REL_HARD, - HrTimerMode::AbsolutePinnedHard => hrtimer_mode_HRTIMER_MODE_ABS_PINNED_HARD, - HrTimerMode::RelativePinnedHard => hrtimer_mode_HRTIMER_MODE_REL_PINNED_HARD, - } +impl<C: ClockSource> HrTimerExpires for Instant<C> { + #[inline] + fn as_nanos(&self) -> i64 { + Instant::<C>::as_nanos(self) + } +} + +impl HrTimerExpires for Delta { + #[inline] + fn as_nanos(&self) -> i64 { + Delta::as_nanos(*self) } } +mod private { + use crate::time::ClockSource; + + pub trait Sealed {} + + impl<C: ClockSource> Sealed for super::AbsoluteMode<C> {} + impl<C: ClockSource> Sealed for super::RelativeMode<C> {} + impl<C: ClockSource> Sealed for super::AbsolutePinnedMode<C> {} + impl<C: ClockSource> Sealed for super::RelativePinnedMode<C> {} + impl<C: ClockSource> Sealed for super::AbsoluteSoftMode<C> {} + impl<C: ClockSource> Sealed for super::RelativeSoftMode<C> {} + impl<C: ClockSource> Sealed for super::AbsolutePinnedSoftMode<C> {} + impl<C: ClockSource> Sealed for super::RelativePinnedSoftMode<C> {} + impl<C: ClockSource> Sealed for super::AbsoluteHardMode<C> {} + impl<C: ClockSource> Sealed for super::RelativeHardMode<C> {} + impl<C: ClockSource> Sealed for super::AbsolutePinnedHardMode<C> {} + impl<C: ClockSource> Sealed for super::RelativePinnedHardMode<C> {} +} + +/// Operational mode of [`HrTimer`]. +pub trait HrTimerMode: private::Sealed { + /// The C representation of hrtimer mode. + const C_MODE: bindings::hrtimer_mode; + + /// Type representing the clock source. + type Clock: ClockSource; + + /// Type representing the expiration specification (absolute or relative time). + type Expires: HrTimerExpires; +} + +/// Timer that expires at a fixed point in time. +pub struct AbsoluteMode<C: ClockSource>(PhantomData<C>); + +impl<C: ClockSource> HrTimerMode for AbsoluteMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS; + + type Clock = C; + type Expires = Instant<C>; +} + +/// Timer that expires after a delay from now. +pub struct RelativeMode<C: ClockSource>(PhantomData<C>); + +impl<C: ClockSource> HrTimerMode for RelativeMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration time, pinned to its current CPU. +pub struct AbsolutePinnedMode<C: ClockSource>(PhantomData<C>); +impl<C: ClockSource> HrTimerMode for AbsolutePinnedMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED; + + type Clock = C; + type Expires = Instant<C>; +} + +/// Timer with relative expiration time, pinned to its current CPU. +pub struct RelativePinnedMode<C: ClockSource>(PhantomData<C>); +impl<C: ClockSource> HrTimerMode for RelativePinnedMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, handled in soft irq context. +pub struct AbsoluteSoftMode<C: ClockSource>(PhantomData<C>); +impl<C: ClockSource> HrTimerMode for AbsoluteSoftMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_SOFT; + + type Clock = C; + type Expires = Instant<C>; +} + +/// Timer with relative expiration, handled in soft irq context. +pub struct RelativeSoftMode<C: ClockSource>(PhantomData<C>); +impl<C: ClockSource> HrTimerMode for RelativeSoftMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_SOFT; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, pinned to CPU and handled in soft irq context. +pub struct AbsolutePinnedSoftMode<C: ClockSource>(PhantomData<C>); +impl<C: ClockSource> HrTimerMode for AbsolutePinnedSoftMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_SOFT; + + type Clock = C; + type Expires = Instant<C>; +} + +/// Timer with absolute expiration, pinned to CPU and handled in soft irq context. +pub struct RelativePinnedSoftMode<C: ClockSource>(PhantomData<C>); +impl<C: ClockSource> HrTimerMode for RelativePinnedSoftMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_SOFT; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, handled in hard irq context. +pub struct AbsoluteHardMode<C: ClockSource>(PhantomData<C>); +impl<C: ClockSource> HrTimerMode for AbsoluteHardMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_HARD; + + type Clock = C; + type Expires = Instant<C>; +} + +/// Timer with relative expiration, handled in hard irq context. +pub struct RelativeHardMode<C: ClockSource>(PhantomData<C>); +impl<C: ClockSource> HrTimerMode for RelativeHardMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_HARD; + + type Clock = C; + type Expires = Delta; +} + +/// Timer with absolute expiration, pinned to CPU and handled in hard irq context. +pub struct AbsolutePinnedHardMode<C: ClockSource>(PhantomData<C>); +impl<C: ClockSource> HrTimerMode for AbsolutePinnedHardMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_HARD; + + type Clock = C; + type Expires = Instant<C>; +} + +/// Timer with relative expiration, pinned to CPU and handled in hard irq context. +pub struct RelativePinnedHardMode<C: ClockSource>(PhantomData<C>); +impl<C: ClockSource> HrTimerMode for RelativePinnedHardMode<C> { + const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_HARD; + + type Clock = C; + type Expires = Delta; +} + /// Use to implement the [`HasHrTimer<T>`] trait. /// /// See [`module`] documentation for an example. @@ -496,12 +596,16 @@ macro_rules! impl_has_hr_timer { impl$({$($generics:tt)*})? HasHrTimer<$timer_type:ty> for $self:ty - { self.$field:ident } + { + mode : $mode:ty, + field : self.$field:ident $(,)? + } $($rest:tt)* ) => { // SAFETY: This implementation of `raw_get_timer` only compiles if the // field has the right type. unsafe impl$(<$($generics)*>)? $crate::time::hrtimer::HasHrTimer<$timer_type> for $self { + type TimerMode = $mode; #[inline] unsafe fn raw_get_timer( @@ -517,7 +621,7 @@ macro_rules! impl_has_hr_timer { ) -> *mut Self { // SAFETY: As per the safety requirement of this function, `ptr` // is pointing inside a `$timer_type`. - unsafe { ::kernel::container_of!(ptr, $timer_type, $field).cast_mut() } + unsafe { ::kernel::container_of!(ptr, $timer_type, $field) } } } } diff --git a/rust/kernel/time/hrtimer/arc.rs b/rust/kernel/time/hrtimer/arc.rs index ccf1e66e5b2d..ed490a7a8950 100644 --- a/rust/kernel/time/hrtimer/arc.rs +++ b/rust/kernel/time/hrtimer/arc.rs @@ -4,8 +4,8 @@ use super::HasHrTimer; use super::HrTimer; use super::HrTimerCallback; use super::HrTimerHandle; +use super::HrTimerMode; use super::HrTimerPointer; -use super::Ktime; use super::RawHrTimerCallback; use crate::sync::Arc; use crate::sync::ArcBorrow; @@ -54,9 +54,13 @@ where T: HasHrTimer<T>, T: for<'a> HrTimerCallback<Pointer<'a> = Self>, { + type TimerMode = <T as HasHrTimer<T>>::TimerMode; type TimerHandle = ArcHrTimerHandle<T>; - fn start(self, expires: Ktime) -> ArcHrTimerHandle<T> { + fn start( + self, + expires: <<T as HasHrTimer<T>>::TimerMode as HrTimerMode>::Expires, + ) -> ArcHrTimerHandle<T> { // SAFETY: // - We keep `self` alive by wrapping it in a handle below. // - Since we generate the pointer passed to `start` from a valid diff --git a/rust/kernel/time/hrtimer/pin.rs b/rust/kernel/time/hrtimer/pin.rs index 293ca9cf058c..aef16d9ee2f0 100644 --- a/rust/kernel/time/hrtimer/pin.rs +++ b/rust/kernel/time/hrtimer/pin.rs @@ -4,7 +4,7 @@ use super::HasHrTimer; use super::HrTimer; use super::HrTimerCallback; use super::HrTimerHandle; -use super::Ktime; +use super::HrTimerMode; use super::RawHrTimerCallback; use super::UnsafeHrTimerPointer; use core::pin::Pin; @@ -54,9 +54,13 @@ where T: HasHrTimer<T>, T: HrTimerCallback<Pointer<'a> = Self>, { + type TimerMode = <T as HasHrTimer<T>>::TimerMode; type TimerHandle = PinHrTimerHandle<'a, T>; - unsafe fn start(self, expires: Ktime) -> Self::TimerHandle { + unsafe fn start( + self, + expires: <<T as HasHrTimer<T>>::TimerMode as HrTimerMode>::Expires, + ) -> Self::TimerHandle { // Cast to pointer let self_ptr: *const T = self.get_ref(); @@ -79,7 +83,7 @@ where unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { // `HrTimer` is `repr(C)` - let timer_ptr = ptr as *mut HrTimer<T>; + let timer_ptr = ptr.cast::<HrTimer<T>>(); // SAFETY: By the safety requirement of this function, `timer_ptr` // points to a `HrTimer<T>` contained in an `T`. diff --git a/rust/kernel/time/hrtimer/pin_mut.rs b/rust/kernel/time/hrtimer/pin_mut.rs index 6033572d35ad..767d0a4e8a2c 100644 --- a/rust/kernel/time/hrtimer/pin_mut.rs +++ b/rust/kernel/time/hrtimer/pin_mut.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 use super::{ - HasHrTimer, HrTimer, HrTimerCallback, HrTimerHandle, Ktime, RawHrTimerCallback, + HasHrTimer, HrTimer, HrTimerCallback, HrTimerHandle, HrTimerMode, RawHrTimerCallback, UnsafeHrTimerPointer, }; use core::{marker::PhantomData, pin::Pin, ptr::NonNull}; @@ -52,9 +52,13 @@ where T: HasHrTimer<T>, T: HrTimerCallback<Pointer<'a> = Self>, { + type TimerMode = <T as HasHrTimer<T>>::TimerMode; type TimerHandle = PinMutHrTimerHandle<'a, T>; - unsafe fn start(mut self, expires: Ktime) -> Self::TimerHandle { + unsafe fn start( + mut self, + expires: <<T as HasHrTimer<T>>::TimerMode as HrTimerMode>::Expires, + ) -> Self::TimerHandle { // SAFETY: // - We promise not to move out of `self`. We only pass `self` // back to the caller as a `Pin<&mut self>`. @@ -83,7 +87,7 @@ where unsafe extern "C" fn run(ptr: *mut bindings::hrtimer) -> bindings::hrtimer_restart { // `HrTimer` is `repr(C)` - let timer_ptr = ptr as *mut HrTimer<T>; + let timer_ptr = ptr.cast::<HrTimer<T>>(); // SAFETY: By the safety requirement of this function, `timer_ptr` // points to a `HrTimer<T>` contained in an `T`. diff --git a/rust/kernel/time/hrtimer/tbox.rs b/rust/kernel/time/hrtimer/tbox.rs index 29526a5da203..ec08303315f2 100644 --- a/rust/kernel/time/hrtimer/tbox.rs +++ b/rust/kernel/time/hrtimer/tbox.rs @@ -4,8 +4,8 @@ use super::HasHrTimer; use super::HrTimer; use super::HrTimerCallback; use super::HrTimerHandle; +use super::HrTimerMode; use super::HrTimerPointer; -use super::Ktime; use super::RawHrTimerCallback; use crate::prelude::*; use core::ptr::NonNull; @@ -64,9 +64,13 @@ where T: for<'a> HrTimerCallback<Pointer<'a> = Pin<Box<T, A>>>, A: crate::alloc::Allocator, { + type TimerMode = <T as HasHrTimer<T>>::TimerMode; type TimerHandle = BoxHrTimerHandle<T, A>; - fn start(self, expires: Ktime) -> Self::TimerHandle { + fn start( + self, + expires: <<T as HasHrTimer<T>>::TimerMode as HrTimerMode>::Expires, + ) -> Self::TimerHandle { // SAFETY: // - We will not move out of this box during timer callback (we pass an // immutable reference to the callback). diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 22985b6f6982..dc0a02f5c3cf 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -2,14 +2,16 @@ //! Kernel types. +use crate::ffi::c_void; use core::{ cell::UnsafeCell, marker::{PhantomData, PhantomPinned}, - mem::{ManuallyDrop, MaybeUninit}, + mem::MaybeUninit, ops::{Deref, DerefMut}, - ptr::NonNull, }; -use pin_init::{PinInit, Zeroable}; +use pin_init::{PinInit, Wrapper, Zeroable}; + +pub use crate::sync::aref::{ARef, AlwaysRefCounted}; /// Used to transfer ownership to and from foreign (non-Rust) languages. /// @@ -21,15 +23,10 @@ use pin_init::{PinInit, Zeroable}; /// /// # Safety /// -/// Implementers must ensure that [`into_foreign`] returns a pointer which meets the alignment -/// requirements of [`PointedTo`]. -/// -/// [`into_foreign`]: Self::into_foreign -/// [`PointedTo`]: Self::PointedTo +/// - Implementations must satisfy the guarantees of [`Self::into_foreign`]. pub unsafe trait ForeignOwnable: Sized { - /// Type used when the value is foreign-owned. In practical terms only defines the alignment of - /// the pointer. - type PointedTo; + /// The alignment of pointers returned by `into_foreign`. + const FOREIGN_ALIGN: usize; /// Type used to immutably borrow a value that is currently foreign-owned. type Borrowed<'a>; @@ -39,18 +36,21 @@ pub unsafe trait ForeignOwnable: Sized { /// Converts a Rust-owned object to a foreign-owned one. /// + /// The foreign representation is a pointer to void. Aside from the guarantees listed below, + /// there are no other guarantees for this pointer. For example, it might be invalid, dangling + /// or pointing to uninitialized memory. Using it in any way except for [`from_foreign`], + /// [`try_from_foreign`], [`borrow`], or [`borrow_mut`] can result in undefined behavior. + /// /// # Guarantees /// - /// The return value is guaranteed to be well-aligned, but there are no other guarantees for - /// this pointer. For example, it might be null, dangling, or point to uninitialized memory. - /// Using it in any way except for [`ForeignOwnable::from_foreign`], [`ForeignOwnable::borrow`], - /// [`ForeignOwnable::try_from_foreign`] can result in undefined behavior. + /// - Minimum alignment of returned pointer is [`Self::FOREIGN_ALIGN`]. + /// - The returned pointer is not null. /// /// [`from_foreign`]: Self::from_foreign /// [`try_from_foreign`]: Self::try_from_foreign /// [`borrow`]: Self::borrow /// [`borrow_mut`]: Self::borrow_mut - fn into_foreign(self) -> *mut Self::PointedTo; + fn into_foreign(self) -> *mut c_void; /// Converts a foreign-owned object back to a Rust-owned one. /// @@ -60,7 +60,7 @@ pub unsafe trait ForeignOwnable: Sized { /// must not be passed to `from_foreign` more than once. /// /// [`into_foreign`]: Self::into_foreign - unsafe fn from_foreign(ptr: *mut Self::PointedTo) -> Self; + unsafe fn from_foreign(ptr: *mut c_void) -> Self; /// Tries to convert a foreign-owned object back to a Rust-owned one. /// @@ -72,7 +72,7 @@ pub unsafe trait ForeignOwnable: Sized { /// `ptr` must either be null or satisfy the safety requirements for [`from_foreign`]. /// /// [`from_foreign`]: Self::from_foreign - unsafe fn try_from_foreign(ptr: *mut Self::PointedTo) -> Option<Self> { + unsafe fn try_from_foreign(ptr: *mut c_void) -> Option<Self> { if ptr.is_null() { None } else { @@ -95,7 +95,7 @@ pub unsafe trait ForeignOwnable: Sized { /// /// [`into_foreign`]: Self::into_foreign /// [`from_foreign`]: Self::from_foreign - unsafe fn borrow<'a>(ptr: *mut Self::PointedTo) -> Self::Borrowed<'a>; + unsafe fn borrow<'a>(ptr: *mut c_void) -> Self::Borrowed<'a>; /// Borrows a foreign-owned object mutably. /// @@ -123,23 +123,24 @@ pub unsafe trait ForeignOwnable: Sized { /// [`from_foreign`]: Self::from_foreign /// [`borrow`]: Self::borrow /// [`Arc`]: crate::sync::Arc - unsafe fn borrow_mut<'a>(ptr: *mut Self::PointedTo) -> Self::BorrowedMut<'a>; + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Self::BorrowedMut<'a>; } -// SAFETY: The `into_foreign` function returns a pointer that is dangling, but well-aligned. +// SAFETY: The pointer returned by `into_foreign` comes from a well aligned +// pointer to `()`. unsafe impl ForeignOwnable for () { - type PointedTo = (); + const FOREIGN_ALIGN: usize = core::mem::align_of::<()>(); type Borrowed<'a> = (); type BorrowedMut<'a> = (); - fn into_foreign(self) -> *mut Self::PointedTo { + fn into_foreign(self) -> *mut c_void { core::ptr::NonNull::dangling().as_ptr() } - unsafe fn from_foreign(_: *mut Self::PointedTo) -> Self {} + unsafe fn from_foreign(_: *mut c_void) -> Self {} - unsafe fn borrow<'a>(_: *mut Self::PointedTo) -> Self::Borrowed<'a> {} - unsafe fn borrow_mut<'a>(_: *mut Self::PointedTo) -> Self::BorrowedMut<'a> {} + unsafe fn borrow<'a>(_: *mut c_void) -> Self::Borrowed<'a> {} + unsafe fn borrow_mut<'a>(_: *mut c_void) -> Self::BorrowedMut<'a> {} } /// Runs a cleanup function/closure when dropped. @@ -353,17 +354,6 @@ impl<T> Opaque<T> { } } - /// Create an opaque pin-initializer from the given pin-initializer. - pub fn pin_init(slot: impl PinInit<T>) -> impl PinInit<Self> { - Self::ffi_init(|ptr: *mut T| { - // SAFETY: - // - `ptr` is a valid pointer to uninitialized memory, - // - `slot` is not accessed on error; the call is infallible, - // - `slot` is pinned in memory. - let _ = unsafe { PinInit::<T>::__pinned_init(slot, ptr) }; - }) - } - /// Creates a pin-initializer from the given initializer closure. /// /// The returned initializer calls the given closure with the pointer to the inner `T` of this @@ -377,7 +367,7 @@ impl<T> Opaque<T> { // initialize the `T`. unsafe { pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| { - init_func(Self::raw_get(slot)); + init_func(Self::cast_into(slot)); Ok(()) }) } @@ -397,7 +387,7 @@ impl<T> Opaque<T> { // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully // initialize the `T`. unsafe { - pin_init::pin_init_from_closure::<_, E>(move |slot| init_func(Self::raw_get(slot))) + pin_init::pin_init_from_closure::<_, E>(move |slot| init_func(Self::cast_into(slot))) } } @@ -410,178 +400,29 @@ impl<T> Opaque<T> { /// /// This function is useful to get access to the value without creating intermediate /// references. - pub const fn raw_get(this: *const Self) -> *mut T { + pub const fn cast_into(this: *const Self) -> *mut T { UnsafeCell::raw_get(this.cast::<UnsafeCell<MaybeUninit<T>>>()).cast::<T>() } -} - -/// Types that are _always_ reference counted. -/// -/// It allows such types to define their own custom ref increment and decrement functions. -/// Additionally, it allows users to convert from a shared reference `&T` to an owned reference -/// [`ARef<T>`]. -/// -/// This is usually implemented by wrappers to existing structures on the C side of the code. For -/// Rust code, the recommendation is to use [`Arc`](crate::sync::Arc) to create reference-counted -/// instances of a type. -/// -/// # Safety -/// -/// Implementers must ensure that increments to the reference count keep the object alive in memory -/// at least until matching decrements are performed. -/// -/// Implementers must also ensure that all instances are reference-counted. (Otherwise they -/// won't be able to honour the requirement that [`AlwaysRefCounted::inc_ref`] keep the object -/// alive.) -pub unsafe trait AlwaysRefCounted { - /// Increments the reference count on the object. - fn inc_ref(&self); - /// Decrements the reference count on the object. - /// - /// Frees the object when the count reaches zero. - /// - /// # Safety - /// - /// Callers must ensure that there was a previous matching increment to the reference count, - /// and that the object is no longer used after its reference count is decremented (as it may - /// result in the object being freed), unless the caller owns another increment on the refcount - /// (e.g., it calls [`AlwaysRefCounted::inc_ref`] twice, then calls - /// [`AlwaysRefCounted::dec_ref`] once). - unsafe fn dec_ref(obj: NonNull<Self>); -} - -/// An owned reference to an always-reference-counted object. -/// -/// The object's reference count is automatically decremented when an instance of [`ARef`] is -/// dropped. It is also automatically incremented when a new instance is created via -/// [`ARef::clone`]. -/// -/// # Invariants -/// -/// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`ARef`] instance. In -/// particular, the [`ARef`] instance owns an increment on the underlying object's reference count. -pub struct ARef<T: AlwaysRefCounted> { - ptr: NonNull<T>, - _p: PhantomData<T>, -} - -// SAFETY: It is safe to send `ARef<T>` to another thread when the underlying `T` is `Sync` because -// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs -// `T` to be `Send` because any thread that has an `ARef<T>` may ultimately access `T` using a -// mutable reference, for example, when the reference count reaches zero and `T` is dropped. -unsafe impl<T: AlwaysRefCounted + Sync + Send> Send for ARef<T> {} - -// SAFETY: It is safe to send `&ARef<T>` to another thread when the underlying `T` is `Sync` -// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, -// it needs `T` to be `Send` because any thread that has a `&ARef<T>` may clone it and get an -// `ARef<T>` on that thread, so the thread may ultimately access `T` using a mutable reference, for -// example, when the reference count reaches zero and `T` is dropped. -unsafe impl<T: AlwaysRefCounted + Sync + Send> Sync for ARef<T> {} - -impl<T: AlwaysRefCounted> ARef<T> { - /// Creates a new instance of [`ARef`]. - /// - /// It takes over an increment of the reference count on the underlying object. - /// - /// # Safety - /// - /// Callers must ensure that the reference count was incremented at least once, and that they - /// are properly relinquishing one increment. That is, if there is only one increment, callers - /// must not use the underlying object anymore -- it is only safe to do so via the newly - /// created [`ARef`]. - pub unsafe fn from_raw(ptr: NonNull<T>) -> Self { - // INVARIANT: The safety requirements guarantee that the new instance now owns the - // increment on the refcount. - Self { - ptr, - _p: PhantomData, - } - } - - /// Consumes the `ARef`, returning a raw pointer. - /// - /// This function does not change the refcount. After calling this function, the caller is - /// responsible for the refcount previously managed by the `ARef`. - /// - /// # Examples - /// - /// ``` - /// use core::ptr::NonNull; - /// use kernel::types::{ARef, AlwaysRefCounted}; - /// - /// struct Empty {} - /// - /// # // SAFETY: TODO. - /// unsafe impl AlwaysRefCounted for Empty { - /// fn inc_ref(&self) {} - /// unsafe fn dec_ref(_obj: NonNull<Self>) {} - /// } - /// - /// let mut data = Empty {}; - /// let ptr = NonNull::<Empty>::new(&mut data).unwrap(); - /// # // SAFETY: TODO. - /// let data_ref: ARef<Empty> = unsafe { ARef::from_raw(ptr) }; - /// let raw_ptr: NonNull<Empty> = ARef::into_raw(data_ref); - /// - /// assert_eq!(ptr, raw_ptr); - /// ``` - pub fn into_raw(me: Self) -> NonNull<T> { - ManuallyDrop::new(me).ptr + /// The opposite operation of [`Opaque::cast_into`]. + pub const fn cast_from(this: *const T) -> *const Self { + this.cast() } } -impl<T: AlwaysRefCounted> Clone for ARef<T> { - fn clone(&self) -> Self { - self.inc_ref(); - // SAFETY: We just incremented the refcount above. - unsafe { Self::from_raw(self.ptr) } - } -} - -impl<T: AlwaysRefCounted> Deref for ARef<T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - // SAFETY: The type invariants guarantee that the object is valid. - unsafe { self.ptr.as_ref() } - } -} - -impl<T: AlwaysRefCounted> From<&T> for ARef<T> { - fn from(b: &T) -> Self { - b.inc_ref(); - // SAFETY: We just incremented the refcount above. - unsafe { Self::from_raw(NonNull::from(b)) } - } -} - -impl<T: AlwaysRefCounted> Drop for ARef<T> { - fn drop(&mut self) { - // SAFETY: The type invariants guarantee that the `ARef` owns the reference we're about to - // decrement. - unsafe { T::dec_ref(self.ptr) }; +impl<T> Wrapper<T> for Opaque<T> { + /// Create an opaque pin-initializer from the given pin-initializer. + fn pin_init<E>(slot: impl PinInit<T, E>) -> impl PinInit<Self, E> { + Self::try_ffi_init(|ptr: *mut T| { + // SAFETY: + // - `ptr` is a valid pointer to uninitialized memory, + // - `slot` is not accessed on error, + // - `slot` is pinned in memory. + unsafe { PinInit::<T, E>::__pinned_init(slot, ptr) } + }) } } -/// A sum type that always holds either a value of type `L` or `R`. -/// -/// # Examples -/// -/// ``` -/// use kernel::types::Either; -/// -/// let left_value: Either<i32, &str> = Either::Left(7); -/// let right_value: Either<i32, &str> = Either::Right("right value"); -/// ``` -pub enum Either<L, R> { - /// Constructs an instance of [`Either`] containing a value of type `L`. - Left(L), - - /// Constructs an instance of [`Either`] containing a value of type `R`. - Right(R), -} - /// Zero-sized type to mark types not [`Send`]. /// /// Add this type as a field to your struct if your type should not be sent to a different task. diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index 6d70edd8086a..a8fb4764185a 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -8,14 +8,57 @@ use crate::{ alloc::{Allocator, Flags}, bindings, error::Result, - ffi::c_void, + ffi::{c_char, c_void}, prelude::*, transmute::{AsBytes, FromBytes}, }; use core::mem::{size_of, MaybeUninit}; -/// The type used for userspace addresses. -pub type UserPtr = usize; +/// A pointer into userspace. +/// +/// This is the Rust equivalent to C pointers tagged with `__user`. +#[repr(transparent)] +#[derive(Copy, Clone)] +pub struct UserPtr(*mut c_void); + +impl UserPtr { + /// Create a `UserPtr` from an integer representing the userspace address. + #[inline] + pub fn from_addr(addr: usize) -> Self { + Self(addr as *mut c_void) + } + + /// Create a `UserPtr` from a pointer representing the userspace address. + #[inline] + pub fn from_ptr(addr: *mut c_void) -> Self { + Self(addr) + } + + /// Cast this userspace pointer to a raw const void pointer. + /// + /// It is up to the caller to use the returned pointer correctly. + #[inline] + pub fn as_const_ptr(self) -> *const c_void { + self.0 + } + + /// Cast this userspace pointer to a raw mutable void pointer. + /// + /// It is up to the caller to use the returned pointer correctly. + #[inline] + pub fn as_mut_ptr(self) -> *mut c_void { + self.0 + } + + /// Increment this user pointer by `add` bytes. + /// + /// This addition is wrapping, so wrapping around the address space does not result in a panic + /// even if `CONFIG_RUST_OVERFLOW_CHECKS` is enabled. + #[inline] + pub fn wrapping_byte_add(self, add: usize) -> UserPtr { + UserPtr(self.0.wrapping_byte_add(add)) + } +} /// A pointer to an area in userspace memory, which can be either read-only or read-write. /// @@ -177,7 +220,7 @@ impl UserSliceReader { pub fn skip(&mut self, num_skip: usize) -> Result { // Update `self.length` first since that's the fallible part of this operation. self.length = self.length.checked_sub(num_skip).ok_or(EFAULT)?; - self.ptr = self.ptr.wrapping_add(num_skip); + self.ptr = self.ptr.wrapping_byte_add(num_skip); Ok(()) } @@ -224,11 +267,11 @@ impl UserSliceReader { } // SAFETY: `out_ptr` points into a mutable slice of length `len`, so we may write // that many bytes to it. - let res = unsafe { bindings::copy_from_user(out_ptr, self.ptr as *const c_void, len) }; + let res = unsafe { bindings::copy_from_user(out_ptr, self.ptr.as_const_ptr(), len) }; if res != 0 { return Err(EFAULT); } - self.ptr = self.ptr.wrapping_add(len); + self.ptr = self.ptr.wrapping_byte_add(len); self.length -= len; Ok(()) } @@ -240,7 +283,7 @@ impl UserSliceReader { pub fn read_slice(&mut self, out: &mut [u8]) -> Result { // SAFETY: The types are compatible and `read_raw` doesn't write uninitialized bytes to // `out`. - let out = unsafe { &mut *(out as *mut [u8] as *mut [MaybeUninit<u8>]) }; + let out = unsafe { &mut *(core::ptr::from_mut(out) as *mut [MaybeUninit<u8>]) }; self.read_raw(out) } @@ -262,14 +305,14 @@ impl UserSliceReader { let res = unsafe { bindings::_copy_from_user( out.as_mut_ptr().cast::<c_void>(), - self.ptr as *const c_void, + self.ptr.as_const_ptr(), len, ) }; if res != 0 { return Err(EFAULT); } - self.ptr = self.ptr.wrapping_add(len); + self.ptr = self.ptr.wrapping_byte_add(len); self.length -= len; // SAFETY: The read above has initialized all bytes in `out`, and since `T` implements // `FromBytes`, any bit-pattern is a valid value for this type. @@ -291,6 +334,65 @@ impl UserSliceReader { unsafe { buf.inc_len(len) }; Ok(()) } + + /// Read a NUL-terminated string from userspace and return it. + /// + /// The string is read into `buf` and a NUL-terminator is added if the end of `buf` is reached. + /// Since there must be space to add a NUL-terminator, the buffer must not be empty. The + /// returned `&CStr` points into `buf`. + /// + /// Fails with [`EFAULT`] if the read happens on a bad address (some data may have been + /// copied). + #[doc(alias = "strncpy_from_user")] + pub fn strcpy_into_buf<'buf>(self, buf: &'buf mut [u8]) -> Result<&'buf CStr> { + if buf.is_empty() { + return Err(EINVAL); + } + + // SAFETY: The types are compatible and `strncpy_from_user` doesn't write uninitialized + // bytes to `buf`. + let mut dst = unsafe { &mut *(core::ptr::from_mut(buf) as *mut [MaybeUninit<u8>]) }; + + // We never read more than `self.length` bytes. + if dst.len() > self.length { + dst = &mut dst[..self.length]; + } + + let mut len = raw_strncpy_from_user(dst, self.ptr)?; + if len < dst.len() { + // Add one to include the NUL-terminator. + len += 1; + } else if len < buf.len() { + // This implies that `len == dst.len() < buf.len()`. + // + // This means that we could not fill the entire buffer, but we had to stop reading + // because we hit the `self.length` limit of this `UserSliceReader`. Since we did not + // fill the buffer, we treat this case as if we tried to read past the `self.length` + // limit and received a page fault, which is consistent with other `UserSliceReader` + // methods that also return page faults when you exceed `self.length`. + return Err(EFAULT); + } else { + // This implies that `len == buf.len()`. + // + // This means that we filled the buffer exactly. In this case, we add a NUL-terminator + // and return it. Unlike the `len < dst.len()` branch, don't modify `len` because it + // already represents the length including the NUL-terminator. + // + // SAFETY: Due to the check at the beginning, the buffer is not empty. + unsafe { *buf.last_mut().unwrap_unchecked() = 0 }; + } + + // This method consumes `self`, so it can only be called once, thus we do not need to + // update `self.length`. This sidesteps concerns such as whether `self.length` should be + // incremented by `len` or `len-1` in the `len == buf.len()` case. + + // SAFETY: There are two cases: + // * If we hit the `len < dst.len()` case, then `raw_strncpy_from_user` guarantees that + // this slice contains exactly one NUL byte at the end of the string. + // * Otherwise, `raw_strncpy_from_user` guarantees that the string contained no NUL bytes, + // and we have since added a NUL byte at the end. + Ok(unsafe { CStr::from_bytes_with_nul_unchecked(&buf[..len]) }) + } } /// A writer for [`UserSlice`]. @@ -327,11 +429,11 @@ impl UserSliceWriter { } // SAFETY: `data_ptr` points into an immutable slice of length `len`, so we may read // that many bytes from it. - let res = unsafe { bindings::copy_to_user(self.ptr as *mut c_void, data_ptr, len) }; + let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), data_ptr, len) }; if res != 0 { return Err(EFAULT); } - self.ptr = self.ptr.wrapping_add(len); + self.ptr = self.ptr.wrapping_byte_add(len); self.length -= len; Ok(()) } @@ -354,16 +456,53 @@ impl UserSliceWriter { // is a compile-time constant. let res = unsafe { bindings::_copy_to_user( - self.ptr as *mut c_void, - (value as *const T).cast::<c_void>(), + self.ptr.as_mut_ptr(), + core::ptr::from_ref(value).cast::<c_void>(), len, ) }; if res != 0 { return Err(EFAULT); } - self.ptr = self.ptr.wrapping_add(len); + self.ptr = self.ptr.wrapping_byte_add(len); self.length -= len; Ok(()) } } + +/// Reads a nul-terminated string into `dst` and returns the length. +/// +/// This reads from userspace until a NUL byte is encountered, or until `dst.len()` bytes have been +/// read. Fails with [`EFAULT`] if a read happens on a bad address (some data may have been +/// copied). When the end of the buffer is encountered, no NUL byte is added, so the string is +/// *not* guaranteed to be NUL-terminated when `Ok(dst.len())` is returned. +/// +/// # Guarantees +/// +/// When this function returns `Ok(len)`, it is guaranteed that the first `len` bytes of `dst` are +/// initialized and non-zero. Furthermore, if `len < dst.len()`, then `dst[len]` is a NUL byte. +#[inline] +fn raw_strncpy_from_user(dst: &mut [MaybeUninit<u8>], src: UserPtr) -> Result<usize> { + // CAST: Slice lengths are guaranteed to be `<= isize::MAX`. + let len = dst.len() as isize; + + // SAFETY: `dst` is valid for writing `dst.len()` bytes. + let res = unsafe { + bindings::strncpy_from_user( + dst.as_mut_ptr().cast::<c_char>(), + src.as_const_ptr().cast::<c_char>(), + len, + ) + }; + + if res < 0 { + return Err(Error::from_errno(res as i32)); + } + + #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] + assert!(res <= len); + + // GUARANTEES: `strncpy_from_user` was successful, so `dst` has contents in accordance with the + // guarantees of this function. + Ok(res as usize) +} diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index d092112d843f..b9343d5bc00f 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -26,7 +26,7 @@ //! * The [`WorkItemPointer`] trait is implemented for the pointer type that points at a something //! that implements [`WorkItem`]. //! -//! ## Example +//! ## Examples //! //! This example defines a struct that holds an integer and can be scheduled on the workqueue. When //! the struct is executed, it will print the integer. Since there is only one `work_struct` field, @@ -131,10 +131,69 @@ //! # print_2_later(MyStruct::new(41, 42).unwrap()); //! ``` //! +//! This example shows how you can schedule delayed work items: +//! +//! ``` +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, impl_has_delayed_work, new_delayed_work, DelayedWork, WorkItem}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value: i32, +//! #[pin] +//! work: DelayedWork<MyStruct>, +//! } +//! +//! impl_has_delayed_work! { +//! impl HasDelayedWork<Self> for MyStruct { self.work } +//! } +//! +//! impl MyStruct { +//! fn new(value: i32) -> Result<Arc<Self>> { +//! Arc::pin_init( +//! pin_init!(MyStruct { +//! value, +//! work <- new_delayed_work!("MyStruct::work"), +//! }), +//! GFP_KERNEL, +//! ) +//! } +//! } +//! +//! impl WorkItem for MyStruct { +//! type Pointer = Arc<MyStruct>; +//! +//! fn run(this: Arc<MyStruct>) { +//! pr_info!("The value is: {}\n", this.value); +//! } +//! } +//! +//! /// This method will enqueue the struct for execution on the system workqueue, where its value +//! /// will be printed 12 jiffies later. +//! fn print_later(val: Arc<MyStruct>) { +//! let _ = workqueue::system().enqueue_delayed(val, 12); +//! } +//! +//! /// It is also possible to use the ordinary `enqueue` method together with `DelayedWork`. This +//! /// is equivalent to calling `enqueue_delayed` with a delay of zero. +//! fn print_now(val: Arc<MyStruct>) { +//! let _ = workqueue::system().enqueue(val); +//! } +//! # print_later(MyStruct::new(42).unwrap()); +//! # print_now(MyStruct::new(42).unwrap()); +//! ``` +//! //! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h) -use crate::alloc::{AllocError, Flags}; -use crate::{prelude::*, sync::Arc, sync::LockClassKey, types::Opaque}; +use crate::{ + alloc::{AllocError, Flags}, + container_of, + prelude::*, + sync::Arc, + sync::LockClassKey, + time::Jiffies, + types::Opaque, +}; use core::marker::PhantomData; /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. @@ -146,6 +205,33 @@ macro_rules! new_work { } pub use new_work; +/// Creates a [`DelayedWork`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_delayed_work { + () => { + $crate::workqueue::DelayedWork::new( + $crate::optional_name!(), + $crate::static_lock_class!(), + $crate::c_str!(::core::concat!( + ::core::file!(), + ":", + ::core::line!(), + "_timer" + )), + $crate::static_lock_class!(), + ) + }; + ($name:literal) => { + $crate::workqueue::DelayedWork::new( + $crate::c_str!($name), + $crate::static_lock_class!(), + $crate::c_str!(::core::concat!($name, "_timer")), + $crate::static_lock_class!(), + ) + }; +} +pub use new_delayed_work; + /// A kernel work queue. /// /// Wraps the kernel's C `struct workqueue_struct`. @@ -170,7 +256,7 @@ impl Queue { pub unsafe fn from_raw<'a>(ptr: *const bindings::workqueue_struct) -> &'a Queue { // SAFETY: The `Queue` type is `#[repr(transparent)]`, so the pointer cast is valid. The // caller promises that the pointer is not dangling. - unsafe { &*(ptr as *const Queue) } + unsafe { &*ptr.cast::<Queue>() } } /// Enqueues a work item. @@ -198,7 +284,7 @@ impl Queue { unsafe { w.__enqueue(move |work_ptr| { bindings::queue_work_on( - bindings::wq_misc_consts_WORK_CPU_UNBOUND as _, + bindings::wq_misc_consts_WORK_CPU_UNBOUND as ffi::c_int, queue_ptr, work_ptr, ) @@ -206,6 +292,42 @@ impl Queue { } } + /// Enqueues a delayed work item. + /// + /// This may fail if the work item is already enqueued in a workqueue. + /// + /// The work item will be submitted using `WORK_CPU_UNBOUND`. + pub fn enqueue_delayed<W, const ID: u64>(&self, w: W, delay: Jiffies) -> W::EnqueueOutput + where + W: RawDelayedWorkItem<ID> + Send + 'static, + { + let queue_ptr = self.0.get(); + + // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other + // `__enqueue` requirements are not relevant since `W` is `Send` and static. + // + // The call to `bindings::queue_delayed_work_on` will dereference the provided raw pointer, + // which is ok because `__enqueue` guarantees that the pointer is valid for the duration of + // this closure, and the safety requirements of `RawDelayedWorkItem` expands this + // requirement to apply to the entire `delayed_work`. + // + // Furthermore, if the C workqueue code accesses the pointer after this call to + // `__enqueue`, then the work item was successfully enqueued, and + // `bindings::queue_delayed_work_on` will have returned true. In this case, `__enqueue` + // promises that the raw pointer will stay valid until we call the function pointer in the + // `work_struct`, so the access is ok. + unsafe { + w.__enqueue(move |work_ptr| { + bindings::queue_delayed_work_on( + bindings::wq_misc_consts_WORK_CPU_UNBOUND as ffi::c_int, + queue_ptr, + container_of!(work_ptr, bindings::delayed_work, work), + delay, + ) + }) + } + } + /// Tries to spawn the given function or closure as a work item. /// /// This method can fail because it allocates memory to store the work item. @@ -298,6 +420,16 @@ pub unsafe trait RawWorkItem<const ID: u64> { F: FnOnce(*mut bindings::work_struct) -> bool; } +/// A raw delayed work item. +/// +/// # Safety +/// +/// If the `__enqueue` method in the `RawWorkItem` implementation calls the closure, then the +/// provided pointer must point at the `work` field of a valid `delayed_work`, and the guarantees +/// that `__enqueue` provides about accessing the `work_struct` must also apply to the rest of the +/// `delayed_work` struct. +pub unsafe trait RawDelayedWorkItem<const ID: u64>: RawWorkItem<ID> {} + /// Defines the method that should be called directly when a work item is executed. /// /// This trait is implemented by `Pin<KBox<T>>` and [`Arc<T>`], and is mainly intended to be @@ -403,11 +535,11 @@ impl<T: ?Sized, const ID: u64> Work<T, ID> { // // A pointer cast would also be ok due to `#[repr(transparent)]`. We use `addr_of!` so that // the compiler does not complain that the `work` field is unused. - unsafe { Opaque::raw_get(core::ptr::addr_of!((*ptr).work)) } + unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).work)) } } } -/// Declares that a type has a [`Work<T, ID>`] field. +/// Declares that a type contains a [`Work<T, ID>`]. /// /// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro /// like this: @@ -506,6 +638,178 @@ impl_has_work! { impl{T} HasWork<Self> for ClosureWork<T> { self.work } } +/// Links for a delayed work item. +/// +/// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] +/// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue in +/// a delayed manner. +/// +/// Wraps the kernel's C `struct delayed_work`. +/// +/// This is a helper type used to associate a `delayed_work` with the [`WorkItem`] that uses it. +/// +/// [`run`]: WorkItemPointer::run +#[pin_data] +#[repr(transparent)] +pub struct DelayedWork<T: ?Sized, const ID: u64 = 0> { + #[pin] + dwork: Opaque<bindings::delayed_work>, + _inner: PhantomData<T>, +} + +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl<T: ?Sized, const ID: u64> Send for DelayedWork<T, ID> {} +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl<T: ?Sized, const ID: u64> Sync for DelayedWork<T, ID> {} + +impl<T: ?Sized, const ID: u64> DelayedWork<T, ID> { + /// Creates a new instance of [`DelayedWork`]. + #[inline] + pub fn new( + work_name: &'static CStr, + work_key: Pin<&'static LockClassKey>, + timer_name: &'static CStr, + timer_key: Pin<&'static LockClassKey>, + ) -> impl PinInit<Self> + where + T: WorkItem<ID>, + { + pin_init!(Self { + dwork <- Opaque::ffi_init(|slot: *mut bindings::delayed_work| { + // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as + // the work item function. + unsafe { + bindings::init_work_with_key( + core::ptr::addr_of_mut!((*slot).work), + Some(T::Pointer::run), + false, + work_name.as_char_ptr(), + work_key.as_ptr(), + ) + } + + // SAFETY: The `delayed_work_timer_fn` function pointer can be used here because + // the timer is embedded in a `struct delayed_work`, and only ever scheduled via + // the core workqueue code, and configured to run in irqsafe context. + unsafe { + bindings::timer_init_key( + core::ptr::addr_of_mut!((*slot).timer), + Some(bindings::delayed_work_timer_fn), + bindings::TIMER_IRQSAFE, + timer_name.as_char_ptr(), + timer_key.as_ptr(), + ) + } + }), + _inner: PhantomData, + }) + } + + /// Get a pointer to the inner `delayed_work`. + /// + /// # Safety + /// + /// The provided pointer must not be dangling and must be properly aligned. (But the memory + /// need not be initialized.) + #[inline] + pub unsafe fn raw_as_work(ptr: *const Self) -> *mut Work<T, ID> { + // SAFETY: The caller promises that the pointer is aligned and not dangling. + let dw: *mut bindings::delayed_work = + unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).dwork)) }; + // SAFETY: The caller promises that the pointer is aligned and not dangling. + let wrk: *mut bindings::work_struct = unsafe { core::ptr::addr_of_mut!((*dw).work) }; + // CAST: Work and work_struct have compatible layouts. + wrk.cast() + } +} + +/// Declares that a type contains a [`DelayedWork<T, ID>`]. +/// +/// # Safety +/// +/// The `HasWork<T, ID>` implementation must return a `work_struct` that is stored in the `work` +/// field of a `delayed_work` with the same access rules as the `work_struct`. +pub unsafe trait HasDelayedWork<T, const ID: u64 = 0>: HasWork<T, ID> {} + +/// Used to safely implement the [`HasDelayedWork<T, ID>`] trait. +/// +/// This macro also implements the [`HasWork`] trait, so you do not need to use [`impl_has_work!`] +/// when using this macro. +/// +/// # Examples +/// +/// ``` +/// use kernel::sync::Arc; +/// use kernel::workqueue::{self, impl_has_delayed_work, DelayedWork}; +/// +/// struct MyStruct<'a, T, const N: usize> { +/// work_field: DelayedWork<MyStruct<'a, T, N>, 17>, +/// f: fn(&'a [T; N]), +/// } +/// +/// impl_has_delayed_work! { +/// impl{'a, T, const N: usize} HasDelayedWork<MyStruct<'a, T, N>, 17> +/// for MyStruct<'a, T, N> { self.work_field } +/// } +/// ``` +#[macro_export] +macro_rules! impl_has_delayed_work { + ($(impl$({$($generics:tt)*})? + HasDelayedWork<$work_type:ty $(, $id:tt)?> + for $self:ty + { self.$field:ident } + )*) => {$( + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right + // type. + unsafe impl$(<$($generics)+>)? + $crate::workqueue::HasDelayedWork<$work_type $(, $id)?> for $self {} + + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right + // type. + unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { + #[inline] + unsafe fn raw_get_work( + ptr: *mut Self + ) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { + // SAFETY: The caller promises that the pointer is not dangling. + let ptr: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = unsafe { + ::core::ptr::addr_of_mut!((*ptr).$field) + }; + + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { $crate::workqueue::DelayedWork::raw_as_work(ptr) } + } + + #[inline] + unsafe fn work_container_of( + ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, + ) -> *mut Self { + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + let ptr = unsafe { $crate::workqueue::Work::raw_get(ptr) }; + + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + let delayed_work = unsafe { + $crate::container_of!(ptr, $crate::bindings::delayed_work, work) + }; + + let delayed_work: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = + delayed_work.cast(); + + // SAFETY: The caller promises that the pointer points at a field of the right type + // in the right kind of struct. + unsafe { $crate::container_of!(delayed_work, Self, $field) } + } + } + )*}; +} +pub use impl_has_delayed_work; + // SAFETY: The `__enqueue` implementation in RawWorkItem uses a `work_struct` initialized with the // `run` method of this trait as the function pointer because: // - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. @@ -522,7 +826,7 @@ where { unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. - let ptr = ptr as *mut Work<T, ID>; + let ptr = ptr.cast::<Work<T, ID>>(); // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. @@ -567,6 +871,16 @@ where } } +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for Arc<T> +where + T: WorkItem<ID, Pointer = Self>, + T: HasDelayedWork<T, ID>, +{ +} + // SAFETY: TODO. unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<KBox<T>> where @@ -575,7 +889,7 @@ where { unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. - let ptr = ptr as *mut Work<T, ID>; + let ptr = ptr.cast::<Work<T, ID>>(); // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. @@ -617,6 +931,16 @@ where } } +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of +// the `delayed_work` has the same access rules as its `work` field. +unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for Pin<KBox<T>> +where + T: WorkItem<ID, Pointer = Self>, + T: HasDelayedWork<T, ID>, +{ +} + /// Returns the system work queue (`system_wq`). /// /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs index 75719e7bb491..a49d6db28845 100644 --- a/rust/kernel/xarray.rs +++ b/rust/kernel/xarray.rs @@ -7,9 +7,10 @@ use crate::{ alloc, bindings, build_assert, error::{Error, Result}, + ffi::c_void, types::{ForeignOwnable, NotThreadSafe, Opaque}, }; -use core::{iter, marker::PhantomData, mem, pin::Pin, ptr::NonNull}; +use core::{iter, marker::PhantomData, pin::Pin, ptr::NonNull}; use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; /// An array which efficiently maps sparse integer indices to owned objects. @@ -101,7 +102,7 @@ impl<T: ForeignOwnable> XArray<T> { }) } - fn iter(&self) -> impl Iterator<Item = NonNull<T::PointedTo>> + '_ { + fn iter(&self) -> impl Iterator<Item = NonNull<c_void>> + '_ { let mut index = 0; // SAFETY: `self.xa` is always valid by the type invariant. @@ -179,7 +180,7 @@ impl<T> From<StoreError<T>> for Error { impl<'a, T: ForeignOwnable> Guard<'a, T> { fn load<F, U>(&self, index: usize, f: F) -> Option<U> where - F: FnOnce(NonNull<T::PointedTo>) -> U, + F: FnOnce(NonNull<c_void>) -> U, { // SAFETY: `self.xa.xa` is always valid by the type invariant. let ptr = unsafe { bindings::xa_load(self.xa.xa.get(), index) }; @@ -230,7 +231,7 @@ impl<'a, T: ForeignOwnable> Guard<'a, T> { gfp: alloc::Flags, ) -> Result<Option<T>, StoreError<T>> { build_assert!( - mem::align_of::<T::PointedTo>() >= 4, + T::FOREIGN_ALIGN >= 4, "pointers stored in XArray must be 4-byte aligned" ); let new = value.into_foreign(); diff --git a/rust/macros/module.rs b/rust/macros/module.rs index 2ddd2eeb2852..5ee54a00c0b6 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -57,7 +57,7 @@ impl<'a> ModInfoBuilder<'a> { {cfg} #[doc(hidden)] #[cfg_attr(not(target_os = \"macos\"), link_section = \".modinfo\")] - #[used] + #[used(compiler)] pub static __{module}_{counter}: [u8; {length}] = *{string}; ", cfg = if builtin { @@ -94,7 +94,6 @@ struct ModuleInfo { type_: String, license: String, name: String, - author: Option<String>, authors: Option<Vec<String>>, description: Option<String>, alias: Option<Vec<String>>, @@ -108,7 +107,6 @@ impl ModuleInfo { const EXPECTED_KEYS: &[&str] = &[ "type", "name", - "author", "authors", "description", "license", @@ -134,7 +132,6 @@ impl ModuleInfo { match key.as_str() { "type" => info.type_ = expect_ident(it), "name" => info.name = expect_string_ascii(it), - "author" => info.author = Some(expect_string(it)), "authors" => info.authors = Some(expect_string_array(it)), "description" => info.description = Some(expect_string(it)), "license" => info.license = expect_string_ascii(it), @@ -179,9 +176,6 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { // Rust does not allow hyphens in identifiers, use underscore instead. let ident = info.name.replace('-', "_"); let mut modinfo = ModInfoBuilder::new(ident.as_ref()); - if let Some(author) = info.author { - modinfo.emit("author", &author); - } if let Some(authors) = info.authors { for author in authors { modinfo.emit("author", &author); @@ -249,7 +243,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { // key or a new section. For the moment, keep it simple. #[cfg(MODULE)] #[doc(hidden)] - #[used] + #[used(compiler)] static __IS_RUST_MODULE: () = (); static mut __MOD: ::core::mem::MaybeUninit<{type_}> = @@ -273,7 +267,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[cfg(MODULE)] #[doc(hidden)] - #[used] + #[used(compiler)] #[link_section = \".init.data\"] static __UNIQUE_ID___addressable_init_module: unsafe extern \"C\" fn() -> i32 = init_module; @@ -293,7 +287,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[cfg(MODULE)] #[doc(hidden)] - #[used] + #[used(compiler)] #[link_section = \".exit.data\"] static __UNIQUE_ID___addressable_cleanup_module: extern \"C\" fn() = cleanup_module; @@ -303,7 +297,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))] #[doc(hidden)] #[link_section = \"{initcall_section}\"] - #[used] + #[used(compiler)] pub static __{ident}_initcall: extern \"C\" fn() -> ::kernel::ffi::c_int = __{ident}_init; diff --git a/rust/pin-init/README.md b/rust/pin-init/README.md index 2d0cda961d45..a4c01a8d78b2 100644 --- a/rust/pin-init/README.md +++ b/rust/pin-init/README.md @@ -125,7 +125,7 @@ impl DriverData { fn new() -> impl PinInit<Self, Error> { try_pin_init!(Self { status <- CMutex::new(0), - buffer: Box::init(pin_init::zeroed())?, + buffer: Box::init(pin_init::init_zeroed())?, }? Error) } } diff --git a/rust/pin-init/examples/big_struct_in_place.rs b/rust/pin-init/examples/big_struct_in_place.rs index 30d44a334ffd..c05139927486 100644 --- a/rust/pin-init/examples/big_struct_in_place.rs +++ b/rust/pin-init/examples/big_struct_in_place.rs @@ -4,6 +4,7 @@ use pin_init::*; // Struct with size over 1GiB #[derive(Debug)] +#[allow(dead_code)] pub struct BigStruct { buf: [u8; 1024 * 1024 * 1024], a: u64, @@ -20,20 +21,23 @@ pub struct ManagedBuf { impl ManagedBuf { pub fn new() -> impl Init<Self> { - init!(ManagedBuf { buf <- zeroed() }) + init!(ManagedBuf { buf <- init_zeroed() }) } } fn main() { - // we want to initialize the struct in-place, otherwise we would get a stackoverflow - let buf: Box<BigStruct> = Box::init(init!(BigStruct { - buf <- zeroed(), - a: 7, - b: 186, - c: 7789, - d: 34, - managed_buf <- ManagedBuf::new(), - })) - .unwrap(); - println!("{}", core::mem::size_of_val(&*buf)); + #[cfg(any(feature = "std", feature = "alloc"))] + { + // we want to initialize the struct in-place, otherwise we would get a stackoverflow + let buf: Box<BigStruct> = Box::init(init!(BigStruct { + buf <- init_zeroed(), + a: 7, + b: 186, + c: 7789, + d: 34, + managed_buf <- ManagedBuf::new(), + })) + .unwrap(); + println!("{}", core::mem::size_of_val(&*buf)); + } } diff --git a/rust/pin-init/examples/linked_list.rs b/rust/pin-init/examples/linked_list.rs index 0bbc7b8d83a1..f9e117c7dfe0 100644 --- a/rust/pin-init/examples/linked_list.rs +++ b/rust/pin-init/examples/linked_list.rs @@ -14,8 +14,9 @@ use core::{ use pin_init::*; -#[expect(unused_attributes)] +#[allow(unused_attributes)] mod error; +#[allow(unused_imports)] use error::Error; #[pin_data(PinnedDrop)] @@ -39,6 +40,7 @@ impl ListHead { } #[inline] + #[allow(dead_code)] pub fn insert_next(list: &ListHead) -> impl PinInit<Self, Infallible> + '_ { try_pin_init!(&this in Self { prev: list.next.prev().replace(unsafe { Link::new_unchecked(this)}), @@ -112,6 +114,7 @@ impl Link { } #[inline] + #[allow(dead_code)] fn prev(&self) -> &Link { unsafe { &(*self.0.get().as_ptr()).prev } } @@ -138,7 +141,12 @@ impl Link { } #[allow(dead_code)] +#[cfg(not(any(feature = "std", feature = "alloc")))] +fn main() {} + +#[allow(dead_code)] #[cfg_attr(test, test)] +#[cfg(any(feature = "std", feature = "alloc"))] fn main() -> Result<(), Error> { let a = Box::pin_init(ListHead::new())?; stack_pin_init!(let b = ListHead::insert_next(&a)); diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs index 3e3630780c96..9f295226cd64 100644 --- a/rust/pin-init/examples/mutex.rs +++ b/rust/pin-init/examples/mutex.rs @@ -12,14 +12,15 @@ use core::{ pin::Pin, sync::atomic::{AtomicBool, Ordering}, }; +#[cfg(feature = "std")] use std::{ sync::Arc, - thread::{self, park, sleep, Builder, Thread}, + thread::{self, sleep, Builder, Thread}, time::Duration, }; use pin_init::*; -#[expect(unused_attributes)] +#[allow(unused_attributes)] #[path = "./linked_list.rs"] pub mod linked_list; use linked_list::*; @@ -36,6 +37,7 @@ impl SpinLock { .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) .is_err() { + #[cfg(feature = "std")] while self.inner.load(Ordering::Relaxed) { thread::yield_now(); } @@ -94,7 +96,8 @@ impl<T> CMutex<T> { // println!("wait list length: {}", self.wait_list.size()); while self.locked.get() { drop(sguard); - park(); + #[cfg(feature = "std")] + thread::park(); sguard = self.spin_lock.acquire(); } // This does have an effect, as the ListHead inside wait_entry implements Drop! @@ -131,8 +134,11 @@ impl<T> Drop for CMutexGuard<'_, T> { let sguard = self.mtx.spin_lock.acquire(); self.mtx.locked.set(false); if let Some(list_field) = self.mtx.wait_list.next() { - let wait_entry = list_field.as_ptr().cast::<WaitEntry>(); - unsafe { (*wait_entry).thread.unpark() }; + let _wait_entry = list_field.as_ptr().cast::<WaitEntry>(); + #[cfg(feature = "std")] + unsafe { + (*_wait_entry).thread.unpark() + }; } drop(sguard); } @@ -159,52 +165,61 @@ impl<T> DerefMut for CMutexGuard<'_, T> { struct WaitEntry { #[pin] wait_list: ListHead, + #[cfg(feature = "std")] thread: Thread, } impl WaitEntry { #[inline] fn insert_new(list: &ListHead) -> impl PinInit<Self> + '_ { - pin_init!(Self { - thread: thread::current(), - wait_list <- ListHead::insert_prev(list), - }) + #[cfg(feature = "std")] + { + pin_init!(Self { + thread: thread::current(), + wait_list <- ListHead::insert_prev(list), + }) + } + #[cfg(not(feature = "std"))] + { + pin_init!(Self { + wait_list <- ListHead::insert_prev(list), + }) + } } } -#[cfg(not(any(feature = "std", feature = "alloc")))] -fn main() {} - -#[allow(dead_code)] #[cfg_attr(test, test)] -#[cfg(any(feature = "std", feature = "alloc"))] +#[allow(dead_code)] fn main() { - let mtx: Pin<Arc<CMutex<usize>>> = Arc::pin_init(CMutex::new(0)).unwrap(); - let mut handles = vec![]; - let thread_count = 20; - let workload = if cfg!(miri) { 100 } else { 1_000 }; - for i in 0..thread_count { - let mtx = mtx.clone(); - handles.push( - Builder::new() - .name(format!("worker #{i}")) - .spawn(move || { - for _ in 0..workload { - *mtx.lock() += 1; - } - println!("{i} halfway"); - sleep(Duration::from_millis((i as u64) * 10)); - for _ in 0..workload { - *mtx.lock() += 1; - } - println!("{i} finished"); - }) - .expect("should not fail"), - ); - } - for h in handles { - h.join().expect("thread panicked"); + #[cfg(feature = "std")] + { + let mtx: Pin<Arc<CMutex<usize>>> = Arc::pin_init(CMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = if cfg!(miri) { 100 } else { 1_000 }; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}", &*mtx.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); } - println!("{:?}", &*mtx.lock()); - assert_eq!(*mtx.lock(), workload * thread_count * 2); } diff --git a/rust/pin-init/examples/pthread_mutex.rs b/rust/pin-init/examples/pthread_mutex.rs index 5acc5108b954..49b004c8c137 100644 --- a/rust/pin-init/examples/pthread_mutex.rs +++ b/rust/pin-init/examples/pthread_mutex.rs @@ -44,6 +44,7 @@ mod pthread_mtx { pub enum Error { #[allow(dead_code)] IO(std::io::Error), + #[allow(dead_code)] Alloc, } @@ -61,6 +62,7 @@ mod pthread_mtx { } impl<T> PThreadMutex<T> { + #[allow(dead_code)] pub fn new(data: T) -> impl PinInit<Self, Error> { fn init_raw() -> impl PinInit<UnsafeCell<libc::pthread_mutex_t>, Error> { let init = |slot: *mut UnsafeCell<libc::pthread_mutex_t>| { @@ -103,6 +105,7 @@ mod pthread_mtx { }? Error) } + #[allow(dead_code)] pub fn lock(&self) -> PThreadMutexGuard<'_, T> { // SAFETY: raw is always initialized unsafe { libc::pthread_mutex_lock(self.raw.get()) }; @@ -137,6 +140,7 @@ mod pthread_mtx { } #[cfg_attr(test, test)] +#[cfg_attr(all(test, miri), ignore)] fn main() { #[cfg(all(any(feature = "std", feature = "alloc"), not(windows)))] { diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs index 48531413ab94..0e165daa9798 100644 --- a/rust/pin-init/examples/static_init.rs +++ b/rust/pin-init/examples/static_init.rs @@ -3,6 +3,7 @@ #![allow(clippy::undocumented_unsafe_blocks)] #![cfg_attr(feature = "alloc", feature(allocator_api))] #![cfg_attr(not(RUSTC_LINT_REASONS_IS_STABLE), feature(lint_reasons))] +#![allow(unused_imports)] use core::{ cell::{Cell, UnsafeCell}, @@ -12,12 +13,13 @@ use core::{ time::Duration, }; use pin_init::*; +#[cfg(feature = "std")] use std::{ sync::Arc, thread::{sleep, Builder}, }; -#[expect(unused_attributes)] +#[allow(unused_attributes)] mod mutex; use mutex::*; @@ -82,42 +84,41 @@ unsafe impl PinInit<CMutex<usize>> for CountInit { pub static COUNT: StaticInit<CMutex<usize>, CountInit> = StaticInit::new(CountInit); -#[cfg(not(any(feature = "std", feature = "alloc")))] -fn main() {} - -#[cfg(any(feature = "std", feature = "alloc"))] fn main() { - let mtx: Pin<Arc<CMutex<usize>>> = Arc::pin_init(CMutex::new(0)).unwrap(); - let mut handles = vec![]; - let thread_count = 20; - let workload = 1_000; - for i in 0..thread_count { - let mtx = mtx.clone(); - handles.push( - Builder::new() - .name(format!("worker #{i}")) - .spawn(move || { - for _ in 0..workload { - *COUNT.lock() += 1; - std::thread::sleep(std::time::Duration::from_millis(10)); - *mtx.lock() += 1; - std::thread::sleep(std::time::Duration::from_millis(10)); - *COUNT.lock() += 1; - } - println!("{i} halfway"); - sleep(Duration::from_millis((i as u64) * 10)); - for _ in 0..workload { - std::thread::sleep(std::time::Duration::from_millis(10)); - *mtx.lock() += 1; - } - println!("{i} finished"); - }) - .expect("should not fail"), - ); - } - for h in handles { - h.join().expect("thread panicked"); + #[cfg(feature = "std")] + { + let mtx: Pin<Arc<CMutex<usize>>> = Arc::pin_init(CMutex::new(0)).unwrap(); + let mut handles = vec![]; + let thread_count = 20; + let workload = 1_000; + for i in 0..thread_count { + let mtx = mtx.clone(); + handles.push( + Builder::new() + .name(format!("worker #{i}")) + .spawn(move || { + for _ in 0..workload { + *COUNT.lock() += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + *mtx.lock() += 1; + std::thread::sleep(std::time::Duration::from_millis(10)); + *COUNT.lock() += 1; + } + println!("{i} halfway"); + sleep(Duration::from_millis((i as u64) * 10)); + for _ in 0..workload { + std::thread::sleep(std::time::Duration::from_millis(10)); + *mtx.lock() += 1; + } + println!("{i} finished"); + }) + .expect("should not fail"), + ); + } + for h in handles { + h.join().expect("thread panicked"); + } + println!("{:?}, {:?}", &*mtx.lock(), &*COUNT.lock()); + assert_eq!(*mtx.lock(), workload * thread_count * 2); } - println!("{:?}, {:?}", &*mtx.lock(), &*COUNT.lock()); - assert_eq!(*mtx.lock(), workload * thread_count * 2); } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 557b5948cddc..90f18e9a2912 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -188,6 +188,7 @@ impl<T> StackInit<T> { } #[test] +#[cfg(feature = "std")] fn stack_init_reuse() { use ::std::{borrow::ToOwned, println, string::String}; use core::pin::pin; diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index 9ab34036e6bc..62e013a5cc20 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -148,7 +148,7 @@ //! fn new() -> impl PinInit<Self, Error> { //! try_pin_init!(Self { //! status <- CMutex::new(0), -//! buffer: Box::init(pin_init::zeroed())?, +//! buffer: Box::init(pin_init::init_zeroed())?, //! }? Error) //! } //! } @@ -742,7 +742,7 @@ macro_rules! stack_try_pin_init { /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] /// pointer named `this` inside of the initializer. -/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the +/// - Using struct update syntax one can place `..Zeroable::init_zeroed()` at the very end of the /// struct, this initializes every field with 0 and then runs all initializers specified in the /// body. This can only be done if [`Zeroable`] is implemented for the struct. /// @@ -769,7 +769,7 @@ macro_rules! stack_try_pin_init { /// }); /// let init = pin_init!(Buf { /// buf: [1; 64], -/// ..Zeroable::zeroed() +/// ..Zeroable::init_zeroed() /// }); /// ``` /// @@ -805,7 +805,7 @@ macro_rules! pin_init { /// ```rust /// # #![feature(allocator_api)] /// # #[path = "../examples/error.rs"] mod error; use error::Error; -/// use pin_init::{pin_data, try_pin_init, PinInit, InPlaceInit, zeroed}; +/// use pin_init::{pin_data, try_pin_init, PinInit, InPlaceInit, init_zeroed}; /// /// #[pin_data] /// struct BigBuf { @@ -817,7 +817,7 @@ macro_rules! pin_init { /// impl BigBuf { /// fn new() -> impl PinInit<Self, Error> { /// try_pin_init!(Self { -/// big: Box::init(zeroed())?, +/// big: Box::init(init_zeroed())?, /// small: [0; 1024 * 1024], /// ptr: core::ptr::null_mut(), /// }? Error) @@ -866,7 +866,7 @@ macro_rules! try_pin_init { /// # #[path = "../examples/error.rs"] mod error; use error::Error; /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; /// # use pin_init::InPlaceInit; -/// use pin_init::{init, Init, zeroed}; +/// use pin_init::{init, Init, init_zeroed}; /// /// struct BigBuf { /// small: [u8; 1024 * 1024], @@ -875,7 +875,7 @@ macro_rules! try_pin_init { /// impl BigBuf { /// fn new() -> impl Init<Self> { /// init!(Self { -/// small <- zeroed(), +/// small <- init_zeroed(), /// }) /// } /// } @@ -913,7 +913,7 @@ macro_rules! init { /// # #![feature(allocator_api)] /// # use core::alloc::AllocError; /// # use pin_init::InPlaceInit; -/// use pin_init::{try_init, Init, zeroed}; +/// use pin_init::{try_init, Init, init_zeroed}; /// /// struct BigBuf { /// big: Box<[u8; 1024 * 1024 * 1024]>, @@ -923,7 +923,7 @@ macro_rules! init { /// impl BigBuf { /// fn new() -> impl Init<Self, AllocError> { /// try_init!(Self { -/// big: Box::init(zeroed())?, +/// big: Box::init(init_zeroed())?, /// small: [0; 1024 * 1024], /// }? AllocError) /// } @@ -953,7 +953,7 @@ macro_rules! try_init { /// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is /// structurally pinned. /// -/// # Example +/// # Examples /// /// This will succeed: /// ``` @@ -1170,7 +1170,7 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// /// ```rust /// # #![expect(clippy::disallowed_names)] - /// use pin_init::{init, zeroed, Init}; + /// use pin_init::{init, init_zeroed, Init}; /// /// struct Foo { /// buf: [u8; 1_000_000], @@ -1183,7 +1183,7 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// } /// /// let foo = init!(Foo { - /// buf <- zeroed() + /// buf <- init_zeroed() /// }).chain(|foo| { /// foo.setup(); /// Ok(()) @@ -1390,20 +1390,44 @@ where unsafe { pin_init_from_closure(init) } } -// SAFETY: Every type can be initialized by-value. -unsafe impl<T, E> Init<T, E> for T { - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: TODO. +// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of `slot`. +unsafe impl<T> Init<T> for T { + unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { + // SAFETY: `slot` is valid for writes by the safety requirements of this function. + unsafe { slot.write(self) }; + Ok(()) + } +} + +// SAFETY: the `__pinned_init` function always returns `Ok(())` and initializes every field of +// `slot`. Additionally, all pinning invariants of `T` are upheld. +unsafe impl<T> PinInit<T> for T { + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), Infallible> { + // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self) }; Ok(()) } } -// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`. -unsafe impl<T, E> PinInit<T, E> for T { +// SAFETY: when the `__init` function returns with +// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. +// - `Err(err)`, slot was not written to. +unsafe impl<T, E> Init<T, E> for Result<T, E> { + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: `slot` is valid for writes by the safety requirements of this function. + unsafe { slot.write(self?) }; + Ok(()) + } +} + +// SAFETY: when the `__pinned_init` function returns with +// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. +// - `Err(err)`, slot was not written to. +unsafe impl<T, E> PinInit<T, E> for Result<T, E> { unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: TODO. - unsafe { self.__init(slot) } + // SAFETY: `slot` is valid for writes by the safety requirements of this function. + unsafe { slot.write(self?) }; + Ok(()) } } @@ -1471,7 +1495,45 @@ pub unsafe trait PinnedDrop: __internal::HasPinData { /// ```rust,ignore /// let val: Self = unsafe { core::mem::zeroed() }; /// ``` -pub unsafe trait Zeroable {} +pub unsafe trait Zeroable { + /// Create a new zeroed `Self`. + /// + /// The returned initializer will write `0x00` to every byte of the given `slot`. + #[inline] + fn init_zeroed() -> impl Init<Self> + where + Self: Sized, + { + init_zeroed() + } + + /// Create a `Self` consisting of all zeroes. + /// + /// Whenever a type implements [`Zeroable`], this function should be preferred over + /// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`. + /// + /// # Examples + /// + /// ``` + /// use pin_init::{Zeroable, zeroed}; + /// + /// #[derive(Zeroable)] + /// struct Point { + /// x: u32, + /// y: u32, + /// } + /// + /// let point: Point = zeroed(); + /// assert_eq!(point.x, 0); + /// assert_eq!(point.y, 0); + /// ``` + fn zeroed() -> Self + where + Self: Sized, + { + zeroed() + } +} /// Marker trait for types that allow `Option<Self>` to be set to all zeroes in order to write /// `None` to that location. @@ -1484,11 +1546,21 @@ pub unsafe trait ZeroableOption {} // SAFETY: by the safety requirement of `ZeroableOption`, this is valid. unsafe impl<T: ZeroableOption> Zeroable for Option<T> {} -/// Create a new zeroed T. +// SAFETY: `Option<&T>` is part of the option layout optimization guarantee: +// <https://doc.rust-lang.org/stable/std/option/index.html#representation>. +unsafe impl<T> ZeroableOption for &T {} +// SAFETY: `Option<&mut T>` is part of the option layout optimization guarantee: +// <https://doc.rust-lang.org/stable/std/option/index.html#representation>. +unsafe impl<T> ZeroableOption for &mut T {} +// SAFETY: `Option<NonNull<T>>` is part of the option layout optimization guarantee: +// <https://doc.rust-lang.org/stable/std/option/index.html#representation>. +unsafe impl<T> ZeroableOption for NonNull<T> {} + +/// Create an initializer for a zeroed `T`. /// /// The returned initializer will write `0x00` to every byte of the given `slot`. #[inline] -pub fn zeroed<T: Zeroable>() -> impl Init<T> { +pub fn init_zeroed<T: Zeroable>() -> impl Init<T> { // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` // and because we write all zeroes, the memory is initialized. unsafe { @@ -1499,6 +1571,31 @@ pub fn zeroed<T: Zeroable>() -> impl Init<T> { } } +/// Create a `T` consisting of all zeroes. +/// +/// Whenever a type implements [`Zeroable`], this function should be preferred over +/// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`. +/// +/// # Examples +/// +/// ``` +/// use pin_init::{Zeroable, zeroed}; +/// +/// #[derive(Zeroable)] +/// struct Point { +/// x: u32, +/// y: u32, +/// } +/// +/// let point: Point = zeroed(); +/// assert_eq!(point.x, 0); +/// assert_eq!(point.y, 0); +/// ``` +pub const fn zeroed<T: Zeroable>() -> T { + // SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`. + unsafe { core::mem::zeroed() } +} + macro_rules! impl_zeroable { ($($({$($generics:tt)*})? $t:ty, )*) => { // SAFETY: Safety comments written in the macro invocation. @@ -1536,7 +1633,6 @@ impl_zeroable! { Option<NonZeroU128>, Option<NonZeroUsize>, Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>, Option<NonZeroI128>, Option<NonZeroIsize>, - {<T>} Option<NonNull<T>>, // SAFETY: `null` pointer is valid. // @@ -1566,6 +1662,22 @@ macro_rules! impl_tuple_zeroable { impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); +macro_rules! impl_fn_zeroable_option { + ([$($abi:literal),* $(,)?] $args:tt) => { + $(impl_fn_zeroable_option!({extern $abi} $args);)* + $(impl_fn_zeroable_option!({unsafe extern $abi} $args);)* + }; + ({$($prefix:tt)*} {$(,)?}) => {}; + ({$($prefix:tt)*} {$ret:ident, $($rest:ident),* $(,)?}) => { + // SAFETY: function pointers are part of the option layout optimization: + // <https://doc.rust-lang.org/stable/std/option/index.html#representation>. + unsafe impl<$ret, $($rest),*> ZeroableOption for $($prefix)* fn($($rest),*) -> $ret {} + impl_fn_zeroable_option!({$($prefix)*} {$($rest),*,}); + }; +} + +impl_fn_zeroable_option!(["Rust", "C"] { A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U }); + /// This trait allows creating an instance of `Self` which contains exactly one /// [structurally pinned value](https://doc.rust-lang.org/std/pin/index.html#projections-and-structural-pinning). /// diff --git a/rust/pin-init/src/macros.rs b/rust/pin-init/src/macros.rs index 935d77745d1d..9ced630737b8 100644 --- a/rust/pin-init/src/macros.rs +++ b/rust/pin-init/src/macros.rs @@ -1030,7 +1030,7 @@ macro_rules! __pin_data { /// /// This macro has multiple internal call configurations, these are always the very first ident: /// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros. -/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled. +/// - `with_update_parsed`: when the `..Zeroable::init_zeroed()` syntax has been handled. /// - `init_slot`: recursively creates the code that initializes all fields in `slot`. /// - `make_initializer`: recursively create the struct initializer that guarantees that every /// field has been initialized exactly once. @@ -1059,7 +1059,7 @@ macro_rules! __init_internal { @data($data, $($use_data)?), @has_data($has_data, $get_data), @construct_closure($construct_closure), - @zeroed(), // Nothing means default behavior. + @init_zeroed(), // Nothing means default behavior. ) }; ( @@ -1074,7 +1074,7 @@ macro_rules! __init_internal { @has_data($has_data:ident, $get_data:ident), // `pin_init_from_closure` or `init_from_closure`. @construct_closure($construct_closure:ident), - @munch_fields(..Zeroable::zeroed()), + @munch_fields(..Zeroable::init_zeroed()), ) => { $crate::__init_internal!(with_update_parsed: @this($($this)?), @@ -1084,7 +1084,7 @@ macro_rules! __init_internal { @data($data, $($use_data)?), @has_data($has_data, $get_data), @construct_closure($construct_closure), - @zeroed(()), // `()` means zero all fields not mentioned. + @init_zeroed(()), // `()` means zero all fields not mentioned. ) }; ( @@ -1124,7 +1124,7 @@ macro_rules! __init_internal { @has_data($has_data:ident, $get_data:ident), // `pin_init_from_closure` or `init_from_closure`. @construct_closure($construct_closure:ident), - @zeroed($($init_zeroed:expr)?), + @init_zeroed($($init_zeroed:expr)?), ) => {{ // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return // type and shadow it later when we insert the arbitrary user code. That way there will be @@ -1196,7 +1196,7 @@ macro_rules! __init_internal { @data($data:ident), @slot($slot:ident), @guards($($guards:ident,)*), - @munch_fields($(..Zeroable::zeroed())? $(,)?), + @munch_fields($(..Zeroable::init_zeroed())? $(,)?), ) => { // Endpoint of munching, no fields are left. If execution reaches this point, all fields // have been initialized. Therefore we can now dismiss the guards by forgetting them. @@ -1300,11 +1300,11 @@ macro_rules! __init_internal { (make_initializer: @slot($slot:ident), @type_name($t:path), - @munch_fields(..Zeroable::zeroed() $(,)?), + @munch_fields(..Zeroable::init_zeroed() $(,)?), @acc($($acc:tt)*), ) => { // Endpoint, nothing more to munch, create the initializer. Since the users specified - // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have + // `..Zeroable::init_zeroed()`, the slot will already have been zeroed and all field that have // not been overwritten are thus zero and initialized. We still check that all fields are // actually accessible by using the struct update syntax ourselves. // We are inside of a closure that is never executed and thus we can abuse `slot` to diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs index c98d7a8cde77..31c2f713313f 100644 --- a/rust/uapi/lib.rs +++ b/rust/uapi/lib.rs @@ -14,6 +14,9 @@ #![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))] #![allow( clippy::all, + clippy::cast_lossless, + clippy::ptr_as_ptr, + clippy::ref_as_ptr, clippy::undocumented_unsafe_blocks, dead_code, missing_docs, |