From 16df3873c80388a71c4695909a51be0c809b889c Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Thu, 23 Apr 2026 16:11:44 +0900 Subject: [PATCH 001/131] gpu: nova-core: simplify and_then with condition to filter This code is more simply expressed using Option::filter instead of the and_then with conditional. This fixes the following warning with latest nightly Rust clippy build: warning: manual implementation of `Option::filter` --> drivers/gpu/nova-core/firmware.rs:391:14 | 391 | .and_then(|hdr| { | ______________^ 392 | | if hdr.bin_magic == BIN_MAGIC { 393 | | Some(hdr) 394 | | } else { ... | 397 | | }) | |______________^ help: try: `filter(|hdr| hdr.bin_magic == BIN_MAGIC)` | = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#manual_filter = note: `-D clippy::manual-filter` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::manual_filter)]` Cc: stable@vger.kernel.org Fixes: d6cb7319e64e ("gpu: nova-core: firmware: add support for common firmware header") Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260423-fix-filter-v1-1-b3b197c65daf@nvidia.com [aliceryhl: add Fixes: tag and quote the warning it fixes] Signed-off-by: Alice Ryhl --- drivers/gpu/nova-core/firmware.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 6c2ab69cb605..3aac073efee2 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -388,13 +388,7 @@ fn new(fw: &'a firmware::Firmware) -> Result { // Extract header. .and_then(BinHdr::from_bytes_copy) // Validate header. - .and_then(|hdr| { - if hdr.bin_magic == BIN_MAGIC { - Some(hdr) - } else { - None - } - }) + .filter(|hdr| hdr.bin_magic == BIN_MAGIC) .map(|hdr| Self { hdr, fw }) .ok_or(EINVAL) } From 7b00648f48f5ac90d66f9eab4dfe121308a1076e Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Thu, 9 Apr 2026 10:51:24 -0700 Subject: [PATCH 002/131] drm/tyr: Use register! macro for GPU_CONTROL Define the GPU_CONTROL register block with the kernel's register! macro and switch the current GPU control paths over to the new typed register definitions. This replaces manual register constants, bit masks, shifts, and the hand-written GpuId parsing code with typed register and field accessors. It also adds helpers for combining split 64-bit registers and uses the new definitions in reset, L2 power-on, and GPU info readout/logging paths. This reduces open-coded bit manipulation making the code easier to read and maintain. Acked-by: Boris Brezillon Signed-off-by: Daniel Almeida Co-developed-by: Deborah Brouwer Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260409-b4-tyr-use-register-macro-v5-v5-1-8abfff8a0204@collabora.com [aliceryhl: reformat long comment] Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/driver.rs | 19 +- drivers/gpu/drm/tyr/gpu.rs | 173 +++---- drivers/gpu/drm/tyr/regs.rs | 900 ++++++++++++++++++++++++++++++++-- 3 files changed, 926 insertions(+), 166 deletions(-) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 279710b36a10..e8acd2d69468 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -13,7 +13,10 @@ devres::Devres, drm, drm::ioctl, - io::poll, + io::{ + poll, + Io, // + }, new_mutex, of, platform, @@ -34,7 +37,7 @@ gem::TyrObject, gpu, gpu::GpuInfo, - regs, // + regs::gpu_control::*, // }; pub(crate) type IoMem = kernel::io::mem::IoMem; @@ -66,11 +69,15 @@ pub(crate) struct TyrDrmDeviceData { } fn issue_soft_reset(dev: &Device, iomem: &Devres) -> Result { - regs::GPU_CMD.write(dev, iomem, regs::GPU_CMD_SOFT_RESET)?; + let io = (*iomem).access(dev)?; + io.write_reg(GPU_COMMAND::reset(ResetMode::SoftReset)); poll::read_poll_timeout( - || regs::GPU_IRQ_RAWSTAT.read(dev, iomem), - |status| *status & regs::GPU_IRQ_RAWSTAT_RESET_COMPLETED != 0, + || { + let io = (*iomem).access(dev)?; + Ok(io.read(GPU_IRQ_RAWSTAT)) + }, + |status| status.reset_completed(), time::Delta::from_millis(1), time::Delta::from_millis(100), ) @@ -115,7 +122,7 @@ fn probe( gpu::l2_power_on(pdev.as_ref(), &iomem)?; let gpu_info = GpuInfo::new(pdev.as_ref(), &iomem)?; - gpu_info.log(pdev); + gpu_info.log(pdev.as_ref()); let platform: ARef = pdev.into(); diff --git a/drivers/gpu/drm/tyr/gpu.rs b/drivers/gpu/drm/tyr/gpu.rs index a88775160f98..1e1e4103c575 100644 --- a/drivers/gpu/drm/tyr/gpu.rs +++ b/drivers/gpu/drm/tyr/gpu.rs @@ -5,14 +5,16 @@ DerefMut, // }; use kernel::{ - bits::genmask_u32, device::{ Bound, Device, // }, devres::Devres, - io::poll, - platform, + io::{ + poll, + register::Array, + Io, // + }, prelude::*, time::Delta, transmute::AsBytes, @@ -21,7 +23,10 @@ use crate::{ driver::IoMem, - regs, // + regs::{ + gpu_control::*, + join_u64, // + }, // }; /// Struct containing information that can be queried by userspace. This is read from @@ -29,120 +34,95 @@ /// /// # Invariants /// -/// - The layout of this struct identical to the C `struct drm_panthor_gpu_info`. +/// - The layout of this struct is identical to the C `struct drm_panthor_gpu_info`. #[repr(transparent)] #[derive(Clone, Copy)] pub(crate) struct GpuInfo(pub(crate) uapi::drm_panthor_gpu_info); impl GpuInfo { pub(crate) fn new(dev: &Device, iomem: &Devres) -> Result { - let gpu_id = regs::GPU_ID.read(dev, iomem)?; - let csf_id = regs::GPU_CSF_ID.read(dev, iomem)?; - let gpu_rev = regs::GPU_REVID.read(dev, iomem)?; - let core_features = regs::GPU_CORE_FEATURES.read(dev, iomem)?; - let l2_features = regs::GPU_L2_FEATURES.read(dev, iomem)?; - let tiler_features = regs::GPU_TILER_FEATURES.read(dev, iomem)?; - let mem_features = regs::GPU_MEM_FEATURES.read(dev, iomem)?; - let mmu_features = regs::GPU_MMU_FEATURES.read(dev, iomem)?; - let thread_features = regs::GPU_THREAD_FEATURES.read(dev, iomem)?; - let max_threads = regs::GPU_THREAD_MAX_THREADS.read(dev, iomem)?; - let thread_max_workgroup_size = regs::GPU_THREAD_MAX_WORKGROUP_SIZE.read(dev, iomem)?; - let thread_max_barrier_size = regs::GPU_THREAD_MAX_BARRIER_SIZE.read(dev, iomem)?; - let coherency_features = regs::GPU_COHERENCY_FEATURES.read(dev, iomem)?; - - let texture_features = regs::GPU_TEXTURE_FEATURES0.read(dev, iomem)?; - - let as_present = regs::GPU_AS_PRESENT.read(dev, iomem)?; - - let shader_present = u64::from(regs::GPU_SHADER_PRESENT_LO.read(dev, iomem)?); - let shader_present = - shader_present | u64::from(regs::GPU_SHADER_PRESENT_HI.read(dev, iomem)?) << 32; - - let tiler_present = u64::from(regs::GPU_TILER_PRESENT_LO.read(dev, iomem)?); - let tiler_present = - tiler_present | u64::from(regs::GPU_TILER_PRESENT_HI.read(dev, iomem)?) << 32; - - let l2_present = u64::from(regs::GPU_L2_PRESENT_LO.read(dev, iomem)?); - let l2_present = l2_present | u64::from(regs::GPU_L2_PRESENT_HI.read(dev, iomem)?) << 32; + let io = (*iomem).access(dev)?; Ok(Self(uapi::drm_panthor_gpu_info { - gpu_id, - gpu_rev, - csf_id, - l2_features, - tiler_features, - mem_features, - mmu_features, - thread_features, - max_threads, - thread_max_workgroup_size, - thread_max_barrier_size, - coherency_features, - // TODO: Add texture_features_{1,2,3}. - texture_features: [texture_features, 0, 0, 0], - as_present, + gpu_id: io.read(GPU_ID).into_raw(), + gpu_rev: io.read(REVIDR).into_raw(), + csf_id: io.read(CSF_ID).into_raw(), + l2_features: io.read(L2_FEATURES).into_raw(), + tiler_features: io.read(TILER_FEATURES).into_raw(), + mem_features: io.read(MEM_FEATURES).into_raw(), + mmu_features: io.read(MMU_FEATURES).into_raw(), + thread_features: io.read(THREAD_FEATURES).into_raw(), + max_threads: io.read(THREAD_MAX_THREADS).into_raw(), + thread_max_workgroup_size: io.read(THREAD_MAX_WORKGROUP_SIZE).into_raw(), + thread_max_barrier_size: io.read(THREAD_MAX_BARRIER_SIZE).into_raw(), + coherency_features: io.read(COHERENCY_FEATURES).into_raw(), + texture_features: [ + io.read(TEXTURE_FEATURES::at(0)).supported_formats().get(), + io.read(TEXTURE_FEATURES::at(1)).supported_formats().get(), + io.read(TEXTURE_FEATURES::at(2)).supported_formats().get(), + io.read(TEXTURE_FEATURES::at(3)).supported_formats().get(), + ], + as_present: io.read(AS_PRESENT).into_raw(), selected_coherency: uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE, - shader_present, - l2_present, - tiler_present, - core_features, + shader_present: join_u64( + io.read(SHADER_PRESENT_LO).into_raw(), + io.read(SHADER_PRESENT_HI).into_raw(), + ), + l2_present: join_u64( + io.read(L2_PRESENT_LO).into_raw(), + io.read(L2_PRESENT_HI).into_raw(), + ), + tiler_present: join_u64( + io.read(TILER_PRESENT_LO).into_raw(), + io.read(TILER_PRESENT_HI).into_raw(), + ), + core_features: io.read(CORE_FEATURES).into_raw(), + // Padding must be zero. pad: 0, + //GPU_FEATURES register is not available; it was introduced in arch 11.x. gpu_features: 0, })) } - pub(crate) fn log(&self, pdev: &platform::Device) { - let gpu_id = GpuId::from(self.gpu_id); + pub(crate) fn log(&self, dev: &Device) { + let gpu_id = GPU_ID::from_raw(self.gpu_id); - let model_name = if let Some(model) = GPU_MODELS - .iter() - .find(|&f| f.arch_major == gpu_id.arch_major && f.prod_major == gpu_id.prod_major) - { + let model_name = if let Some(model) = GPU_MODELS.iter().find(|&f| { + f.arch_major == gpu_id.arch_major().get() && f.prod_major == gpu_id.prod_major().get() + }) { model.name } else { "unknown" }; dev_info!( - pdev, + dev, "mali-{} id 0x{:x} major 0x{:x} minor 0x{:x} status 0x{:x}", model_name, - self.gpu_id >> 16, - gpu_id.ver_major, - gpu_id.ver_minor, - gpu_id.ver_status + gpu_id.into_raw() >> 16, + gpu_id.ver_major().get(), + gpu_id.ver_minor().get(), + gpu_id.ver_status().get() ); dev_info!( - pdev, + dev, "Features: L2:{:#x} Tiler:{:#x} Mem:{:#x} MMU:{:#x} AS:{:#x}", self.l2_features, self.tiler_features, self.mem_features, self.mmu_features, - self.as_present + self.as_present, ); dev_info!( - pdev, + dev, "shader_present=0x{:016x} l2_present=0x{:016x} tiler_present=0x{:016x}", self.shader_present, self.l2_present, - self.tiler_present + self.tiler_present, ); } - - /// Returns the number of virtual address bits supported by the GPU. - #[expect(dead_code)] - pub(crate) fn va_bits(&self) -> u32 { - self.mmu_features & genmask_u32(0..=7) - } - - /// Returns the number of physical address bits supported by the GPU. - #[expect(dead_code)] - pub(crate) fn pa_bits(&self) -> u32 { - (self.mmu_features >> 8) & genmask_u32(0..=7) - } } impl Deref for GpuInfo { @@ -182,38 +162,17 @@ struct GpuModels { prod_major: 7, }]; -#[allow(dead_code)] -pub(crate) struct GpuId { - pub(crate) arch_major: u32, - pub(crate) arch_minor: u32, - pub(crate) arch_rev: u32, - pub(crate) prod_major: u32, - pub(crate) ver_major: u32, - pub(crate) ver_minor: u32, - pub(crate) ver_status: u32, -} - -impl From for GpuId { - fn from(value: u32) -> Self { - GpuId { - arch_major: (value & genmask_u32(28..=31)) >> 28, - arch_minor: (value & genmask_u32(24..=27)) >> 24, - arch_rev: (value & genmask_u32(20..=23)) >> 20, - prod_major: (value & genmask_u32(16..=19)) >> 16, - ver_major: (value & genmask_u32(12..=15)) >> 12, - ver_minor: (value & genmask_u32(4..=11)) >> 4, - ver_status: value & genmask_u32(0..=3), - } - } -} - /// Powers on the l2 block. pub(crate) fn l2_power_on(dev: &Device, iomem: &Devres) -> Result { - regs::L2_PWRON_LO.write(dev, iomem, 1)?; + let io = (*iomem).access(dev)?; + io.write_reg(L2_PWRON_LO::zeroed().with_const_request::<1>()); poll::read_poll_timeout( - || regs::L2_READY_LO.read(dev, iomem), - |status| *status == 1, + || { + let io = (*iomem).access(dev)?; + Ok(io.read(L2_READY_LO)) + }, + |status| status.ready() == 1, Delta::from_millis(1), Delta::from_millis(100), ) diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index 611870c2e6af..3b246ade6096 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -1,5 +1,25 @@ // SPDX-License-Identifier: GPL-2.0 or MIT +//! # Definitions +//! +//! - **CEU**: Command Execution Unit - A hardware component that executes commands (instructions) +//! from the command stream. +//! - **CS**: Command Stream - A sequence of instructions (commands) used to control a particular +//! job or sequence of jobs. The instructions exist in one or more command buffers. +//! - **CSF**: Command Stream Frontend - The interface and implementation for job submission +//! exposed to the host CPU driver. This includes the global interface, as well as CSG and CS +//! interfaces. +//! - **CSG**: Command Stream Group - A group of related command streams. The CSF manages multiple +//! CSGs, and each CSG contains multiple CSs. +//! - **CSHW**: Command Stream Hardware - The hardware interpreting command streams, including the +//! iterator control aspects. Implements the CSF in conjunction with the MCU. +//! - **GLB**: Global - Prefix for global interface registers that control operations common to +//! all CSs. +//! - **JASID**: Job Address Space ID - Identifies the address space for a job. +//! - **MCU**: Microcontroller Unit - Implements the CSF in conjunction with the command stream +//! hardware. +//! - **MMU**: Memory Management Unit - Handles address translation and memory access protection. + // We don't expect that all the registers and fields will be used, even in the // future. // @@ -41,64 +61,838 @@ pub(crate) fn write(&self, dev: &Device, iomem: &Devres, value: u3 } } -pub(crate) const GPU_ID: Register<0x0> = Register; -pub(crate) const GPU_L2_FEATURES: Register<0x4> = Register; -pub(crate) const GPU_CORE_FEATURES: Register<0x8> = Register; -pub(crate) const GPU_CSF_ID: Register<0x1c> = Register; -pub(crate) const GPU_REVID: Register<0x280> = Register; -pub(crate) const GPU_TILER_FEATURES: Register<0xc> = Register; -pub(crate) const GPU_MEM_FEATURES: Register<0x10> = Register; -pub(crate) const GPU_MMU_FEATURES: Register<0x14> = Register; -pub(crate) const GPU_AS_PRESENT: Register<0x18> = Register; -pub(crate) const GPU_IRQ_RAWSTAT: Register<0x20> = Register; +/// Combine two 32-bit values into a single 64-bit value. +pub(crate) fn join_u64(lo: u32, hi: u32) -> u64 { + (u64::from(lo)) | ((u64::from(hi)) << 32) +} -pub(crate) const GPU_IRQ_RAWSTAT_FAULT: u32 = bit_u32(0); -pub(crate) const GPU_IRQ_RAWSTAT_PROTECTED_FAULT: u32 = bit_u32(1); -pub(crate) const GPU_IRQ_RAWSTAT_RESET_COMPLETED: u32 = bit_u32(8); -pub(crate) const GPU_IRQ_RAWSTAT_POWER_CHANGED_SINGLE: u32 = bit_u32(9); -pub(crate) const GPU_IRQ_RAWSTAT_POWER_CHANGED_ALL: u32 = bit_u32(10); -pub(crate) const GPU_IRQ_RAWSTAT_CLEAN_CACHES_COMPLETED: u32 = bit_u32(17); -pub(crate) const GPU_IRQ_RAWSTAT_DOORBELL_STATUS: u32 = bit_u32(18); -pub(crate) const GPU_IRQ_RAWSTAT_MCU_STATUS: u32 = bit_u32(19); +/// Read a logical 64-bit value from split 32-bit registers without tearing. +pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn() -> u32) -> u64 { + loop { + let hi1 = hi_read(); + let lo = lo_read(); + let hi2 = hi_read(); -pub(crate) const GPU_IRQ_CLEAR: Register<0x24> = Register; -pub(crate) const GPU_IRQ_MASK: Register<0x28> = Register; -pub(crate) const GPU_IRQ_STAT: Register<0x2c> = Register; -pub(crate) const GPU_CMD: Register<0x30> = Register; -pub(crate) const GPU_CMD_SOFT_RESET: u32 = 1 | (1 << 8); -pub(crate) const GPU_CMD_HARD_RESET: u32 = 1 | (2 << 8); -pub(crate) const GPU_THREAD_FEATURES: Register<0xac> = Register; -pub(crate) const GPU_THREAD_MAX_THREADS: Register<0xa0> = Register; -pub(crate) const GPU_THREAD_MAX_WORKGROUP_SIZE: Register<0xa4> = Register; -pub(crate) const GPU_THREAD_MAX_BARRIER_SIZE: Register<0xa8> = Register; -pub(crate) const GPU_TEXTURE_FEATURES0: Register<0xb0> = Register; -pub(crate) const GPU_SHADER_PRESENT_LO: Register<0x100> = Register; -pub(crate) const GPU_SHADER_PRESENT_HI: Register<0x104> = Register; -pub(crate) const GPU_TILER_PRESENT_LO: Register<0x110> = Register; -pub(crate) const GPU_TILER_PRESENT_HI: Register<0x114> = Register; -pub(crate) const GPU_L2_PRESENT_LO: Register<0x120> = Register; -pub(crate) const GPU_L2_PRESENT_HI: Register<0x124> = Register; -pub(crate) const L2_READY_LO: Register<0x160> = Register; -pub(crate) const L2_READY_HI: Register<0x164> = Register; -pub(crate) const L2_PWRON_LO: Register<0x1a0> = Register; -pub(crate) const L2_PWRON_HI: Register<0x1a4> = Register; -pub(crate) const L2_PWRTRANS_LO: Register<0x220> = Register; -pub(crate) const L2_PWRTRANS_HI: Register<0x204> = Register; -pub(crate) const L2_PWRACTIVE_LO: Register<0x260> = Register; -pub(crate) const L2_PWRACTIVE_HI: Register<0x264> = Register; + if hi1 == hi2 { + return join_u64(lo, hi1); + } + } +} -pub(crate) const MCU_CONTROL: Register<0x700> = Register; -pub(crate) const MCU_CONTROL_ENABLE: u32 = 1; -pub(crate) const MCU_CONTROL_AUTO: u32 = 2; -pub(crate) const MCU_CONTROL_DISABLE: u32 = 0; +/// These registers correspond to the GPU_CONTROL register page. +/// They are involved in GPU configuration and control. +pub(crate) mod gpu_control { + use core::convert::TryFrom; + use kernel::{ + error::{ + code::EINVAL, + Error, // + }, + num::Bounded, + register, + uapi, // + }; + use pin_init::Zeroable; -pub(crate) const MCU_STATUS: Register<0x704> = Register; -pub(crate) const MCU_STATUS_DISABLED: u32 = 0; -pub(crate) const MCU_STATUS_ENABLED: u32 = 1; -pub(crate) const MCU_STATUS_HALT: u32 = 2; -pub(crate) const MCU_STATUS_FATAL: u32 = 3; + register! { + /// GPU identification register. + pub(crate) GPU_ID(u32) @ 0x0 { + /// Status of the GPU release. + 3:0 ver_status; + /// Minor release version number. + 11:4 ver_minor; + /// Major release version number. + 15:12 ver_major; + /// Product identifier. + 19:16 prod_major; + /// Architecture patch revision. + 23:20 arch_rev; + /// Architecture minor revision. + 27:24 arch_minor; + /// Architecture major revision. + 31:28 arch_major; + } -pub(crate) const GPU_COHERENCY_FEATURES: Register<0x300> = Register; + /// Level 2 cache features register. + pub(crate) L2_FEATURES(u32) @ 0x4 { + /// Cache line size. + 7:0 line_size; + /// Cache associativity. + 15:8 associativity; + /// Cache slice size. + 23:16 cache_size; + /// External bus width. + 31:24 bus_width; + } + + /// Shader core features. + pub(crate) CORE_FEATURES(u32) @ 0x8 { + /// Shader core variant. + 7:0 core_variant; + } + + /// Tiler features. + pub(crate) TILER_FEATURES(u32) @ 0xc { + /// Log of the tiler's bin size. + 5:0 bin_size; + /// Maximum number of active levels. + 11:8 max_levels; + } + + /// Memory system features. + pub(crate) MEM_FEATURES(u32) @ 0x10 { + 0:0 coherent_core_group => bool; + 1:1 coherent_super_group => bool; + 11:8 l2_slices; + } + + /// Memory management unit features. + pub(crate) MMU_FEATURES(u32) @ 0x14 { + /// Number of bits supported in virtual addresses. + 7:0 va_bits; + /// Number of bits supported in physical addresses. + 15:8 pa_bits; + } + + /// Address spaces present. + pub(crate) AS_PRESENT(u32) @ 0x18 { + 31:0 present; + } + + /// CSF version information. + pub(crate) CSF_ID(u32) @ 0x1c { + /// MCU revision ID. + 3:0 mcu_rev; + /// MCU minor revision number. + 9:4 mcu_minor; + /// MCU major revision number. + 15:10 mcu_major; + /// CSHW revision ID. + 19:16 cshw_rev; + /// CSHW minor revision number. + 25:20 cshw_minor; + /// CSHW major revision number. + 31:26 cshw_major; + } + + /// IRQ sources raw status. + /// Writing to this register forces bits on, but does not clear them. + pub(crate) GPU_IRQ_RAWSTAT(u32) @ 0x20 { + /// A GPU fault has occurred, a 1-bit boolean flag. + 0:0 gpu_fault => bool; + /// A GPU fault has occurred, a 1-bit boolean flag. + 1:1 gpu_protected_fault => bool; + /// Reset has completed, a 1-bit boolean flag. + 8:8 reset_completed => bool; + /// Set when a single power domain has powered up or down, a 1-bit boolean flag. + 9:9 power_changed_single => bool; + /// Set when the all pending power domain changes are completed, a 1-bit boolean flag. + 10:10 power_changed_all => bool; + /// Set when cache cleaning has completed, a 1-bit boolean flag. + 17:17 clean_caches_completed => bool; + /// Mirrors the doorbell interrupt line to the CPU, a 1-bit boolean flag. + 18:18 doorbell_mirror => bool; + /// MCU requires attention, a 1-bit boolean flag. + 19:19 mcu_status => bool; + } + + /// IRQ sources to clear. Write only. + pub(crate) GPU_IRQ_CLEAR(u32) @ 0x24 { + /// Clear the GPU_FAULT interrupt, a 1-bit boolean flag. + 0:0 gpu_fault => bool; + /// Clear the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. + 1:1 gpu_protected_fault => bool; + /// Clear the RESET_COMPLETED interrupt, a 1-bit boolean flag. + 8:8 reset_completed => bool; + /// Clear the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. + 9:9 power_changed_single => bool; + /// Clear the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. + 10:10 power_changed_all => bool; + /// Clear the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. + 17:17 clean_caches_completed => bool; + /// Clear the MCU_STATUS interrupt, a 1-bit boolean flag. + 19:19 mcu_status => bool; + } + + /// IRQ sources enabled. + pub(crate) GPU_IRQ_MASK(u32) @ 0x28 { + /// Enable the GPU_FAULT interrupt, a 1-bit boolean flag. + 0:0 gpu_fault => bool; + /// Enable the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. + 1:1 gpu_protected_fault => bool; + /// Enable the RESET_COMPLETED interrupt, a 1-bit boolean flag. + 8:8 reset_completed => bool; + /// Enable the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. + 9:9 power_changed_single => bool; + /// Enable the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. + 10:10 power_changed_all => bool; + /// Enable the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. + 17:17 clean_caches_completed => bool; + /// Enable the DOORBELL_MIRROR interrupt, a 1-bit boolean flag. + 18:18 doorbell_mirror => bool; + /// Enable the MCU_STATUS interrupt, a 1-bit boolean flag. + 19:19 mcu_status => bool; + } + + /// IRQ status for enabled sources. Read only. + pub(crate) GPU_IRQ_STATUS(u32) @ 0x2c { + /// GPU_FAULT interrupt status, a 1-bit boolean flag. + 0:0 gpu_fault => bool; + /// GPU_PROTECTED_FAULT interrupt status, a 1-bit boolean flag. + 1:1 gpu_protected_fault => bool; + /// RESET_COMPLETED interrupt status, a 1-bit boolean flag. + 8:8 reset_completed => bool; + /// POWER_CHANGED_SINGLE interrupt status, a 1-bit boolean flag. + 9:9 power_changed_single => bool; + /// POWER_CHANGED_ALL interrupt status, a 1-bit boolean flag. + 10:10 power_changed_all => bool; + /// CLEAN_CACHES_COMPLETED interrupt status, a 1-bit boolean flag. + 17:17 clean_caches_completed => bool; + /// DOORBELL_MIRROR interrupt status, a 1-bit boolean flag. + 18:18 doorbell_mirror => bool; + /// MCU_STATUS interrupt status, a 1-bit boolean flag. + 19:19 mcu_status => bool; + } + } + + /// Helpers for GPU_COMMAND Register + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum GpuCommand { + /// No operation. This is the default value. + Nop = 0, + /// Reset the GPU. + Reset = 1, + /// Flush caches. + FlushCaches = 4, + /// Clear GPU faults. + ClearFault = 7, + } + + impl TryFrom> for GpuCommand { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(GpuCommand::Nop), + 1 => Ok(GpuCommand::Reset), + 4 => Ok(GpuCommand::FlushCaches), + 7 => Ok(GpuCommand::ClearFault), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(cmd: GpuCommand) -> Self { + (cmd as u8).into() + } + } + + /// Reset mode for [`GPU_COMMAND::reset()`]. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum ResetMode { + /// Stop all external bus interfaces, then reset the entire GPU. + SoftReset = 1, + /// Force a full GPU reset. + HardReset = 2, + } + + impl TryFrom> for ResetMode { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 1 => Ok(ResetMode::SoftReset), + 2 => Ok(ResetMode::HardReset), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(mode: ResetMode) -> Self { + Bounded::try_new(mode as u32).unwrap() + } + } + + /// Cache flush mode for [`GPU_COMMAND::flush_caches()`]. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum FlushMode { + /// No flush. + None = 0, + /// Clean the caches. + Clean = 1, + /// Invalidate the caches. + Invalidate = 2, + /// Clean and invalidate the caches. + CleanInvalidate = 3, + } + + impl TryFrom> for FlushMode { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(FlushMode::None), + 1 => Ok(FlushMode::Clean), + 2 => Ok(FlushMode::Invalidate), + 3 => Ok(FlushMode::CleanInvalidate), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(mode: FlushMode) -> Self { + Bounded::try_new(mode as u32).unwrap() + } + } + + register! { + /// GPU command register. + /// + /// Use the constructor methods to create commands: + /// - [`GPU_COMMAND::nop()`] + /// - [`GPU_COMMAND::reset()`] + /// - [`GPU_COMMAND::flush_caches()`] + /// - [`GPU_COMMAND::clear_fault()`] + pub(crate) GPU_COMMAND (u32) @ 0x30 { + 7:0 command ?=> GpuCommand; + } + /// Internal alias for GPU_COMMAND in reset mode. + /// Use [`GPU_COMMAND::reset()`] instead. + GPU_COMMAND_RESET (u32) => GPU_COMMAND { + 7:0 command ?=> GpuCommand; + 11:8 reset_mode ?=> ResetMode; + } + + /// Internal alias for GPU_COMMAND in cache flush mode. + /// Use [`GPU_COMMAND::flush_caches()`] instead. + GPU_COMMAND_FLUSH (u32) => GPU_COMMAND { + 7:0 command ?=> GpuCommand; + /// L2 cache flush mode. + 11:8 l2_flush ?=> FlushMode; + /// Shader core load/store cache flush mode. + 15:12 lsc_flush ?=> FlushMode; + /// Shader core other caches flush mode. + 19:16 other_flush ?=> FlushMode; + } + } + + impl GPU_COMMAND { + /// Create a NOP command. + pub(crate) fn nop() -> Self { + Self::zeroed() + } + + /// Create a reset command with the specified reset mode. + pub(crate) fn reset(mode: ResetMode) -> Self { + Self::from_raw( + GPU_COMMAND_RESET::zeroed() + .with_command(GpuCommand::Reset) + .with_reset_mode(mode) + .into_raw(), + ) + } + + /// Create a cache flush command with the specified flush modes. + pub(crate) fn flush_caches(l2: FlushMode, lsc: FlushMode, other: FlushMode) -> Self { + Self::from_raw( + GPU_COMMAND_FLUSH::zeroed() + .with_command(GpuCommand::FlushCaches) + .with_l2_flush(l2) + .with_lsc_flush(lsc) + .with_other_flush(other) + .into_raw(), + ) + } + + /// Create a clear fault command. + pub(crate) fn clear_fault() -> Self { + Self::zeroed().with_command(GpuCommand::ClearFault) + } + } + + register! { + /// GPU status register. Read only. + pub(crate) GPU_STATUS(u32) @ 0x34 { + /// GPU active, a 1-bit boolean flag. + 0:0 gpu_active => bool; + /// Power manager active, a 1-bit boolean flag + 1:1 pwr_active => bool; + /// Page fault active, a 1-bit boolean flag. + 4:4 page_fault => bool; + /// Protected mode active, a 1-bit boolean flag. + 7:7 protected_mode_active => bool; + /// Debug mode active, a 1-bit boolean flag. + 8:8 gpu_dbg_enabled => bool; + } + } + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum ExceptionType { + /// Exception type: No error. + Ok = 0x00, + /// Exception type: GPU external bus error. + GpuBusFault = 0x80, + /// Exception type: GPU shareability error. + GpuShareabilityFault = 0x88, + /// Exception type: System shareability error. + SystemShareabilityFault = 0x89, + /// Exception type: GPU cacheability error. + GpuCacheabilityFault = 0x8A, + } + + impl TryFrom> for ExceptionType { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0x00 => Ok(ExceptionType::Ok), + 0x80 => Ok(ExceptionType::GpuBusFault), + 0x88 => Ok(ExceptionType::GpuShareabilityFault), + 0x89 => Ok(ExceptionType::SystemShareabilityFault), + 0x8A => Ok(ExceptionType::GpuCacheabilityFault), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(exc: ExceptionType) -> Self { + (exc as u8).into() + } + } + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum AccessType { + /// Access type: An atomic (read/write) transaction. + Atomic = 0, + /// Access type: An execute transaction. + Execute = 1, + /// Access type: A read transaction. + Read = 2, + /// Access type: A write transaction. + Write = 3, + } + + impl From> for AccessType { + fn from(val: Bounded) -> Self { + match val.get() { + 0 => AccessType::Atomic, + 1 => AccessType::Execute, + 2 => AccessType::Read, + 3 => AccessType::Write, + _ => unreachable!(), + } + } + } + + impl From for Bounded { + fn from(access: AccessType) -> Self { + Bounded::try_new(access as u32).unwrap() + } + } + + register! { + /// GPU fault status register. Read only. + pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c { + /// Exception type. + 7:0 exception_type ?=> ExceptionType; + /// Access type. + 9:8 access_type => AccessType; + /// The GPU_FAULTADDRESS is valid, a 1-bit boolean flag. + 10:10 address_valid => bool; + /// The JASID field is valid, a 1-bit boolean flag. + 11:11 jasid_valid => bool; + /// JASID of the fault, if known. + 15:12 jasid; + /// ID of the source that triggered the fault. + 31:16 source_id; + } + + /// GPU fault address. Read only. + /// Once a fault is reported, it must be manually cleared by issuing a + /// [`GPU_COMMAND::clear_fault()`] command to the [`GPU_COMMAND`] register. No further GPU + /// faults will be reported until the previous fault has been cleared. + pub(crate) GPU_FAULTADDRESS_LO(u32) @ 0x40 { + 31:0 pointer; + } + + pub(crate) GPU_FAULTADDRESS_HI(u32) @ 0x44 { + 31:0 pointer; + } + + /// Level 2 cache configuration. + pub(crate) L2_CONFIG(u32) @ 0x48 { + /// Requested cache size. + 23:16 cache_size; + /// Requested hash function index. + 31:24 hash_function; + } + + /// Global time stamp offset. + pub(crate) TIMESTAMP_OFFSET_LO(u32) @ 0x88 { + 31:0 offset; + } + + pub(crate) TIMESTAMP_OFFSET_HI(u32) @ 0x8c { + 31:0 offset; + } + + /// GPU cycle counter. Read only. + pub(crate) CYCLE_COUNT_LO(u32) @ 0x90 { + 31:0 count; + } + + pub(crate) CYCLE_COUNT_HI(u32) @ 0x94 { + 31:0 count; + } + + /// Global time stamp. Read only. + pub(crate) TIMESTAMP_LO(u32) @ 0x98 { + 31:0 timestamp; + } + + pub(crate) TIMESTAMP_HI(u32) @ 0x9c { + 31:0 timestamp; + } + + /// Maximum number of threads per core. Read only constant. + pub(crate) THREAD_MAX_THREADS(u32) @ 0xa0 { + 31:0 threads; + } + + /// Maximum number of threads per workgroup. Read only constant. + pub(crate) THREAD_MAX_WORKGROUP_SIZE(u32) @ 0xa4 { + 31:0 threads; + } + + /// Maximum number of threads per barrier. Read only constant. + pub(crate) THREAD_MAX_BARRIER_SIZE(u32) @ 0xa8 { + 31:0 threads; + } + + /// Thread features. Read only constant. + pub(crate) THREAD_FEATURES(u32) @ 0xac { + /// Total number of registers per core. + 21:0 max_registers; + /// Implementation technology type. + 23:22 implementation_technology; + /// Maximum number of compute tasks waiting. + 31:24 max_task_queue; + } + + /// Support flags for compressed texture formats. Read only constant. + /// + /// A bitmap where each bit indicates support for a specific compressed texture format. + /// The bit position maps to an opaque format ID (`texture_features_key_t` in spec). + pub(crate) TEXTURE_FEATURES(u32)[4] @ 0xb0 { + 31:0 supported_formats; + } + + /// Shader core present bitmap. Read only constant. + pub(crate) SHADER_PRESENT_LO(u32) @ 0x100 { + 31:0 value; + } + + pub(crate) SHADER_PRESENT_HI(u32) @ 0x104 { + 31:0 value; + } + + /// Tiler present bitmap. Read only constant. + pub(crate) TILER_PRESENT_LO(u32) @ 0x110 { + 31:0 present; + } + + pub(crate) TILER_PRESENT_HI(u32) @ 0x114 { + 31:0 present; + } + + /// L2 cache present bitmap. Read only constant. + pub(crate) L2_PRESENT_LO(u32) @ 0x120 { + 31:0 present; + } + + pub(crate) L2_PRESENT_HI(u32) @ 0x124 { + 31:0 present; + } + + /// Shader core ready bitmap. Read only. + pub(crate) SHADER_READY_LO(u32) @ 0x140 { + 31:0 ready; + } + + pub(crate) SHADER_READY_HI(u32) @ 0x144 { + 31:0 ready; + } + + /// Tiler ready bitmap. Read only. + pub(crate) TILER_READY_LO(u32) @ 0x150 { + 31:0 ready; + } + + pub(crate) TILER_READY_HI(u32) @ 0x154 { + 31:0 ready; + } + + /// L2 ready bitmap. Read only. + pub(crate) L2_READY_LO(u32) @ 0x160 { + 31:0 ready; + } + + pub(crate) L2_READY_HI(u32) @ 0x164 { + 31:0 ready; + } + + /// Shader core power up bitmap. + pub(crate) SHADER_PWRON_LO(u32) @ 0x180 { + 31:0 request; + } + + pub(crate) SHADER_PWRON_HI(u32) @ 0x184 { + 31:0 request; + } + + /// Tiler power up bitmap. + pub(crate) TILER_PWRON_LO(u32) @ 0x190 { + 31:0 request; + } + + pub(crate) TILER_PWRON_HI(u32) @ 0x194 { + 31:0 request; + } + + /// L2 power up bitmap. + pub(crate) L2_PWRON_LO(u32) @ 0x1a0 { + 31:0 request; + } + + pub(crate) L2_PWRON_HI(u32) @ 0x1a4 { + 31:0 request; + } + + /// Shader core power down bitmap. + pub(crate) SHADER_PWROFF_LO(u32) @ 0x1c0 { + 31:0 request; + } + + pub(crate) SHADER_PWROFF_HI(u32) @ 0x1c4 { + 31:0 request; + } + + /// Tiler power down bitmap. + pub(crate) TILER_PWROFF_LO(u32) @ 0x1d0 { + 31:0 request; + } + + pub(crate) TILER_PWROFF_HI(u32) @ 0x1d4 { + 31:0 request; + } + + /// L2 power down bitmap. + pub(crate) L2_PWROFF_LO(u32) @ 0x1e0 { + 31:0 request; + } + + pub(crate) L2_PWROFF_HI(u32) @ 0x1e4 { + 31:0 request; + } + + /// Shader core power transition bitmap. Read-only. + pub(crate) SHADER_PWRTRANS_LO(u32) @ 0x200 { + 31:0 changing; + } + + pub(crate) SHADER_PWRTRANS_HI(u32) @ 0x204 { + 31:0 changing; + } + + /// Tiler power transition bitmap. Read-only. + pub(crate) TILER_PWRTRANS_LO(u32) @ 0x210 { + 31:0 changing; + } + + pub(crate) TILER_PWRTRANS_HI(u32) @ 0x214 { + 31:0 changing; + } + + /// L2 power transition bitmap. Read-only. + pub(crate) L2_PWRTRANS_LO(u32) @ 0x220 { + 31:0 changing; + } + + pub(crate) L2_PWRTRANS_HI(u32) @ 0x224 { + 31:0 changing; + } + + /// Shader core active bitmap. Read-only. + pub(crate) SHADER_PWRACTIVE_LO(u32) @ 0x240 { + 31:0 active; + } + + pub(crate) SHADER_PWRACTIVE_HI(u32) @ 0x244 { + 31:0 active; + } + + /// Tiler active bitmap. Read-only. + pub(crate) TILER_PWRACTIVE_LO(u32) @ 0x250 { + 31:0 active; + } + + pub(crate) TILER_PWRACTIVE_HI(u32) @ 0x254 { + 31:0 active; + } + + /// L2 active bitmap. Read-only. + pub(crate) L2_PWRACTIVE_LO(u32) @ 0x260 { + 31:0 active; + } + + pub(crate) L2_PWRACTIVE_HI(u32) @ 0x264 { + 31:0 active; + } + + /// Revision ID. Read only constant. + pub(crate) REVIDR(u32) @ 0x280 { + 31:0 revision; + } + + /// Coherency features present. Read only constant. + /// Supported protocols on the interconnect between the GPU and the + /// system into which it is integrated. + pub(crate) COHERENCY_FEATURES(u32) @ 0x300 { + /// ACE-Lite protocol supported, a 1-bit boolean flag. + 0:0 ace_lite => bool; + /// ACE protocol supported, a 1-bit boolean flag. + 1:1 ace => bool; + } + } + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum CoherencyMode { + /// ACE-Lite coherency protocol. + AceLite = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE_LITE as u8, + /// ACE coherency protocol. + Ace = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE as u8, + /// No coherency protocol. + None = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE as u8, + } + + impl TryFrom> for CoherencyMode { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(CoherencyMode::AceLite), + 1 => Ok(CoherencyMode::Ace), + 31 => Ok(CoherencyMode::None), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(mode: CoherencyMode) -> Self { + (mode as u8).into() + } + } + + register! { + /// Coherency enable. An index of which coherency protocols should be used. + /// This register only selects the protocol for coherency messages on the + /// interconnect. This is not to enable or disable coherency controlled by MMU. + pub(crate) COHERENCY_ENABLE(u32) @ 0x304 { + 31:0 l2_cache_protocol_select ?=> CoherencyMode; + } + } + + /// Helpers for MCU_CONTROL register + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum McuControlMode { + /// Disable the MCU. + Disable = 0, + /// Enable the MCU. + Enable = 1, + /// Enable the MCU to execute and automatically reboot after a fast reset. + Auto = 2, + } + + impl TryFrom> for McuControlMode { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(McuControlMode::Disable), + 1 => Ok(McuControlMode::Enable), + 2 => Ok(McuControlMode::Auto), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(mode: McuControlMode) -> Self { + Bounded::try_new(mode as u32).unwrap() + } + } + + register! { + /// MCU control. + pub(crate) MCU_CONTROL(u32) @ 0x700 { + /// Request MCU state change. + 1:0 req ?=> McuControlMode; + } + } + + /// Helpers for MCU_STATUS register + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum McuStatus { + /// MCU is disabled. + Disabled = 0, + /// MCU is enabled. + Enabled = 1, + /// The MCU has halted by itself in an orderly manner to enable the core group to be + /// powered down. + Halt = 2, + /// The MCU has encountered an error that prevents it from continuing. + Fatal = 3, + } + + impl From> for McuStatus { + fn from(val: Bounded) -> Self { + match val.get() { + 0 => McuStatus::Disabled, + 1 => McuStatus::Enabled, + 2 => McuStatus::Halt, + 3 => McuStatus::Fatal, + _ => unreachable!(), + } + } + } + + impl From for Bounded { + fn from(status: McuStatus) -> Self { + Bounded::try_new(status as u32).unwrap() + } + } + + register! { + /// MCU status. Read only. + pub(crate) MCU_STATUS(u32) @ 0x704 { + /// Read current state of MCU. + 1:0 value => McuStatus; + } + } +} pub(crate) const JOB_IRQ_RAWSTAT: Register<0x1000> = Register; pub(crate) const JOB_IRQ_CLEAR: Register<0x1004> = Register; From b00d6a73981ee4fb289e89ab649ce61a8df54cd9 Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Thu, 9 Apr 2026 10:51:25 -0700 Subject: [PATCH 003/131] drm/tyr: Print GPU_ID without filtering Currently, Tyr prints just the upper 16 bits of the GPU_ID in the hex id field, namely ARCH_MAJOR, ARCH_MINOR, ARCH_REV, and PRODUCT_MAJOR. The VERSION_* fields are already printed separately as "major", "minor", and "status". Print the full 32-bit GPU_ID register instead of shifting it, so the hex id reflects the complete register contents. Before this change: mali-g610 id 0xa867 major 0x0 minor 0x0 status 0x5 After this change: mali-g610 GPU_ID 0xa8670005 major 0x0 minor 0x0 status 0x5 Reviewed-by: Boris Brezillon Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260409-b4-tyr-use-register-macro-v5-v5-2-8abfff8a0204@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/gpu.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/tyr/gpu.rs b/drivers/gpu/drm/tyr/gpu.rs index 1e1e4103c575..652556026f50 100644 --- a/drivers/gpu/drm/tyr/gpu.rs +++ b/drivers/gpu/drm/tyr/gpu.rs @@ -97,9 +97,9 @@ pub(crate) fn log(&self, dev: &Device) { dev_info!( dev, - "mali-{} id 0x{:x} major 0x{:x} minor 0x{:x} status 0x{:x}", + "mali-{} GPU_ID 0x{:x} major 0x{:x} minor 0x{:x} status 0x{:x}", model_name, - gpu_id.into_raw() >> 16, + gpu_id.into_raw(), gpu_id.ver_major().get(), gpu_id.ver_minor().get(), gpu_id.ver_status().get() From 566eec206a61604f0b84a858b45265dbef2067e1 Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Thu, 9 Apr 2026 10:51:26 -0700 Subject: [PATCH 004/131] drm/tyr: Use register! macro for JOB_CONTROL Define the JOB_CONTROL register block with the kernel's register! macro and replace the existing hand-written JOB IRQ register definitions with typed register and field accessors. This adds typed definitions for the raw status, clear, mask, and status registers, including the per-CSG interrupt bits and the global interface interrupt bit. This reduces open-coded bit manipulation, keeps the JOB_CONTROL register layout in one place, and makes the definitions easier to read and maintain. Reviewed-by: Boris Brezillon Co-developed-by: Daniel Almeida Signed-off-by: Daniel Almeida Reviewed-by: Daniel Almeida Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260409-b4-tyr-use-register-macro-v5-v5-3-8abfff8a0204@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/regs.rs | 58 ++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index 3b246ade6096..93bc8908a233 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -28,7 +28,6 @@ #![allow(dead_code)] use kernel::{ - bits::bit_u32, device::{ Bound, Device, // @@ -894,14 +893,57 @@ fn from(status: McuStatus) -> Self { } } -pub(crate) const JOB_IRQ_RAWSTAT: Register<0x1000> = Register; -pub(crate) const JOB_IRQ_CLEAR: Register<0x1004> = Register; -pub(crate) const JOB_IRQ_MASK: Register<0x1008> = Register; -pub(crate) const JOB_IRQ_STAT: Register<0x100c> = Register; - -pub(crate) const JOB_IRQ_GLOBAL_IF: u32 = bit_u32(31); - pub(crate) const MMU_IRQ_RAWSTAT: Register<0x2000> = Register; pub(crate) const MMU_IRQ_CLEAR: Register<0x2004> = Register; pub(crate) const MMU_IRQ_MASK: Register<0x2008> = Register; pub(crate) const MMU_IRQ_STAT: Register<0x200c> = Register; + +/// These registers correspond to the JOB_CONTROL register page. +/// They are involved in communication between the firmware running on the MCU and the host. +pub(crate) mod job_control { + use kernel::register; + + register! { + /// Raw status of job interrupts. + /// + /// Write to this register to trigger these interrupts. + /// Writing a 1 to a bit forces that bit on. + pub(crate) JOB_IRQ_RAWSTAT(u32) @ 0x1000 { + /// CSG request. These bits indicate that CSGn requires attention from the host. + 30:0 csg; + /// GLB request. Indicates that the GLB interface requires attention from the host. + 31:31 glb => bool; + } + + /// Clear job interrupts. Write only. + /// + /// Write a 1 to a bit to clear the corresponding bit in [`JOB_IRQ_RAWSTAT`]. + pub(crate) JOB_IRQ_CLEAR(u32) @ 0x1004 { + /// Clear CSG request interrupts. + 30:0 csg; + /// Clear GLB request interrupt. + 31:31 glb => bool; + } + + /// Mask for job interrupts. + /// + /// Set each bit to 1 to enable the corresponding interrupt source or to 0 to disable it. + pub(crate) JOB_IRQ_MASK(u32) @ 0x1008 { + /// Enable CSG request interrupts. + 30:0 csg; + /// Enable GLB request interrupt. + 31:31 glb => bool; + } + + /// Active job interrupts. Read only. + /// + /// This register contains the result of ANDing together [`JOB_IRQ_RAWSTAT`] and + /// [`JOB_IRQ_MASK`]. + pub(crate) JOB_IRQ_STATUS(u32) @ 0x100c { + /// CSG request interrupt status. + 30:0 csg; + /// GLB request interrupt status. + 31:31 glb => bool; + } + } +} From bc8dd92ec67e87244b434304ee25eef0ff70da6e Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Thu, 9 Apr 2026 10:51:27 -0700 Subject: [PATCH 005/131] drm/tyr: Use register! macro for MMU_CONTROL Define the MMU_CONTROL register block with the kernel's register! macro and replace the existing hand-written MMU register definitions with typed register and field accessors. This adds typed definitions for the MMU IRQ registers and the per-address space MMU_AS_CONTROL registers, including TRANSTAB, MEMATTR, LOCKADDR, COMMAND, FAULTSTATUS, STATUS, and TRANSCFG. It also introduces typed enums for MMU commands, fault types, access types, address space modes, memory attributes, and related MMU configuration fields. For logical 64-bit MMU registers that are exposed as split 32-bit MMIO registers, define both the typed 64-bit view and explicit low/high 32-bit registers so the register layout remains documented without relying on native 64-bit MMIO accesses. This reduces open-coded bit manipulation, keeps MMU register layout knowledge in one place, and makes the definitions easier to read and maintain. Reviewed-by: Boris Brezillon Co-developed-by: Daniel Almeida Signed-off-by: Daniel Almeida Reviewed-by: Daniel Almeida Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260409-b4-tyr-use-register-macro-v5-v5-4-8abfff8a0204@collabora.com [aliceryhl: reformat long comment] Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/regs.rs | 728 +++++++++++++++++++++++++++++++++++- 1 file changed, 723 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index 93bc8908a233..725e9f191ded 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -893,11 +893,6 @@ fn from(status: McuStatus) -> Self { } } -pub(crate) const MMU_IRQ_RAWSTAT: Register<0x2000> = Register; -pub(crate) const MMU_IRQ_CLEAR: Register<0x2004> = Register; -pub(crate) const MMU_IRQ_MASK: Register<0x2008> = Register; -pub(crate) const MMU_IRQ_STAT: Register<0x200c> = Register; - /// These registers correspond to the JOB_CONTROL register page. /// They are involved in communication between the firmware running on the MCU and the host. pub(crate) mod job_control { @@ -947,3 +942,726 @@ pub(crate) mod job_control { } } } + +/// These registers correspond to the MMU_CONTROL register page. +/// They are involved in MMU configuration and control. +pub(crate) mod mmu_control { + use kernel::register; + + register! { + /// IRQ sources raw status. + /// + /// This register contains the raw unmasked interrupt sources for MMU status and exception + /// handling. + /// + /// Writing to this register forces bits on. + /// Use [`IRQ_CLEAR`] to clear interrupts. + pub(crate) IRQ_RAWSTAT(u32) @ 0x2000 { + /// Page fault for address spaces. + 15:0 page_fault; + /// Command completed in address spaces. + 31:16 command_completed; + } + + /// IRQ sources to clear. + /// Write a 1 to a bit to clear the corresponding bit in [`IRQ_RAWSTAT`]. + pub(crate) IRQ_CLEAR(u32) @ 0x2004 { + /// Clear the PAGE_FAULT interrupt. + 15:0 page_fault; + /// Clear the COMMAND_COMPLETED interrupt. + 31:16 command_completed; + } + + /// IRQ sources enabled. + /// + /// Set each bit to 1 to enable the corresponding interrupt source, and to 0 to disable it. + pub(crate) IRQ_MASK(u32) @ 0x2008 { + /// Enable the PAGE_FAULT interrupt. + 15:0 page_fault; + /// Enable the COMMAND_COMPLETED interrupt. + 31:16 command_completed; + } + + /// IRQ status for enabled sources. Read only. + /// + /// This register contains the result of ANDing together [`IRQ_RAWSTAT`] and [`IRQ_MASK`]. + pub(crate) IRQ_STATUS(u32) @ 0x200c { + /// PAGE_FAULT interrupt status. + 15:0 page_fault; + /// COMMAND_COMPLETED interrupt status. + 31:16 command_completed; + } + } + + /// Per-address space registers ASn [0..15] within the MMU_CONTROL page. + /// + /// This array contains 16 instances of the MMU_AS_CONTROL register page. + pub(crate) mod mmu_as_control { + use core::convert::TryFrom; + + use kernel::{ + error::{ + code::EINVAL, + Error, // + }, + num::Bounded, + register, // + }; + + /// Maximum number of hardware address space slots. + /// The actual number of slots available is usually lower. + pub(crate) const MAX_AS: usize = 16; + + /// Address space register stride. The elements in the array are spaced 64B apart. + const STRIDE: usize = 0x40; + + register! { + /// Translation table base address. A 64-bit pointer. + /// + /// This field contains the address of the top level of a translation table structure. + /// This must be 16-byte-aligned, so address bits [3:0] are assumed to be zero. + pub(crate) TRANSTAB(u64)[MAX_AS, stride = STRIDE] @ 0x2400 { + /// Base address of the translation table. + 63:0 base; + } + + // TRANSTAB is a logical 64-bit register, but it is laid out in hardware as two + // 32-bit halves. Define it as separate low/high u32 registers so accesses match + // the MMIO register layout and do not rely on native 64-bit MMIO transactions. + pub(crate) TRANSTAB_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2400 { + 31:0 value; + } + + pub(crate) TRANSTAB_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2404 { + 31:0 value; + } + } + + /// Helpers for MEMATTR Register. + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum AllocPolicySelect { + /// Ignore ALLOC_R/ALLOC_W fields. + Impl = 2, + /// Use ALLOC_R/ALLOC_W fields for allocation policy. + Alloc = 3, + } + + impl TryFrom> for AllocPolicySelect { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 2 => Ok(Self::Impl), + 3 => Ok(Self::Alloc), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(val: AllocPolicySelect) -> Self { + Bounded::try_new(val as u8).unwrap() + } + } + + /// Coherency policy for memory attributes. Indicates the shareability of cached accesses. + /// + /// The hardware spec defines different interpretations of these values depending on + /// whether TRANSCFG.MODE is set to IDENTITY or not. IDENTITY mode does not use translation + /// tables (all input addresses map to the same output address); it is deprecated and not + /// used by the driver. This enum assumes that TRANSCFG.MODE is not set to IDENTITY. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum Coherency { + /// Midgard inner domain coherency. + /// + /// Most flexible mode - can map non-coherent, internally coherent, and system/IO + /// coherent memory. Used for non-cacheable memory in MAIR conversion. + MidgardInnerDomain = 0, + /// CPU inner domain coherency. + /// + /// Can map non-coherent and system/IO coherent memory. Used for write-back + /// cacheable memory in MAIR conversion to maintain CPU-GPU cache coherency. + CpuInnerDomain = 1, + /// CPU inner domain with shader coherency. + /// + /// Can map internally coherent and system/IO coherent memory. Used for + /// GPU-internal shared buffers requiring shader coherency. + CpuInnerDomainShaderCoh = 2, + } + + impl TryFrom> for Coherency { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(Self::MidgardInnerDomain), + 1 => Ok(Self::CpuInnerDomain), + 2 => Ok(Self::CpuInnerDomainShaderCoh), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(val: Coherency) -> Self { + Bounded::try_new(val as u8).unwrap() + } + } + + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum MemoryType { + /// Normal memory (shared). + Shared = 0, + /// Normal memory, inner/outer non-cacheable. + NonCacheable = 1, + /// Normal memory, inner/outer write-back cacheable. + WriteBack = 2, + /// Triggers MEMORY_ATTRIBUTE_FAULT. + Fault = 3, + } + + impl From> for MemoryType { + fn from(val: Bounded) -> Self { + match val.get() { + 0 => Self::Shared, + 1 => Self::NonCacheable, + 2 => Self::WriteBack, + 3 => Self::Fault, + _ => unreachable!(), + } + } + } + + impl From for Bounded { + fn from(val: MemoryType) -> Self { + Bounded::try_new(val as u8).unwrap() + } + } + + register! { + /// Stage 1 memory attributes (8-bit bitfield). + /// + /// This is not an actual register, but a bitfield definition used by the MEMATTR + /// register. Each of the 8 bytes in MEMATTR follows this layout. + MMU_MEMATTR_STAGE1(u8) @ 0x0 { + /// Inner cache write allocation policy. + 0:0 alloc_w => bool; + /// Inner cache read allocation policy. + 1:1 alloc_r => bool; + /// Inner allocation policy select. + 3:2 alloc_sel ?=> AllocPolicySelect; + /// Coherency policy. + 5:4 coherency ?=> Coherency; + /// Memory type. + 7:6 memory_type => MemoryType; + } + } + + impl TryFrom> for MMU_MEMATTR_STAGE1 { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + Ok(Self::from_raw(val.get() as u8)) + } + } + + impl From for Bounded { + fn from(val: MMU_MEMATTR_STAGE1) -> Self { + Bounded::try_new(u64::from(val.into_raw())).unwrap() + } + } + + register! { + /// Memory attributes. + /// + /// Each address space can configure up to 8 different memory attribute profiles. + /// Each attribute profile follows the MMU_MEMATTR_STAGE1 layout. + pub(crate) MEMATTR(u64)[MAX_AS, stride = STRIDE] @ 0x2408 { + 7:0 attribute0 ?=> MMU_MEMATTR_STAGE1; + 15:8 attribute1 ?=> MMU_MEMATTR_STAGE1; + 23:16 attribute2 ?=> MMU_MEMATTR_STAGE1; + 31:24 attribute3 ?=> MMU_MEMATTR_STAGE1; + 39:32 attribute4 ?=> MMU_MEMATTR_STAGE1; + 47:40 attribute5 ?=> MMU_MEMATTR_STAGE1; + 55:48 attribute6 ?=> MMU_MEMATTR_STAGE1; + 63:56 attribute7 ?=> MMU_MEMATTR_STAGE1; + } + + // MEMATTR is a logical 64-bit register, but it is laid out in hardware as two + // 32-bit halves. Define it as separate low/high u32 registers so accesses match + // the MMIO register layout and do not rely on native 64-bit MMIO transactions. + pub(crate) MEMATTR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2408 { + 31:0 value; + } + + pub(crate) MEMATTR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x240c { + 31:0 value; + } + + /// Lock region address for each address space. + pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 { + /// Lock region size. + 5:0 size; + /// Lock region base address. + 63:12 base; + } + + // LOCKADDR is a logical 64-bit register, but it is laid out in hardware as two + // 32-bit halves. Define it as separate low/high u32 registers so accesses match + // the MMIO register layout and do not rely on native 64-bit MMIO transactions. + pub(crate) LOCKADDR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2410 { + 31:0 value; + } + + pub(crate) LOCKADDR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2414 { + 31:0 value; + } + } + + /// Helpers for MMU COMMAND register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum MmuCommand { + /// No operation, nothing happens. + Nop = 0, + /// Propagate settings to the MMU. + Update = 1, + /// Lock an address region. + Lock = 2, + /// Unlock an address region. + Unlock = 3, + /// Clean and invalidate the L2 cache, then unlock. + FlushPt = 4, + /// Clean and invalidate all caches, then unlock. + FlushMem = 5, + } + + impl TryFrom> for MmuCommand { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(MmuCommand::Nop), + 1 => Ok(MmuCommand::Update), + 2 => Ok(MmuCommand::Lock), + 3 => Ok(MmuCommand::Unlock), + 4 => Ok(MmuCommand::FlushPt), + 5 => Ok(MmuCommand::FlushMem), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(cmd: MmuCommand) -> Self { + (cmd as u8).into() + } + } + + register! { + /// MMU command register for each address space. Write only. + pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 { + 7:0 command ?=> MmuCommand; + } + } + + /// MMU exception types for FAULTSTATUS register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum MmuExceptionType { + /// No error. + Ok = 0x00, + /// Invalid translation table entry, level 0. + TranslationFault0 = 0xC0, + /// Invalid translation table entry, level 1. + TranslationFault1 = 0xC1, + /// Invalid translation table entry, level 2. + TranslationFault2 = 0xC2, + /// Invalid translation table entry, level 3. + TranslationFault3 = 0xC3, + /// Invalid block descriptor. + TranslationFault4 = 0xC4, + /// Page permission error, level 0. + PermissionFault0 = 0xC8, + /// Page permission error, level 1. + PermissionFault1 = 0xC9, + /// Page permission error, level 2. + PermissionFault2 = 0xCA, + /// Page permission error, level 3. + PermissionFault3 = 0xCB, + /// Access flag not set, level 1. + AccessFlag1 = 0xD9, + /// Access flag not set, level 2. + AccessFlag2 = 0xDA, + /// Access flag not set, level 3. + AccessFlag3 = 0xDB, + /// Virtual address out of range. + AddressSizeFaultIn = 0xE0, + /// Physical address out of range, level 0. + AddressSizeFaultOut0 = 0xE4, + /// Physical address out of range, level 1. + AddressSizeFaultOut1 = 0xE5, + /// Physical address out of range, level 2. + AddressSizeFaultOut2 = 0xE6, + /// Physical address out of range, level 3. + AddressSizeFaultOut3 = 0xE7, + /// Page attribute error, level 0. + MemoryAttributeFault0 = 0xE8, + /// Page attribute error, level 1. + MemoryAttributeFault1 = 0xE9, + /// Page attribute error, level 2. + MemoryAttributeFault2 = 0xEA, + /// Page attribute error, level 3. + MemoryAttributeFault3 = 0xEB, + } + + impl TryFrom> for MmuExceptionType { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0x00 => Ok(MmuExceptionType::Ok), + 0xC0 => Ok(MmuExceptionType::TranslationFault0), + 0xC1 => Ok(MmuExceptionType::TranslationFault1), + 0xC2 => Ok(MmuExceptionType::TranslationFault2), + 0xC3 => Ok(MmuExceptionType::TranslationFault3), + 0xC4 => Ok(MmuExceptionType::TranslationFault4), + 0xC8 => Ok(MmuExceptionType::PermissionFault0), + 0xC9 => Ok(MmuExceptionType::PermissionFault1), + 0xCA => Ok(MmuExceptionType::PermissionFault2), + 0xCB => Ok(MmuExceptionType::PermissionFault3), + 0xD9 => Ok(MmuExceptionType::AccessFlag1), + 0xDA => Ok(MmuExceptionType::AccessFlag2), + 0xDB => Ok(MmuExceptionType::AccessFlag3), + 0xE0 => Ok(MmuExceptionType::AddressSizeFaultIn), + 0xE4 => Ok(MmuExceptionType::AddressSizeFaultOut0), + 0xE5 => Ok(MmuExceptionType::AddressSizeFaultOut1), + 0xE6 => Ok(MmuExceptionType::AddressSizeFaultOut2), + 0xE7 => Ok(MmuExceptionType::AddressSizeFaultOut3), + 0xE8 => Ok(MmuExceptionType::MemoryAttributeFault0), + 0xE9 => Ok(MmuExceptionType::MemoryAttributeFault1), + 0xEA => Ok(MmuExceptionType::MemoryAttributeFault2), + 0xEB => Ok(MmuExceptionType::MemoryAttributeFault3), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(exc: MmuExceptionType) -> Self { + (exc as u8).into() + } + } + + /// Access type for MMU faults. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum MmuAccessType { + /// An atomic (read/write) transaction. + Atomic = 0, + /// An execute transaction. + Execute = 1, + /// A read transaction. + Read = 2, + /// A write transaction. + Write = 3, + } + + impl From> for MmuAccessType { + fn from(val: Bounded) -> Self { + match val.get() { + 0 => MmuAccessType::Atomic, + 1 => MmuAccessType::Execute, + 2 => MmuAccessType::Read, + 3 => MmuAccessType::Write, + _ => unreachable!(), + } + } + } + + impl From for Bounded { + fn from(access: MmuAccessType) -> Self { + Bounded::try_new(access as u32).unwrap() + } + } + + register! { + /// Fault status register for each address space. Read only. + pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c { + /// Exception type. + 7:0 exception_type ?=> MmuExceptionType; + /// Access type. + 9:8 access_type => MmuAccessType; + /// ID of the source that triggered the fault. + 31:16 source_id; + } + + /// Fault address for each address space. Read only. + pub(crate) FAULTADDRESS_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2420 { + 31:0 pointer; + } + + pub(crate) FAULTADDRESS_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2424 { + 31:0 pointer; + } + + /// MMU status register for each address space. Read only. + pub(crate) STATUS(u32)[MAX_AS, stride = STRIDE] @ 0x2428 { + /// External address space command is active, a 1-bit boolean flag. + 0:0 active_ext => bool; + /// Internal address space command is active, a 1-bit boolean flag. + 1:1 active_int => bool; + } + } + + /// Helpers for TRANSCFG register. + /// + /// Address space mode for TRANSCFG register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum AddressSpaceMode { + /// The MMU forces all memory access to fail with a decode fault. + Unmapped = 1, + /// All input addresses map to the same output address (deprecated). + Identity = 2, + /// Translation tables interpreted according to AArch64 4kB granule specification. + Aarch64_4K = 6, + /// Translation tables interpreted according to AArch64 64kB granule specification. + Aarch64_64K = 8, + } + + impl TryFrom> for AddressSpaceMode { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 1 => Ok(AddressSpaceMode::Unmapped), + 2 => Ok(AddressSpaceMode::Identity), + 6 => Ok(AddressSpaceMode::Aarch64_4K), + 8 => Ok(AddressSpaceMode::Aarch64_64K), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(mode: AddressSpaceMode) -> Self { + Bounded::try_new(mode as u64).unwrap() + } + } + + /// Input address range restriction for TRANSCFG register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum InaBits { + /// Invalid VA range (reset value). + Reset = 0, + /// 48-bit VA range. + Bits48 = 7, + /// 47-bit VA range. + Bits47 = 8, + /// 46-bit VA range. + Bits46 = 9, + /// 45-bit VA range. + Bits45 = 10, + /// 44-bit VA range. + Bits44 = 11, + /// 43-bit VA range. + Bits43 = 12, + /// 42-bit VA range. + Bits42 = 13, + /// 41-bit VA range. + Bits41 = 14, + /// 40-bit VA range. + Bits40 = 15, + /// 39-bit VA range. + Bits39 = 16, + /// 38-bit VA range. + Bits38 = 17, + /// 37-bit VA range. + Bits37 = 18, + /// 36-bit VA range. + Bits36 = 19, + /// 35-bit VA range. + Bits35 = 20, + /// 34-bit VA range. + Bits34 = 21, + /// 33-bit VA range. + Bits33 = 22, + /// 32-bit VA range. + Bits32 = 23, + /// 31-bit VA range. + Bits31 = 24, + /// 30-bit VA range. + Bits30 = 25, + /// 29-bit VA range. + Bits29 = 26, + /// 28-bit VA range. + Bits28 = 27, + /// 27-bit VA range. + Bits27 = 28, + /// 26-bit VA range. + Bits26 = 29, + /// 25-bit VA range. + Bits25 = 30, + } + + impl TryFrom> for InaBits { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(InaBits::Reset), + 7 => Ok(InaBits::Bits48), + 8 => Ok(InaBits::Bits47), + 9 => Ok(InaBits::Bits46), + 10 => Ok(InaBits::Bits45), + 11 => Ok(InaBits::Bits44), + 12 => Ok(InaBits::Bits43), + 13 => Ok(InaBits::Bits42), + 14 => Ok(InaBits::Bits41), + 15 => Ok(InaBits::Bits40), + 16 => Ok(InaBits::Bits39), + 17 => Ok(InaBits::Bits38), + 18 => Ok(InaBits::Bits37), + 19 => Ok(InaBits::Bits36), + 20 => Ok(InaBits::Bits35), + 21 => Ok(InaBits::Bits34), + 22 => Ok(InaBits::Bits33), + 23 => Ok(InaBits::Bits32), + 24 => Ok(InaBits::Bits31), + 25 => Ok(InaBits::Bits30), + 26 => Ok(InaBits::Bits29), + 27 => Ok(InaBits::Bits28), + 28 => Ok(InaBits::Bits27), + 29 => Ok(InaBits::Bits26), + 30 => Ok(InaBits::Bits25), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(bits: InaBits) -> Self { + Bounded::try_new(bits as u64).unwrap() + } + } + + /// Translation table memory attributes for TRANSCFG register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + pub(crate) enum PtwMemattr { + /// Invalid (reset value, not valid for enabled address space). + Invalid = 0, + /// Normal memory, inner/outer non-cacheable. + NonCacheable = 1, + /// Normal memory, inner/outer write-back cacheable. + WriteBack = 2, + } + + impl TryFrom> for PtwMemattr { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(PtwMemattr::Invalid), + 1 => Ok(PtwMemattr::NonCacheable), + 2 => Ok(PtwMemattr::WriteBack), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(attr: PtwMemattr) -> Self { + Bounded::try_new(attr as u64).unwrap() + } + } + + /// Translation table memory shareability for TRANSCFG register. + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(u8)] + #[allow(clippy::enum_variant_names)] + pub(crate) enum PtwShareability { + /// Non-shareable. + NonShareable = 0, + /// Outer shareable. + OuterShareable = 2, + /// Inner shareable. + InnerShareable = 3, + } + + impl TryFrom> for PtwShareability { + type Error = Error; + + fn try_from(val: Bounded) -> Result { + match val.get() { + 0 => Ok(PtwShareability::NonShareable), + 2 => Ok(PtwShareability::OuterShareable), + 3 => Ok(PtwShareability::InnerShareable), + _ => Err(EINVAL), + } + } + } + + impl From for Bounded { + fn from(sh: PtwShareability) -> Self { + Bounded::try_new(sh as u64).unwrap() + } + } + + register! { + /// Translation configuration and control. + pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 { + /// Address space mode. + 3:0 mode ?=> AddressSpaceMode; + /// Address input restriction. + 10:6 ina_bits ?=> InaBits; + /// Address output restriction. + 18:14 outa_bits; + /// Translation table concatenation enable, a 1-bit boolean flag. + 22:22 sl_concat_en => bool; + /// Translation table memory attributes. + 25:24 ptw_memattr ?=> PtwMemattr; + /// Translation table memory shareability. + 29:28 ptw_sh ?=> PtwShareability; + /// Inner read allocation hint for translation table walks, a 1-bit boolean flag. + 30:30 r_allocate => bool; + /// Disable hierarchical access permissions. + 33:33 disable_hier_ap => bool; + /// Disable access fault checking. + 34:34 disable_af_fault => bool; + /// Disable execution on all writable pages. + 35:35 wxn => bool; + /// Enable execution on readable pages. + 36:36 xreadable => bool; + /// Page-based hardware attributes for translation table walks. + 63:60 ptw_pbha; + } + + // TRANSCFG is a logical 64-bit register, but it is laid out in hardware as two + // 32-bit halves. Define it as separate low/high u32 registers so accesses match + // the MMIO register layout and do not rely on native 64-bit MMIO transactions. + pub(crate) TRANSCFG_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2430 { + 31:0 value; + } + + pub(crate) TRANSCFG_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2434 { + 31:0 value; + } + + /// Extra fault information for each address space. Read only. + pub(crate) FAULTEXTRA_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2438 { + 31:0 value; + } + + pub(crate) FAULTEXTRA_HI(u32)[MAX_AS, stride = STRIDE] @ 0x243c { + 31:0 value; + } + } + } +} From 4cae5d9b3a6bfb5d0b24872fe8edad570c109e89 Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Thu, 9 Apr 2026 10:51:28 -0700 Subject: [PATCH 006/131] drm/tyr: Remove custom register struct Now that Tyr uses the register! macro, it no longer needs to define a custom register struct or read/write functions, so delete them. Reviewed-by: Boris Brezillon Co-developed-by: Daniel Almeida Signed-off-by: Daniel Almeida Reviewed-by: Daniel Almeida Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260409-b4-tyr-use-register-macro-v5-v5-5-8abfff8a0204@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/regs.rs | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index 725e9f191ded..8509093f8f01 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -27,39 +27,6 @@ // does. #![allow(dead_code)] -use kernel::{ - device::{ - Bound, - Device, // - }, - devres::Devres, - io::Io, - prelude::*, // -}; - -use crate::driver::IoMem; - -/// Represents a register in the Register Set -/// -/// TODO: Replace this with the Nova `register!()` macro when it is available. -/// In particular, this will automatically give us 64bit register reads and -/// writes. -pub(crate) struct Register; - -impl Register { - #[inline] - pub(crate) fn read(&self, dev: &Device, iomem: &Devres) -> Result { - let value = (*iomem).access(dev)?.read32(OFFSET); - Ok(value) - } - - #[inline] - pub(crate) fn write(&self, dev: &Device, iomem: &Devres, value: u32) -> Result { - (*iomem).access(dev)?.write32(value, OFFSET); - Ok(()) - } -} - /// Combine two 32-bit values into a single 64-bit value. pub(crate) fn join_u64(lo: u32, hi: u32) -> u64 { (u64::from(lo)) | ((u64::from(hi)) << 32) From d9a6809478f9815b6455a327aa001737ac7b2c09 Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Thu, 9 Apr 2026 10:51:29 -0700 Subject: [PATCH 007/131] drm/tyr: Add DOORBELL_BLOCK registers DOORBELL_BLOCK_n[0-63] is an array of GPU control register pages. Each block is memory-mappable and contains a single DOORBELL register used to trigger actions in the GPU. Add definitions for the DOORBELL_BLOCK registers using the register! macro so they can be used by future Tyr interfaces. Reviewed-by: Boris Brezillon Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260409-b4-tyr-use-register-macro-v5-v5-6-8abfff8a0204@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/regs.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index 8509093f8f01..562023e5df2f 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -1632,3 +1632,25 @@ fn from(sh: PtwShareability) -> Self { } } } + +/// This module corresponds to the DOORBELL_BLOCK_n[0-63] register pages. +pub(crate) mod doorbell_block { + use kernel::register; + + /// Number of doorbells available. + pub(crate) const NUM_DOORBELLS: usize = 64; + + /// Doorbell block stride (64KiB). + /// + /// Each block occupies a full page, allowing it to be mapped + /// separately into a virtual address space. + const STRIDE: usize = 0x10000; + + register! { + /// Doorbell request register. Write-only. + pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 { + /// Doorbell set. Writing 1 triggers the doorbell. + 0:0 ring => bool; + } + } +} From 98e508e5b016cac054ed0f95694a7a3b240108bb Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 10 Apr 2026 19:41:17 -0700 Subject: [PATCH 008/131] gpu: nova-core: use SizeConstants trait for u64 size constants Replace manual usize-to-u64 conversions of SZ_* constants with the SizeConstants trait's associated constants on u64. With the SizeConstants trait in scope, u64::SZ_1M replaces usize_as_u64(SZ_1M) and similar. This removes several now-unused imports: usize_as_u64, FromSafeCast, and individual SZ_* type-level constants. Reviewed-by: Eliot Courtney Reviewed-by: Joel Fernandes Acked-by: Gary Guo Signed-off-by: John Hubbard Link: https://patch.msgid.link/20260411024118.471294-2-jhubbard@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/fb.rs | 21 +++++++++------------ drivers/gpu/nova-core/gsp/fw.rs | 15 +++++++-------- drivers/gpu/nova-core/regs.rs | 7 +++---- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index bdd5eed760e1..5c304fc03467 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -24,11 +24,8 @@ firmware::gsp::GspFirmware, gpu::Chipset, gsp, - num::{ - usize_as_u64, - FromSafeCast, // - }, - regs, + num::FromSafeCast, + regs, // }; mod hal; @@ -127,8 +124,8 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if f.alternate() { let size = self.len(); - if size < usize_as_u64(SZ_1M) { - let size_kib = size / usize_as_u64(SZ_1K); + if size < u64::SZ_1M { + let size_kib = size / u64::SZ_1K; f.write_fmt(fmt!( "{:#x}..{:#x} ({} KiB)", self.0.start, @@ -136,7 +133,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { size_kib )) } else { - let size_mib = size / usize_as_u64(SZ_1M); + let size_mib = size / u64::SZ_1M; f.write_fmt(fmt!( "{:#x}..{:#x} ({} MiB)", self.0.start, @@ -186,7 +183,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result< let vga_workspace = { let vga_base = { - const NV_PRAMIN_SIZE: u64 = usize_as_u64(SZ_1M); + const NV_PRAMIN_SIZE: u64 = u64::SZ_1M; let base = fb.end - NV_PRAMIN_SIZE; if hal.supports_display(bar) { @@ -196,7 +193,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result< { Some(addr) => { if addr < base { - const VBIOS_WORKSPACE_SIZE: u64 = usize_as_u64(SZ_128K); + const VBIOS_WORKSPACE_SIZE: u64 = u64::SZ_128K; // Point workspace address to end of framebuffer. fb.end - VBIOS_WORKSPACE_SIZE @@ -216,7 +213,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result< let frts = { const FRTS_DOWN_ALIGN: Alignment = Alignment::new::(); - const FRTS_SIZE: u64 = usize_as_u64(SZ_1M); + const FRTS_SIZE: u64 = u64::SZ_1M; let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - FRTS_SIZE; FbRange(frts_base..frts_base + FRTS_SIZE) @@ -256,7 +253,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result< }; let heap = { - const HEAP_SIZE: u64 = usize_as_u64(SZ_1M); + const HEAP_SIZE: u64 = u64::SZ_1M; FbRange(wpr2.start - HEAP_SIZE..wpr2.start) }; diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 0c8a74f0e8ac..9dac3288f3a3 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -17,8 +17,8 @@ KnownSize, // }, sizes::{ - SZ_128K, - SZ_1M, // + SizeConstants, + SZ_128K, // }, transmute::{ AsBytes, @@ -123,7 +123,7 @@ fn client_alloc_size() -> u64 { /// Returns the amount of memory to reserve for management purposes for a framebuffer of size /// `fb_size`. fn management_overhead(fb_size: u64) -> u64 { - let fb_size_gb = fb_size.div_ceil(u64::from_safe_cast(kernel::sizes::SZ_1G)); + let fb_size_gb = fb_size.div_ceil(u64::SZ_1G); u64::from(bindings::GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB) .saturating_mul(fb_size_gb) @@ -145,9 +145,8 @@ impl LibosParams { const LIBOS2: LibosParams = LibosParams { carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2), allowed_heap_size: num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB) - * num::usize_as_u64(SZ_1M) - ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) - * num::usize_as_u64(SZ_1M), + * u64::SZ_1M + ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) * u64::SZ_1M, }; /// Version 3 of the GSP LIBOS (GA102+) @@ -155,9 +154,9 @@ impl LibosParams { carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL), allowed_heap_size: num::u32_as_u64( bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB, - ) * num::usize_as_u64(SZ_1M) + ) * u64::SZ_1M ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MAX_MB) - * num::usize_as_u64(SZ_1M), + * u64::SZ_1M, }; /// Returns the libos parameters corresponding to `chipset`. diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 2f171a4ff9ba..6faeed73901d 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -7,6 +7,7 @@ Io, // }, prelude::*, + sizes::SizeConstants, time, // }; @@ -30,7 +31,6 @@ Architecture, Chipset, // }, - num::FromSafeCast, }; // PMC @@ -150,8 +150,7 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { impl NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE { /// Returns the usable framebuffer size, in bytes. pub(crate) fn usable_fb_size(self) -> u64 { - let size = (u64::from(self.lower_mag()) << u64::from(self.lower_scale())) - * u64::from_safe_cast(kernel::sizes::SZ_1M); + let size = (u64::from(self.lower_mag()) << u64::from(self.lower_scale())) * u64::SZ_1M; if self.ecc_mode_enabled() { // Remove the amount of memory reserved for ECC (one per 16 units). @@ -241,7 +240,7 @@ pub(crate) fn completed(self) -> bool { impl NV_USABLE_FB_SIZE_IN_MB { /// Returns the usable framebuffer size, in bytes. pub(crate) fn usable_fb_size(self) -> u64 { - u64::from(self.value()) * u64::from_safe_cast(kernel::sizes::SZ_1M) + u64::from(self.value()) * u64::SZ_1M } } From f54b0c38bc0db94003978c8f36bc1879a27280bb Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Thu, 5 Feb 2026 16:59:22 -0600 Subject: [PATCH 009/131] gpu: nova-core: program_brom cannot fail Change the signature of the program_brom HAL method to not return anything. None of the implementations can actually fail, so they always return Ok(()). Signed-off-by: Timur Tabi Link: https://patch.msgid.link/20260205225922.2158430-1-ttabi@nvidia.com [acourbot: fix conflicts when applying.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon.rs | 4 ++-- drivers/gpu/nova-core/falcon/hal.rs | 2 +- drivers/gpu/nova-core/falcon/hal/ga102.rs | 8 +++----- drivers/gpu/nova-core/falcon/hal/tu102.rs | 4 +--- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 33927af4134c..24cc2c26e28d 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -488,7 +488,7 @@ pub(crate) fn pio_load + FalconPioLoadable>( } self.pio_wr_dmem_slice(bar, fw.dmem_load_params())?; - self.hal.program_brom(self, bar, &fw.brom_params())?; + self.hal.program_brom(self, bar, &fw.brom_params()); bar.write( WithBase::of::(), @@ -647,7 +647,7 @@ fn dma_load + FalconDmaLoadable>( )?; self.dma_wr(bar, &dma_obj, FalconMem::Dmem, fw.dmem_load_params())?; - self.hal.program_brom(self, bar, &fw.brom_params())?; + self.hal.program_brom(self, bar, &fw.brom_params()); // Set `BootVec` to start of non-secure code. bar.write( diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index a7e5ea8d0272..71df33c79884 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -46,7 +46,7 @@ fn signature_reg_fuse_version( ) -> Result; /// Program the boot ROM registers prior to starting a secure firmware. - fn program_brom(&self, falcon: &Falcon, bar: &Bar0, params: &FalconBromParams) -> Result; + fn program_brom(&self, falcon: &Falcon, bar: &Bar0, params: &FalconBromParams); /// Check if the RISC-V core is active. /// Returns `true` if the RISC-V core is active, `false` otherwise. diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs index 8368a61ddeef..3df1ffa159b8 100644 --- a/drivers/gpu/nova-core/falcon/hal/ga102.rs +++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs @@ -86,7 +86,7 @@ fn signature_reg_fuse_version_ga102( Ok(u16::BITS - reg_fuse_version.leading_zeros()) } -fn program_brom_ga102(bar: &Bar0, params: &FalconBromParams) -> Result { +fn program_brom_ga102(bar: &Bar0, params: &FalconBromParams) { bar.write( WithBase::of::().at(0), regs::NV_PFALCON2_FALCON_BROM_PARAADDR::zeroed().with_value(params.pkc_data_offset), @@ -104,8 +104,6 @@ fn program_brom_ga102(bar: &Bar0, params: &FalconBromParams) -> WithBase::of::(), regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k), ); - - Ok(()) } pub(super) struct Ga102(PhantomData); @@ -131,8 +129,8 @@ fn signature_reg_fuse_version( signature_reg_fuse_version_ga102(&falcon.dev, bar, engine_id_mask, ucode_id) } - fn program_brom(&self, _falcon: &Falcon, bar: &Bar0, params: &FalconBromParams) -> Result { - program_brom_ga102::(bar, params) + fn program_brom(&self, _falcon: &Falcon, bar: &Bar0, params: &FalconBromParams) { + program_brom_ga102::(bar, params); } fn is_riscv_active(&self, bar: &Bar0) -> bool { diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index c7a90266cb44..d8f5d271811b 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -48,9 +48,7 @@ fn signature_reg_fuse_version( Ok(0) } - fn program_brom(&self, _falcon: &Falcon, _bar: &Bar0, _params: &FalconBromParams) -> Result { - Ok(()) - } + fn program_brom(&self, _falcon: &Falcon, _bar: &Bar0, _params: &FalconBromParams) {} fn is_riscv_active(&self, bar: &Bar0) -> bool { bar.read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::()) From d8198971dc72808c7aa29b3556bcaac76702848b Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Wed, 8 Apr 2026 09:21:32 -0500 Subject: [PATCH 010/131] Documentation: gpu: nova: document the IFR header layout Init-from-ROM (IFR) is a special GPU feature used for power management on some Nvidia GPUs. It references data in the VBIOS for its operation, but for drivers the important piece is the header that precedes the VBIOS PCI Expansion ROM. Parsing VBIOS is necessary to boot GSP-RM on Turing, Ampere, and Ada GPUs. Most such GPUs do not need to parse the IFR header in order to find the VBIOS, but the Nvidia GA100 is the exception. GA100 lacks a display engine, so the PRAMIN method (which reads the VBIOS from VRAM via display hardware) is unavailable, forcing the driver to read the ROM directly via PROM. On other similar GPUs, either PRAMIN succeeds before PROM is tried, or the IFR hardware has already applied the ROM offset so that PROM reads transparently skip the IFR header. Note that GH100 also does not have a display engine, but it uses a completely different method to boot GSP-RM. This information is derived from NVIDIA's open-source GPU kernel module driver (aka OpenRM), specifically the NV_PBUS_IFR_FMT_FIXEDx definitions in dev_bus.h and the parsing logic in s_romImgFindPciHeader_TU102(). Signed-off-by: Timur Tabi Reviewed-by: Joel Fernandes Link: https://patch.msgid.link/20260408142132.3911466-1-ttabi@nvidia.com [acourbot: fix TOTAL_DATA_SIZE definition.] Signed-off-by: Alexandre Courbot --- Documentation/gpu/nova/core/vbios.rst | 63 ++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/Documentation/gpu/nova/core/vbios.rst b/Documentation/gpu/nova/core/vbios.rst index efd40087480c..a4fe63422ede 100644 --- a/Documentation/gpu/nova/core/vbios.rst +++ b/Documentation/gpu/nova/core/vbios.rst @@ -46,12 +46,71 @@ region is only accessible to heavy-secure ucode. are of type 0xE0 and can be identified as such. This could be subject to change in future generations. +IFR Header +---------- +On Kepler and later GPUs, the ROM begins with an Init-from-ROM (IFR) header +rather than a standard PCI ROM signature (0xAA55). The driver must parse the +IFR header to find where the PCI ROM images actually start. + +Init-from-ROM (IFR) is a special GPU feature used for power management +on some Nvidia GPUs. It references data in the VBIOS for its operation, +but for drivers the important piece is a header that precedes the +VBIOS PCI Expansion ROM. + +Most such GPUs do not need to parse the IFR header in order to find the +VBIOS, but the Nvidia GA100 is the exception. GA100 lacks a display engine, +so the PRAMIN method (which reads the VBIOS from VRAM via display hardware) +is unavailable, forcing the driver to read the ROM directly via PROM. +On other similar GPUs, either PRAMIN succeeds before PROM is tried, or the +IFR hardware has already applied the ROM offset so that PROM reads +transparently skip the IFR header. + +The driver should first check for the standard 0xAA55 signature at offset 0. +If found, there is no IFR header and the PCI ROM images start at +offset 0. If not found, check for the IFR signature and parse the header to +determine the PCI ROM image offset. + +Fixed Header Format +~~~~~~~~~~~~~~~~~~~ + +The IFR header begins with four 32-bit words at fixed offsets:: + + Offset Name Fields + ------ ------- ------ + 0x00 FIXED0 bits 31:0 - Signature (must be 0x4947564E, ASCII "NVGI") + 0x04 FIXED1 bit 31 - Reserved + bits 30:16 - FIXED_DATA_SIZE Fixed data size (offset to extended section) + bits 15:8 - VERSIONSW Software version + bits 7:0 - Reserved + 0x08 FIXED2 bit 31 - Reserved + bits 30:20 - Reserved (zero) + bits 19:0 - TOTAL_DATA_SIZE Total data size + +Finding the PCI ROM Image Offset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The method to find this offset depends on `VERSIONSW`. + +- **Version 1 and 2**: Read `FIXED_DATA_SIZE` from `FIXED1` to get the extended + section offset. The PCI ROM image is the 32-bit value at `FIXED_DATA_SIZE + 4`. + +- **Version 3**: Read `TOTAL_DATA_SIZE` from `FIXED2`. The 32-bit value at that + offset is a flash status offset. Add 4096 to get the ROM directory offset, + `ROM_DIRECTORY_OFFSET`. The ROM directory must have signature 0x44524652 + (ASCII "RFRD"). The PCI ROM image offset is the 32-bit value at + `ROM_DIRECTORY_OFFSET + 8`. + +The PCI ROM image offset must be 4-byte aligned. All offsets are relative to the +start of ROM (BAR0 + 0x300000). + VBIOS ROM Layout ---------------- -The VBIOS layout is roughly a series of concatenated images laid out as follows:: +The VBIOS (PCI Expansion ROM) is a series of concatenated images laid out as +follows. On GPUs with an IFR header, this layout begins at the image offset +determined by parsing the IFR header. On older GPUs, it begins at offset 0:: +----------------------------------------------------------------------------+ - | VBIOS (Starting at ROM_OFFSET: 0x300000) | + | VBIOS (Starting at ROM_OFFSET: 0x300000 + IFR image offset) | +----------------------------------------------------------------------------+ | +-----------------------------------------------+ | | | PciAt Image (Type 0x00) | | From c1dca0cb0e761568c448a6007855fe5879ad4d37 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 17 Apr 2026 14:13:54 -0500 Subject: [PATCH 011/131] gpu: nova-core: use correct fwsignature for GA100 Although GA100 uses the same GSP-RM firmware as Turing, it has a different signature specifically for it. Fixes: 121ea04cd9f2 ("gpu: nova-core: add support for Turing/GA100 fwsignature") Signed-off-by: Timur Tabi Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260417191359.1307434-2-ttabi@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware/gsp.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index 2fcc255c3bc8..c423191b21f0 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -138,8 +138,7 @@ pub(crate) fn new<'a>( ".fwsignature_tu11x" } Architecture::Turing => ".fwsignature_tu10x", - // GA100 uses the same firmware as Turing - Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_tu10x", + Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_ga100", Architecture::Ampere => ".fwsignature_ga10x", Architecture::Ada => ".fwsignature_ad10x", }; From 44dc8bd6b5b9af46eafa552fb9631e62e8bbf3ac Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 17 Apr 2026 14:13:55 -0500 Subject: [PATCH 012/131] gpu: nova-core: do not consider 0xBB77 as a valid PCI ROM header signature Nvidia GPUs have some PCI expansion ROM sections that have an Nvidia- specific signature instead of 0xAA55. Signature 0xBB77 is actually an internal-only value that has been deprecated for over a decade. Nova-core will never encounter a GPU with that signature, so don't look for it. Signed-off-by: Timur Tabi Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260417191359.1307434-3-ttabi@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/vbios.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index ebda28e596c5..e726594eb130 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -491,7 +491,7 @@ fn new(dev: &device::Device, data: &[u8]) -> Result { // Check for valid ROM signatures. match signature { - 0xAA55 | 0xBB77 | 0x4E56 => {} + 0xAA55 | 0x4E56 => {} _ => { dev_err!(dev, "ROM signature unknown {:#x}\n", signature); return Err(EINVAL); From cedbbc383ab3e21331ef36f90e0dfab89b58934d Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 17 Apr 2026 14:13:56 -0500 Subject: [PATCH 013/131] gpu: nova-core: only boot FRTS if its region is allocated On some Nvidia GPUs (i.e. GA100), the FRTS region is not allocated (its size is set to 0). In such cases, FWSEC-FRTS should not be run. Signed-off-by: Timur Tabi Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260417191359.1307434-4-ttabi@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/boot.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 18f356c9178e..5c56f0539dd7 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -155,7 +155,10 @@ pub(crate) fn boot( let fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?; dev_dbg!(dev, "{:#x?}\n", fb_layout); - Self::run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?; + // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). + if !fb_layout.frts.is_empty() { + Self::run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?; + } let booter_loader = BooterFirmware::new( dev, From 21decd6324902503692529a13523969599755a13 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 17 Apr 2026 14:13:57 -0500 Subject: [PATCH 014/131] gpu: nova-core: add FbHal::frts_size() for GA100 support Introduce FbHal method frts_size() to return the size of the FRTS window. GA100 is a special case in that although there is an FRTS, its size must arbitrarily be set to 0. Note that we cannot use supports_display() to determine the FRTS size because there are other GPUs (e.g. GA102GL) that have display disabled (and so supports_display() returns False), but the FRTS window size still needs to be 1MB. Signed-off-by: Timur Tabi Reviewed-by: Eliot Courtney Acked-by: Gary Guo Link: https://patch.msgid.link/20260417191359.1307434-5-ttabi@nvidia.com [acourbot: apply requested fix to commit message.] [acourbot: fix minor conflict.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 6 +++--- drivers/gpu/nova-core/fb/hal.rs | 3 +++ drivers/gpu/nova-core/fb/hal/ga100.rs | 6 ++++++ drivers/gpu/nova-core/fb/hal/ga102.rs | 4 ++++ drivers/gpu/nova-core/fb/hal/tu102.rs | 11 ++++++++++- 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 5c304fc03467..667f5e850175 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -213,10 +213,10 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result< let frts = { const FRTS_DOWN_ALIGN: Alignment = Alignment::new::(); - const FRTS_SIZE: u64 = u64::SZ_1M; - let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - FRTS_SIZE; + let frts_size: u64 = hal.frts_size(); + let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - frts_size; - FbRange(frts_base..frts_base + FRTS_SIZE) + FbRange(frts_base..frts_base + frts_size) }; let boot = { diff --git a/drivers/gpu/nova-core/fb/hal.rs b/drivers/gpu/nova-core/fb/hal.rs index aba0abd8ee00..1c01a6cbed65 100644 --- a/drivers/gpu/nova-core/fb/hal.rs +++ b/drivers/gpu/nova-core/fb/hal.rs @@ -25,6 +25,9 @@ pub(crate) trait FbHal { /// Returns the VRAM size, in bytes. fn vidmem_size(&self, bar: &Bar0) -> u64; + + /// Returns the FRTS size, in bytes. + fn frts_size(&self) -> u64; } /// Returns the HAL corresponding to `chipset`. diff --git a/drivers/gpu/nova-core/fb/hal/ga100.rs b/drivers/gpu/nova-core/fb/hal/ga100.rs index 1c03783cddef..2f5871d915c3 100644 --- a/drivers/gpu/nova-core/fb/hal/ga100.rs +++ b/drivers/gpu/nova-core/fb/hal/ga100.rs @@ -66,6 +66,12 @@ fn supports_display(&self, bar: &Bar0) -> bool { fn vidmem_size(&self, bar: &Bar0) -> u64 { super::tu102::vidmem_size_gp102(bar) } + + // GA100 is a special case where its FRTS region exists, but is empty. We + // return a size of 0 because we still need to record where the region starts. + fn frts_size(&self) -> u64 { + 0 + } } const GA100: Ga100 = Ga100; diff --git a/drivers/gpu/nova-core/fb/hal/ga102.rs b/drivers/gpu/nova-core/fb/hal/ga102.rs index 4b9f0f74d0e7..3bb66f64bef7 100644 --- a/drivers/gpu/nova-core/fb/hal/ga102.rs +++ b/drivers/gpu/nova-core/fb/hal/ga102.rs @@ -35,6 +35,10 @@ fn supports_display(&self, bar: &Bar0) -> bool { fn vidmem_size(&self, bar: &Bar0) -> u64 { vidmem_size_ga102(bar) } + + fn frts_size(&self) -> u64 { + super::tu102::frts_size_tu102() + } } const GA102: Ga102 = Ga102; diff --git a/drivers/gpu/nova-core/fb/hal/tu102.rs b/drivers/gpu/nova-core/fb/hal/tu102.rs index 281bb796e198..22c174bf1472 100644 --- a/drivers/gpu/nova-core/fb/hal/tu102.rs +++ b/drivers/gpu/nova-core/fb/hal/tu102.rs @@ -2,7 +2,8 @@ use kernel::{ io::Io, - prelude::*, // + prelude::*, + sizes::*, // }; use crate::{ @@ -38,6 +39,10 @@ pub(super) fn vidmem_size_gp102(bar: &Bar0) -> u64 { .usable_fb_size() } +pub(super) const fn frts_size_tu102() -> u64 { + u64::SZ_1M +} + struct Tu102; impl FbHal for Tu102 { @@ -56,6 +61,10 @@ fn supports_display(&self, bar: &Bar0) -> bool { fn vidmem_size(&self, bar: &Bar0) -> u64 { vidmem_size_gp102(bar) } + + fn frts_size(&self) -> u64 { + frts_size_tu102() + } } const TU102: Tu102 = Tu102; From 1a5a3f20501574f843c3b77ae1581dc431826762 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 17 Apr 2026 14:13:58 -0500 Subject: [PATCH 015/131] gpu: nova-core: skip the IFR header if present The GPU's ROM may begin with an Init-from-ROM (IFR) header that precedes the PCI Expansion ROM images (VBIOS). When present, the PROM shadow method must parse this header to determine the offset where the PCI ROM images actually begin, and adjust all subsequent reads accordingly. On most GPUs this is not needed because the IFR microcode has already applied the ROM offset so that PROM reads transparently skip the header. On GA100, for whatever reason, the IFR offset is not applied to PROM reads. Therefore, the search for the PCI expansion must skip the IFR header, if found. Signed-off-by: Timur Tabi Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260417191359.1307434-6-ttabi@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/vbios.rs | 85 +++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index e726594eb130..6bcfb6c5cf44 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -12,6 +12,8 @@ Alignable, Alignment, // }, + register, + sizes::SZ_4K, sync::aref::ARef, transmute::FromBytes, }; @@ -89,13 +91,94 @@ struct VbiosIterator<'a> { last_found: bool, } +// IFR Header in VBIOS. + +register! { + pub(crate) NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 { + 31:0 signature; + } +} + +register! { + pub(crate) NV_PBUS_IFR_FMT_FIXED1(u32) @ 0x300004 { + 30:16 fixed_data_size; + 15:8 version => u8; + } +} + +register! { + pub(crate) NV_PBUS_IFR_FMT_FIXED2(u32) @ 0x300008 { + 19:0 total_data_size; + } +} + +/// Return the byte offset where the PCI Expansion ROM images begin in the GPU's ROM. +/// +/// The GPU's ROM may begin with an Init-from-ROM (IFR) header that precedes +/// the PCI Expansion ROM images (VBIOS). When present, the PROM shadow +/// method must parse this header to determine the offset where the PCI ROM +/// images actually begin, and adjust all subsequent reads accordingly. +/// +/// On most GPUs this is not needed because the IFR microcode has already +/// applied the ROM offset so that PROM reads transparently skip the header. +/// On GA100, for some reason, the IFR offset is not applied to PROM +/// reads. Therefore, the search for the PCI expansion must skip the IFR +/// header, if found. +fn vbios_rom_offset(dev: &device::Device, bar0: &Bar0) -> Result { + /// IFR signature. + const NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE: u32 = u32::from_le_bytes(*b"NVGI"); + /// ROM directory signature. + const NV_ROM_DIRECTORY_IDENTIFIER: u32 = u32::from_le_bytes(*b"RFRD"); + /// Offset of the NV_PMGR_ROM_ADDR_OFFSET register in IFR Extended section. + const IFR_SW_EXT_ROM_ADDR_OFFSET: usize = 4; + /// Size of Redundant Firmware Flash Status section. + const RFW_FLASH_STATUS_SIZE: usize = SZ_4K; + /// Offset in the ROM Directory of the PCI Option ROM offset + const PCI_OPTION_ROM_OFFSET: usize = 8; + + let signature = bar0.read(NV_PBUS_IFR_FMT_FIXED0).signature(); + + if signature == NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE { + let fixed1 = bar0.read(NV_PBUS_IFR_FMT_FIXED1); + + match fixed1.version() { + 1 | 2 => { + let fixed_data_size = usize::from(fixed1.fixed_data_size()); + let pmgr_rom_addr_offset = fixed_data_size + IFR_SW_EXT_ROM_ADDR_OFFSET; + bar0.try_read32(ROM_OFFSET + pmgr_rom_addr_offset) + .map(usize::from_safe_cast) + } + 3 => { + let fixed2 = bar0.read(NV_PBUS_IFR_FMT_FIXED2); + let total_data_size = usize::from(fixed2.total_data_size()); + let flash_status_offset = + usize::from_safe_cast(bar0.try_read32(ROM_OFFSET + total_data_size)?); + let dir_offset = flash_status_offset + RFW_FLASH_STATUS_SIZE; + let dir_sig = bar0.try_read32(ROM_OFFSET + dir_offset)?; + if dir_sig != NV_ROM_DIRECTORY_IDENTIFIER { + dev_err!(dev, "could not find IFR ROM directory\n"); + return Err(EINVAL); + } + bar0.try_read32(ROM_OFFSET + dir_offset + PCI_OPTION_ROM_OFFSET) + .map(usize::from_safe_cast) + } + _ => { + dev_err!(dev, "unsupported IFR header version {}\n", fixed1.version()); + Err(EINVAL) + } + } + } else { + Ok(0) + } +} + impl<'a> VbiosIterator<'a> { fn new(dev: &'a device::Device, bar0: &'a Bar0) -> Result { Ok(Self { dev, bar0, data: KVec::new(), - current_offset: 0, + current_offset: vbios_rom_offset(dev, bar0)?, last_found: false, }) } From a4c6d5d146157228bc03429cf7926d8a14072c84 Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Fri, 17 Apr 2026 14:13:59 -0500 Subject: [PATCH 016/131] gpu: nova-core: enable GA100 GA100 is a compute-only variant of GA102 that boots GSP-RM like a Turing, although it also has its own unique requirements. Now that all the pieces are in place, we can enable GA100 support. Although architecturally like an Ampere, GA100 uses the same GSP-RM firmware files as Turing, and therefore must boot it like Turing does. However, as a compute-only part, GA100 has no display engine. Signed-off-by: Timur Tabi Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260417191359.1307434-7-ttabi@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/hal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index 71df33c79884..f82087df4936 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -77,13 +77,13 @@ pub(super) fn falcon_hal( use Chipset::*; let hal = match chipset { + GA100 | // GA100 boots like Turing so use Turing HAL TU102 | TU104 | TU106 | TU116 | TU117 => { KBox::new(tu102::Tu102::::new(), GFP_KERNEL)? as KBox> } GA102 | GA103 | GA104 | GA106 | GA107 | AD102 | AD103 | AD104 | AD106 | AD107 => { KBox::new(ga102::Ga102::::new(), GFP_KERNEL)? as KBox> } - _ => return Err(ENOTSUPP), }; Ok(hal) From d39e22cf87b664d2398523294055e1d6b96dde1f Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Tue, 7 Apr 2026 12:59:50 +0900 Subject: [PATCH 017/131] gpu: nova: require little endian The driver already assumes little endian in a lot of locations. For example, all the code that reads RPCs out of the command queue just directly interprets the bytes. Make this explicit in Kconfig. Signed-off-by: Eliot Courtney Reviewed-by: Alexandre Courbot Reviewed-by: Joel Fernandes Reviewed-by: Gary Guo Reviewed-by: John Hubbard Link: https://patch.msgid.link/20260407-fix-kconfig-v2-1-6b4fb06c690c@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/Kconfig | 1 + drivers/gpu/nova-core/Kconfig | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/gpu/drm/nova/Kconfig b/drivers/gpu/drm/nova/Kconfig index 3e637ad7b5ba..a2028b8539d7 100644 --- a/drivers/gpu/drm/nova/Kconfig +++ b/drivers/gpu/drm/nova/Kconfig @@ -4,6 +4,7 @@ config DRM_NOVA depends on DRM=y depends on PCI depends on RUST + depends on !CPU_BIG_ENDIAN select AUXILIARY_BUS select NOVA_CORE default n diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig index a4f2380654e2..d8456f8eaa05 100644 --- a/drivers/gpu/nova-core/Kconfig +++ b/drivers/gpu/nova-core/Kconfig @@ -3,6 +3,7 @@ config NOVA_CORE depends on 64BIT depends on PCI depends on RUST + depends on !CPU_BIG_ENDIAN select AUXILIARY_BUS select RUST_FW_LOADER_ABSTRACTIONS default n From 013ff3b4d0222f6154226b6b2b0a8c9af0e3ebc9 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 25 Mar 2026 18:38:57 -0700 Subject: [PATCH 018/131] gpu: nova-core: make WPR heap sizing fallible Make management_overhead() fail on multiplication or alignment overflow instead of silently saturating. Propagate that failure through wpr_heap_size() and the framebuffer layout code that consumes it. Signed-off-by: John Hubbard Link: https://patch.msgid.link/20260326013902.588242-27-jhubbard@nvidia.com [acourbot: remove unrelated WPR2 mention from commit log.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 2 +- drivers/gpu/nova-core/gsp/fw.rs | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 667f5e850175..6ee87050ce69 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -238,7 +238,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result< let wpr2_heap = { const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::new::(); let wpr2_heap_size = - gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end); + gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end)?; let wpr2_heap_addr = (elf.start - wpr2_heap_size).align_down(WPR2_HEAP_DOWN_ALIGN); FbRange(wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN)) diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 9dac3288f3a3..3245793bbe42 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -122,13 +122,14 @@ fn client_alloc_size() -> u64 { /// Returns the amount of memory to reserve for management purposes for a framebuffer of size /// `fb_size`. - fn management_overhead(fb_size: u64) -> u64 { + fn management_overhead(fb_size: u64) -> Result { let fb_size_gb = fb_size.div_ceil(u64::SZ_1G); u64::from(bindings::GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB) - .saturating_mul(fb_size_gb) + .checked_mul(fb_size_gb) + .ok_or(EINVAL)? .align_up(GSP_HEAP_ALIGNMENT) - .unwrap_or(u64::MAX) + .ok_or(EINVAL) } } @@ -170,18 +171,19 @@ pub(crate) fn from_chipset(chipset: Chipset) -> &'static LibosParams { /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size /// of `fb_size` (in bytes) for `chipset`. - pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> u64 { + pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result { // The WPR heap will contain the following: // LIBOS carveout, - self.carveout_size + Ok(self + .carveout_size // RM boot working memory, .saturating_add(GspFwHeapParams::base_rm_size(chipset)) // One RM client, .saturating_add(GspFwHeapParams::client_alloc_size()) // Overhead for memory management. - .saturating_add(GspFwHeapParams::management_overhead(fb_size)) + .saturating_add(GspFwHeapParams::management_overhead(fb_size)?) // Clamp to the supported heap sizes. - .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1) + .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1)) } } From 777123bf2be607701b397f07198370700d2bfbdd Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 10 Apr 2026 19:49:26 -0700 Subject: [PATCH 019/131] gpu: nova-core: factor .fwsignature* selection into a new find_gsp_sigs_section() Keep Gsp::new() from getting too cluttered, by factoring out the selection of .fwsignature* items. This will continue to grow as we add GPUs. Signed-off-by: John Hubbard Reviewed-by: Gary Guo Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260411024953.473149-2-jhubbard@nvidia.com [acourbot: fix minor conflict.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware/gsp.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index c423191b21f0..71b238b76349 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -63,6 +63,18 @@ pub(crate) struct GspFirmware { } impl GspFirmware { + fn find_gsp_sigs_section(chipset: Chipset) -> &'static str { + match chipset.arch() { + Architecture::Turing if matches!(chipset, Chipset::TU116 | Chipset::TU117) => { + ".fwsignature_tu11x" + } + Architecture::Turing => ".fwsignature_tu10x", + Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_ga100", + Architecture::Ampere => ".fwsignature_ga10x", + Architecture::Ada => ".fwsignature_ad10x", + } + } + /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page /// tables expected by the GSP bootloader to load it. pub(crate) fn new<'a>( @@ -131,17 +143,7 @@ pub(crate) fn new<'a>( }, size, signatures: { - let sigs_section = match chipset.arch() { - Architecture::Turing - if matches!(chipset, Chipset::TU116 | Chipset::TU117) => - { - ".fwsignature_tu11x" - } - Architecture::Turing => ".fwsignature_tu10x", - Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_ga100", - Architecture::Ampere => ".fwsignature_ga10x", - Architecture::Ada => ".fwsignature_ad10x", - }; + let sigs_section = Self::find_gsp_sigs_section(chipset); elf::elf64_section(firmware.data(), sigs_section) .ok_or(EINVAL) From b3bdb42619c8fce6d6d5feb36ad33d184aaa4e3d Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 10 Apr 2026 19:49:27 -0700 Subject: [PATCH 020/131] gpu: nova-core: use GPU Architecture to simplify HAL selections Replace per-chipset match arms with Architecture-based matching in the falcon and FB HAL selection functions. This reduces the number of match arms that need updating when new chipsets are added within an existing architecture. Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260411024953.473149-3-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/hal.rs | 18 +++++++++++------- drivers/gpu/nova-core/fb/hal.rs | 18 +++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index f82087df4936..f75b5c315d62 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -9,7 +9,10 @@ FalconBromParams, FalconEngine, // }, - gpu::Chipset, + gpu::{ + Architecture, + Chipset, // + }, }; mod ga102; @@ -74,14 +77,15 @@ fn signature_reg_fuse_version( pub(super) fn falcon_hal( chipset: Chipset, ) -> Result>> { - use Chipset::*; - - let hal = match chipset { - GA100 | // GA100 boots like Turing so use Turing HAL - TU102 | TU104 | TU106 | TU116 | TU117 => { + let hal = match chipset.arch() { + Architecture::Turing => { KBox::new(tu102::Tu102::::new(), GFP_KERNEL)? as KBox> } - GA102 | GA103 | GA104 | GA106 | GA107 | AD102 | AD103 | AD104 | AD106 | AD107 => { + // GA100 boots like Turing so use Turing HAL + Architecture::Ampere if chipset == Chipset::GA100 => { + KBox::new(tu102::Tu102::::new(), GFP_KERNEL)? as KBox> + } + Architecture::Ampere | Architecture::Ada => { KBox::new(ga102::Ga102::::new(), GFP_KERNEL)? as KBox> } }; diff --git a/drivers/gpu/nova-core/fb/hal.rs b/drivers/gpu/nova-core/fb/hal.rs index 1c01a6cbed65..6f928052d0c0 100644 --- a/drivers/gpu/nova-core/fb/hal.rs +++ b/drivers/gpu/nova-core/fb/hal.rs @@ -4,7 +4,10 @@ use crate::{ driver::Bar0, - gpu::Chipset, // + gpu::{ + Architecture, + Chipset, // + }, }; mod ga100; @@ -32,13 +35,10 @@ pub(crate) trait FbHal { /// Returns the HAL corresponding to `chipset`. pub(super) fn fb_hal(chipset: Chipset) -> &'static dyn FbHal { - use Chipset::*; - - match chipset { - TU102 | TU104 | TU106 | TU117 | TU116 => tu102::TU102_HAL, - GA100 => ga100::GA100_HAL, - GA102 | GA103 | GA104 | GA106 | GA107 | AD102 | AD103 | AD104 | AD106 | AD107 => { - ga102::GA102_HAL - } + match chipset.arch() { + Architecture::Turing => tu102::TU102_HAL, + Architecture::Ampere if chipset == Chipset::GA100 => ga100::GA100_HAL, + Architecture::Ampere => ga102::GA102_HAL, + Architecture::Ada => ga102::GA102_HAL, } } From a47e2cd4b1a9184330d43a4dba21e67548eda232 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 10 Apr 2026 19:49:28 -0700 Subject: [PATCH 021/131] gpu: nova-core: Hopper/Blackwell: basic GPU identification Hopper (GH100) and Blackwell identification, including ELF .fwsignature_* items. Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260411024953.473149-4-jhubbard@nvidia.com [acourbot: add separators for both Blackwell architectures in Chipset definition.] [acourbot: make Gsp::boot() return `ENOTSUPP` on new architectures.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/hal.rs | 6 +++++- drivers/gpu/nova-core/fb/hal.rs | 5 ++++- drivers/gpu/nova-core/firmware/gsp.rs | 3 +++ drivers/gpu/nova-core/gpu.rs | 19 +++++++++++++++++++ drivers/gpu/nova-core/gsp/boot.rs | 13 ++++++++++++- 5 files changed, 43 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index f75b5c315d62..a524c8096b67 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -85,7 +85,11 @@ pub(super) fn falcon_hal( Architecture::Ampere if chipset == Chipset::GA100 => { KBox::new(tu102::Tu102::::new(), GFP_KERNEL)? as KBox> } - Architecture::Ampere | Architecture::Ada => { + Architecture::Ampere + | Architecture::Ada + | Architecture::Hopper + | Architecture::BlackwellGB10x + | Architecture::BlackwellGB20x => { KBox::new(ga102::Ga102::::new(), GFP_KERNEL)? as KBox> } }; diff --git a/drivers/gpu/nova-core/fb/hal.rs b/drivers/gpu/nova-core/fb/hal.rs index 6f928052d0c0..8b192a503363 100644 --- a/drivers/gpu/nova-core/fb/hal.rs +++ b/drivers/gpu/nova-core/fb/hal.rs @@ -39,6 +39,9 @@ pub(super) fn fb_hal(chipset: Chipset) -> &'static dyn FbHal { Architecture::Turing => tu102::TU102_HAL, Architecture::Ampere if chipset == Chipset::GA100 => ga100::GA100_HAL, Architecture::Ampere => ga102::GA102_HAL, - Architecture::Ada => ga102::GA102_HAL, + Architecture::Ada + | Architecture::Hopper + | Architecture::BlackwellGB10x + | Architecture::BlackwellGB20x => ga102::GA102_HAL, } } diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index 71b238b76349..e576bc8a9b1c 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -72,6 +72,9 @@ fn find_gsp_sigs_section(chipset: Chipset) -> &'static str { Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_ga100", Architecture::Ampere => ".fwsignature_ga10x", Architecture::Ada => ".fwsignature_ad10x", + Architecture::Hopper => ".fwsignature_gh10x", + Architecture::BlackwellGB10x => ".fwsignature_gb10x", + Architecture::BlackwellGB20x => ".fwsignature_gb20x", } } diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 0f6fe9a1b955..6a01b85f9489 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -86,12 +86,23 @@ fn try_from(value: u32) -> Result { GA104 = 0x174, GA106 = 0x176, GA107 = 0x177, + // Hopper + GH100 = 0x180, // Ada AD102 = 0x192, AD103 = 0x193, AD104 = 0x194, AD106 = 0x196, AD107 = 0x197, + // Blackwell GB10x + GB100 = 0x1a0, + GB102 = 0x1a2, + // Blackwell GB20x + GB202 = 0x1b2, + GB203 = 0x1b3, + GB205 = 0x1b5, + GB206 = 0x1b6, + GB207 = 0x1b7, }); impl Chipset { @@ -103,9 +114,14 @@ pub(crate) const fn arch(self) -> Architecture { Self::GA100 | Self::GA102 | Self::GA103 | Self::GA104 | Self::GA106 | Self::GA107 => { Architecture::Ampere } + Self::GH100 => Architecture::Hopper, Self::AD102 | Self::AD103 | Self::AD104 | Self::AD106 | Self::AD107 => { Architecture::Ada } + Self::GB100 | Self::GB102 => Architecture::BlackwellGB10x, + Self::GB202 | Self::GB203 | Self::GB205 | Self::GB206 | Self::GB207 => { + Architecture::BlackwellGB20x + } } } @@ -137,7 +153,10 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { pub(crate) enum Architecture with TryFrom> { Turing = 0x16, Ampere = 0x17, + Hopper = 0x18, Ada = 0x19, + BlackwellGB10x = 0x1a, + BlackwellGB20x = 0x1b, } } diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 5c56f0539dd7..df105ef4b371 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -31,7 +31,10 @@ gsp::GspFirmware, FIRMWARE_VERSION, // }, - gpu::Chipset, + gpu::{ + Architecture, + Chipset, // + }, gsp::{ commands, sequencer::{ @@ -146,6 +149,14 @@ pub(crate) fn boot( gsp_falcon: &Falcon, sec2_falcon: &Falcon, ) -> Result { + // The FSP boot process of Hopper+ is not supported for now. + if matches!( + chipset.arch(), + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x + ) { + return Err(ENOTSUPP); + } + let dev = pdev.as_ref(); let bios = Vbios::new(dev, bar)?; From 46d455853da6aa20ce07a23cb053bcf5a87098eb Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 10 Apr 2026 19:49:29 -0700 Subject: [PATCH 022/131] gpu: nova-core: add Copy/Clone to Spec and Revision Derive Clone and Copy for Revision and Spec. Both are small value types (4 bytes total) and Copy makes them easier to use in later patches that pass them across function boundaries. Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260411024953.473149-5-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 6a01b85f9489..3e908b65d7a9 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -160,6 +160,7 @@ pub(crate) enum Architecture with TryFrom> { } } +#[derive(Clone, Copy)] pub(crate) struct Revision { major: Bounded, minor: Bounded, @@ -181,6 +182,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { } /// Structure holding a basic description of the GPU: `Chipset` and `Revision`. +#[derive(Clone, Copy)] pub(crate) struct Spec { chipset: Chipset, revision: Revision, From 8ff326fa0aa19babae405b5fadeaa49b57b9a75f Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 10 Apr 2026 19:49:31 -0700 Subject: [PATCH 023/131] gpu: nova-core: move GFW boot wait into a GPU HAL Introduce a GpuHal trait and per-family dispatch so GPU boot behavior can vary by architecture. Move wait_gfw_boot_completion() from the standalone gfw module into gpu/hal/tu102.rs as the first GpuHal implementation. All architectures currently dispatch to this implementation, preserving existing behavior. Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260411024953.473149-7-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gfw.rs | 76 ----------------------- drivers/gpu/nova-core/gpu.rs | 5 +- drivers/gpu/nova-core/gpu/hal.rs | 29 +++++++++ drivers/gpu/nova-core/gpu/hal/tu102.rs | 86 ++++++++++++++++++++++++++ drivers/gpu/nova-core/nova_core.rs | 1 - 5 files changed, 118 insertions(+), 79 deletions(-) delete mode 100644 drivers/gpu/nova-core/gfw.rs create mode 100644 drivers/gpu/nova-core/gpu/hal.rs create mode 100644 drivers/gpu/nova-core/gpu/hal/tu102.rs diff --git a/drivers/gpu/nova-core/gfw.rs b/drivers/gpu/nova-core/gfw.rs deleted file mode 100644 index fb75dd10a172..000000000000 --- a/drivers/gpu/nova-core/gfw.rs +++ /dev/null @@ -1,76 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! GPU Firmware (`GFW`) support, a.k.a `devinit`. -//! -//! Upon reset, the GPU runs some firmware code from the BIOS to setup its core parameters. Most of -//! the GPU is considered unusable until this step is completed, so we must wait on it before -//! performing driver initialization. -//! -//! A clarification about devinit terminology: devinit is a sequence of register read/writes after -//! reset that performs tasks such as: -//! 1. Programming VRAM memory controller timings. -//! 2. Power sequencing. -//! 3. Clock and PLL configuration. -//! 4. Thermal management. -//! -//! devinit itself is a 'script' which is interpreted by an interpreter program typically running -//! on the PMU microcontroller. -//! -//! Note that the devinit sequence also needs to run during suspend/resume. - -use kernel::{ - io::{ - poll::read_poll_timeout, - Io, // - }, - prelude::*, - time::Delta, // -}; - -use crate::{ - driver::Bar0, - regs, // -}; - -/// Wait for the `GFW` (GPU firmware) boot completion signal (`GFW_BOOT`), or a 4 seconds timeout. -/// -/// Upon GPU reset, several microcontrollers (such as PMU, SEC2, GSP etc) run some firmware code to -/// setup its core parameters. Most of the GPU is considered unusable until this step is completed, -/// so it must be waited on very early during driver initialization. -/// -/// The `GFW` code includes several components that need to execute before the driver loads. These -/// components are located in the VBIOS ROM and executed in a sequence on these different -/// microcontrollers. The devinit sequence typically runs on the PMU, and the FWSEC runs on the -/// GSP. -/// -/// This function waits for a signal indicating that core initialization is complete. Before this -/// signal is received, little can be done with the GPU. This signal is set by the FWSEC running on -/// the GSP in Heavy-secured mode. -pub(crate) fn wait_gfw_boot_completion(bar: &Bar0) -> Result { - // Before accessing the completion status in `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05`, we must - // first check `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK`. This is because - // `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05` becomes accessible only after the secure firmware - // (FWSEC) lowers the privilege level to allow CPU (LS/Light-secured) access. We can only - // safely read the status register from CPU (LS/Light-secured) once the mask indicates - // that the privilege level has been lowered. - // - // TIMEOUT: arbitrarily large value. GFW starts running immediately after the GPU is put out of - // reset, and should complete in less time than that. - read_poll_timeout( - || { - Ok( - // Check that FWSEC has lowered its protection level before reading the GFW_BOOT - // status. - bar.read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK) - .read_protection_level0() - && bar - .read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT) - .completed(), - ) - }, - |&gfw_booted| gfw_booted, - Delta::from_millis(1), - Delta::from_secs(4), - ) - .map(|_| ()) -} diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 3e908b65d7a9..659f6a24ee13 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -20,11 +20,12 @@ Falcon, // }, fb::SysmemFlush, - gfw, gsp::Gsp, regs, }; +mod hal; + macro_rules! define_chipset { ({ $($variant:ident = $value:expr),* $(,)* }) => { @@ -274,7 +275,7 @@ pub(crate) fn new<'a>( // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. _: { - gfw::wait_gfw_boot_completion(bar) + hal::gpu_hal(spec.chipset).wait_gfw_boot_completion(bar) .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?; }, diff --git a/drivers/gpu/nova-core/gpu/hal.rs b/drivers/gpu/nova-core/gpu/hal.rs new file mode 100644 index 000000000000..a261c6af92be --- /dev/null +++ b/drivers/gpu/nova-core/gpu/hal.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 + +use kernel::prelude::*; + +use crate::{ + driver::Bar0, + gpu::{ + Architecture, + Chipset, // + }, +}; + +mod tu102; + +pub(crate) trait GpuHal { + /// Waits for GFW_BOOT completion if required by this hardware family. + fn wait_gfw_boot_completion(&self, bar: &Bar0) -> Result; +} + +pub(super) fn gpu_hal(chipset: Chipset) -> &'static dyn GpuHal { + match chipset.arch() { + Architecture::Turing + | Architecture::Ampere + | Architecture::Ada + | Architecture::Hopper + | Architecture::BlackwellGB10x + | Architecture::BlackwellGB20x => tu102::TU102_HAL, + } +} diff --git a/drivers/gpu/nova-core/gpu/hal/tu102.rs b/drivers/gpu/nova-core/gpu/hal/tu102.rs new file mode 100644 index 000000000000..08dd4434bd72 --- /dev/null +++ b/drivers/gpu/nova-core/gpu/hal/tu102.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! GPU Firmware (`GFW`) support, a.k.a `devinit`. +//! +//! Upon reset, the GPU runs some firmware code from the BIOS to setup its core parameters. Most of +//! the GPU is considered unusable until this step is completed, so we must wait on it before +//! performing driver initialization. +//! +//! A clarification about devinit terminology: devinit is a sequence of register read/writes after +//! reset that performs tasks such as: +//! 1. Programming VRAM memory controller timings. +//! 2. Power sequencing. +//! 3. Clock and PLL configuration. +//! 4. Thermal management. +//! +//! devinit itself is a 'script' which is interpreted by an interpreter program typically running +//! on the PMU microcontroller. +//! +//! Note that the devinit sequence also needs to run during suspend/resume. + +use kernel::{ + io::{ + poll::read_poll_timeout, + Io, // + }, + prelude::*, + time::Delta, // +}; + +use crate::{ + driver::Bar0, + regs, // +}; + +use super::GpuHal; + +struct Tu102; + +impl GpuHal for Tu102 { + /// Wait for the `GFW` (GPU firmware) boot completion signal (`GFW_BOOT`), or a 4 seconds + /// timeout. + /// + /// Upon GPU reset, several microcontrollers (such as PMU, SEC2, GSP etc) run some firmware + /// code to setup its core parameters. Most of the GPU is considered unusable until this step + /// is completed, so it must be waited on very early during driver initialization. + /// + /// The `GFW` code includes several components that need to execute before the driver loads. + /// These components are located in the VBIOS ROM and executed in a sequence on these different + /// microcontrollers. The devinit sequence typically runs on the PMU, and the FWSEC runs on the + /// GSP. + /// + /// This function waits for a signal indicating that core initialization is complete. Before + /// this signal is received, little can be done with the GPU. This signal is set by the FWSEC + /// running on the GSP in Heavy-secured mode. + fn wait_gfw_boot_completion(&self, bar: &Bar0) -> Result { + // Before accessing the completion status in `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05`, we must + // first check `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK`. This is because + // `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05` becomes accessible only after the secure firmware + // (FWSEC) lowers the privilege level to allow CPU (LS/Light-secured) access. We can only + // safely read the status register from CPU (LS/Light-secured) once the mask indicates + // that the privilege level has been lowered. + // + // TIMEOUT: arbitrarily large value. GFW starts running immediately after the GPU is put + // out of reset, and should complete in less time than that. + read_poll_timeout( + || { + Ok( + // Check that FWSEC has lowered its protection level before reading the + // GFW_BOOT status. + bar.read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK) + .read_protection_level0() + && bar + .read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT) + .completed(), + ) + }, + |&gfw_booted| gfw_booted, + Delta::from_millis(1), + Delta::from_secs(4), + ) + .map(|_| ()) + } +} + +const TU102: Tu102 = Tu102; +pub(super) const TU102_HAL: &dyn GpuHal = &TU102; diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 04a1fa6b25f8..3a609f6937e4 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -17,7 +17,6 @@ mod falcon; mod fb; mod firmware; -mod gfw; mod gpu; mod gsp; #[macro_use] From 11a63a5335eb7b5da4ca38014fa83be5d437144d Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 10 Apr 2026 19:49:32 -0700 Subject: [PATCH 024/131] gpu: nova-core: Hopper/Blackwell: skip GFW boot waiting Hopper and Blackwell GPUs use FSP-based secure boot and do not require waiting for GFW_BOOT completion. Add a Gh100 GPU HAL that returns Ok(()) for wait_gfw_boot_completion(), and route Hopper and Blackwell architectures to it. Signed-off-by: John Hubbard Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260411024953.473149-8-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu/hal.rs | 11 +++++------ drivers/gpu/nova-core/gpu/hal/gh100.rs | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 drivers/gpu/nova-core/gpu/hal/gh100.rs diff --git a/drivers/gpu/nova-core/gpu/hal.rs b/drivers/gpu/nova-core/gpu/hal.rs index a261c6af92be..788de20ab5d3 100644 --- a/drivers/gpu/nova-core/gpu/hal.rs +++ b/drivers/gpu/nova-core/gpu/hal.rs @@ -10,6 +10,7 @@ }, }; +mod gh100; mod tu102; pub(crate) trait GpuHal { @@ -19,11 +20,9 @@ pub(crate) trait GpuHal { pub(super) fn gpu_hal(chipset: Chipset) -> &'static dyn GpuHal { match chipset.arch() { - Architecture::Turing - | Architecture::Ampere - | Architecture::Ada - | Architecture::Hopper - | Architecture::BlackwellGB10x - | Architecture::BlackwellGB20x => tu102::TU102_HAL, + Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL, + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + gh100::GH100_HAL + } } } diff --git a/drivers/gpu/nova-core/gpu/hal/gh100.rs b/drivers/gpu/nova-core/gpu/hal/gh100.rs new file mode 100644 index 000000000000..1ed5bccdda1d --- /dev/null +++ b/drivers/gpu/nova-core/gpu/hal/gh100.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +use kernel::prelude::*; + +use crate::driver::Bar0; + +use super::GpuHal; + +struct Gh100; + +impl GpuHal for Gh100 { + fn wait_gfw_boot_completion(&self, _bar: &Bar0) -> Result { + Ok(()) + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn GpuHal = &GH100; From 6c82e66e5e61331f72f1c25170ce7306e880067c Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Tue, 28 Apr 2026 12:19:28 -0700 Subject: [PATCH 025/131] drm/tyr: move clock cleanup into Clocks Drop impl Currently Tyr disables its clocks from TyrDrmDeviceData::drop(), which causes them to be shut down before any other fields in TyrDrmDeviceData are dropped. This prevents us from using the clocks when dropping the other fields in TyrDrmDeviceData. In order to better control when the clocks are dropped, move this cleanup logic into a Drop implementation on the Clocks struct itself. Since it serves no further purpose, remove the PinnedDrop implementation for TyrDrmDeviceData. Also, while here, remove the #[pin_data] annotation from both the struct Clocks and struct Regulators since neither of these structs need this macro to create structurally pinned fields. Reviewed-by: Boris Brezillon Reviewed-by: Daniel Almeida Reviewed-by: Alice Ryhl Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260428-fw-boot-prerequisites-v1-1-c69af9abe1af@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/driver.rs | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index e8acd2d69468..bb56a9b996a0 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -52,7 +52,7 @@ pub(crate) struct TyrPlatformDriverData { _device: ARef, } -#[pin_data(PinnedDrop)] +#[pin_data] pub(crate) struct TyrDrmDeviceData { pub(crate) pdev: ARef, @@ -157,17 +157,6 @@ impl PinnedDrop for TyrPlatformDriverData { fn drop(self: Pin<&mut Self>) {} } -#[pinned_drop] -impl PinnedDrop for TyrDrmDeviceData { - fn drop(self: Pin<&mut Self>) { - // TODO: the type-state pattern for Clks will fix this. - let clks = self.clks.lock(); - clks.core.disable_unprepare(); - clks.stacks.disable_unprepare(); - clks.coregroup.disable_unprepare(); - } -} - // We need to retain the name "panthor" to achieve drop-in compatibility with // the C driver in the userspace stack. const INFO: drm::DriverInfo = drm::DriverInfo { @@ -191,14 +180,20 @@ impl drm::Driver for TyrDrmDriver { } } -#[pin_data] struct Clocks { core: Clk, stacks: OptionalClk, coregroup: OptionalClk, } -#[pin_data] +impl Drop for Clocks { + fn drop(&mut self) { + self.core.disable_unprepare(); + self.stacks.disable_unprepare(); + self.coregroup.disable_unprepare(); + } +} + struct Regulators { _mali: Regulator, _sram: Regulator, From 6f3096c427e71cdacdfd5cf02907357d5f68cd0e Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Tue, 28 Apr 2026 12:19:29 -0700 Subject: [PATCH 026/131] drm/tyr: rename TyrObject to BoData Currently the GEM inner driver data object is called `TyrObject` which is a fairly generic name. To make the code easier to understand, rename `TyrObject` to `BoData` so that the name better reflects its role. No functional change is intended. Reviewed-by: Daniel Almeida Reviewed-by: Alice Ryhl Signed-off-by: Boris Brezillon Co-developed-by: Deborah Brouwer Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260428-fw-boot-prerequisites-v1-2-c69af9abe1af@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/driver.rs | 4 ++-- drivers/gpu/drm/tyr/gem.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index bb56a9b996a0..bb90324043ed 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -34,7 +34,7 @@ use crate::{ file::TyrDrmFileData, - gem::TyrObject, + gem::BoData, gpu, gpu::GpuInfo, regs::gpu_control::*, // @@ -171,7 +171,7 @@ fn drop(self: Pin<&mut Self>) {} impl drm::Driver for TyrDrmDriver { type Data = TyrDrmDeviceData; type File = TyrDrmFileData; - type Object = drm::gem::Object; + type Object = drm::gem::Object; const INFO: drm::DriverInfo = INFO; diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs index 5cc6eb0b5d3f..66c427a16c31 100644 --- a/drivers/gpu/drm/tyr/gem.rs +++ b/drivers/gpu/drm/tyr/gem.rs @@ -12,13 +12,13 @@ /// GEM Object inner driver data #[pin_data] -pub(crate) struct TyrObject {} +pub(crate) struct BoData {} -impl gem::DriverObject for TyrObject { +impl gem::DriverObject for BoData { type Driver = TyrDrmDriver; type Args = (); fn new(_dev: &TyrDrmDevice, _size: usize, _args: ()) -> impl PinInit { - try_pin_init!(TyrObject {}) + try_pin_init!(BoData {}) } } From 09a796e7069f435d488545910eb1dc7135d30d75 Mon Sep 17 00:00:00 2001 From: Alvin Sun Date: Tue, 28 Apr 2026 12:19:30 -0700 Subject: [PATCH 027/131] drm/tyr: use shmem GEM object type in TyrDrmDriver Tyr buffer objects are shmem-backed, so the driver should use drm::gem::shmem::Object as its GEM object type instead of the base drm::gem::Object type. Switching to the shmem GEM object type matches how Tyr allocates and manages its buffer objects, and uses the shmem-specific GEM abstraction provided by the DRM Rust bindings. Select RUST_DRM_GEM_SHMEM_HELPER to ensure the required helpers are available when DRM_TYR is enabled. Signed-off-by: Alvin Sun Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260428-fw-boot-prerequisites-v1-3-c69af9abe1af@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/Kconfig | 1 + drivers/gpu/drm/tyr/driver.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/tyr/Kconfig b/drivers/gpu/drm/tyr/Kconfig index e933e6478027..51a68ef8212c 100644 --- a/drivers/gpu/drm/tyr/Kconfig +++ b/drivers/gpu/drm/tyr/Kconfig @@ -8,6 +8,7 @@ config DRM_TYR depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE depends on COMMON_CLK default n + select RUST_DRM_GEM_SHMEM_HELPER help Rust DRM driver for ARM Mali CSF-based GPUs. diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index bb90324043ed..cdb9b13bdb32 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -171,7 +171,7 @@ fn drop(self: Pin<&mut Self>) {} impl drm::Driver for TyrDrmDriver { type Data = TyrDrmDeviceData; type File = TyrDrmFileData; - type Object = drm::gem::Object; + type Object = drm::gem::shmem::Object; const INFO: drm::DriverInfo = INFO; From a9bc67761db8cb3207c977210c81be63a539b135 Mon Sep 17 00:00:00 2001 From: Beata Michalska Date: Tue, 28 Apr 2026 12:19:31 -0700 Subject: [PATCH 028/131] drm/tyr: set DMA mask using GPU physical address Configure the device DMA mask during probe using the GPU's physical address capability reported in GpuInfo. This ensures DMA allocations use an appropriate address mask. Reviewed-by: Boris Brezillon Reviewed-by: Daniel Almeida Reviewed-by: Alice Ryhl Signed-off-by: Beata Michalska Co-developed-by: Deborah Brouwer Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260428-fw-boot-prerequisites-v1-4-c69af9abe1af@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/driver.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index cdb9b13bdb32..e20a5978eed6 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -11,6 +11,10 @@ Device, // }, devres::Devres, + dma::{ + Device as DmaDevice, + DmaMask, // + }, drm, drm::ioctl, io::{ @@ -124,6 +128,14 @@ fn probe( let gpu_info = GpuInfo::new(pdev.as_ref(), &iomem)?; gpu_info.log(pdev.as_ref()); + let pa_bits = MMU_FEATURES::from_raw(gpu_info.mmu_features) + .pa_bits() + .get(); + // SAFETY: No concurrent DMA allocations or mappings can be made because + // the device is still being probed and therefore isn't being used by + // other threads of execution. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(pa_bits)?)? }; + let platform: ARef = pdev.into(); let data = try_pin_init!(TyrDrmDeviceData { From 610e892bdb57043c7769982c2bff0260b6007b75 Mon Sep 17 00:00:00 2001 From: Deborah Brouwer Date: Tue, 28 Apr 2026 12:19:32 -0700 Subject: [PATCH 029/131] drm/tyr: add shmem backing for GEM objects Add support for GEM buffer objects backed by shared memory. This introduces the BoCreateArgs structure for passing creation parameters including flags, and adds a flags field to BoData. Co-developed-by: Boris Brezillon Signed-off-by: Boris Brezillon Signed-off-by: Deborah Brouwer Link: https://patch.msgid.link/20260428-fw-boot-prerequisites-v1-5-c69af9abe1af@collabora.com Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/gem.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs index 66c427a16c31..1640a161754b 100644 --- a/drivers/gpu/drm/tyr/gem.rs +++ b/drivers/gpu/drm/tyr/gem.rs @@ -1,4 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 or MIT +//! GEM buffer object management for the Tyr driver. +//! +//! This module provides buffer object (BO) management functionality using +//! DRM's GEM subsystem with shmem backing. use kernel::{ drm::gem, @@ -10,15 +14,23 @@ TyrDrmDriver, // }; -/// GEM Object inner driver data +/// Tyr's DriverObject type for GEM objects. #[pin_data] -pub(crate) struct BoData {} +pub(crate) struct BoData { + flags: u32, +} + +/// Provides a way to pass arguments when creating BoData +/// as required by the gem::DriverObject trait. +pub(crate) struct BoCreateArgs { + flags: u32, +} impl gem::DriverObject for BoData { type Driver = TyrDrmDriver; - type Args = (); + type Args = BoCreateArgs; - fn new(_dev: &TyrDrmDevice, _size: usize, _args: ()) -> impl PinInit { - try_pin_init!(BoData {}) + fn new(_dev: &TyrDrmDevice, _size: usize, args: BoCreateArgs) -> impl PinInit { + try_pin_init!(Self { flags: args.flags }) } } From c2d72717e0a9dd01c6a1bac61a1462a8e04bd179 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 16 Apr 2026 13:10:54 +0000 Subject: [PATCH 030/131] drm/gpuvm: take refcount on DRM device Currently GPUVM relies on the owner implicitly holding a refcount to the drm device, and it does not implicitly take a refcount on the drm device. This design is error-prone, so take a refcount on the device. Suggested-by: Danilo Krummrich Signed-off-by: Alice Ryhl Fixes: 546ca4d35dcc ("drm/gpuvm: convert WARN() to drm_WARN() variants") Link: https://patch.msgid.link/20260416-gpuvm-drm-dev-get-v1-1-f3bc06571e73@google.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/drm_gpuvm.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index 44acfe4120d2..000e7910a899 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -25,6 +25,7 @@ * */ +#include #include #include @@ -1117,6 +1118,7 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, gpuvm->drm = drm; gpuvm->r_obj = r_obj; + drm_dev_get(drm); drm_gem_object_get(r_obj); drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range); @@ -1160,13 +1162,15 @@ static void drm_gpuvm_free(struct kref *kref) { struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref); + struct drm_device *drm = gpuvm->drm; drm_gpuvm_fini(gpuvm); - if (drm_WARN_ON(gpuvm->drm, !gpuvm->ops->vm_free)) + if (drm_WARN_ON(drm, !gpuvm->ops->vm_free)) return; gpuvm->ops->vm_free(gpuvm); + drm_dev_put(drm); } /** From 82b78182eacf82c1847c6f1fd93d91c15efb69cf Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 9 Apr 2026 15:26:06 +0000 Subject: [PATCH 031/131] rust: drm: add base GPUVM immediate mode abstraction Add a GPUVM abstraction to be used by Rust GPU drivers. GPUVM keeps track of a GPU's virtual address (VA) space and manages the corresponding virtual mappings represented by "GPU VA" objects. It also keeps track of the gem::Object used to back the mappings through GpuVmBo. This abstraction is only usable by drivers that wish to use GPUVM in immediate mode. This allows us to build the locking scheme into the API design. It means that the GEM mutex is used for the GEM gpuva list, and that the resv lock is used for the extobj list. The evicted list is not yet used in this version. This abstraction provides a special handle called the UniqueRefGpuVm, which is a wrapper around ARef that provides access to the interval tree. Generally, all changes to the address space requires mutable access to this unique handle. Signed-off-by: Asahi Lina Co-developed-by: Daniel Almeida Signed-off-by: Daniel Almeida Reviewed-by: Daniel Almeida Co-developed-by: Alice Ryhl Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20260409-gpuvm-rust-v6-1-b16e6ada7261@google.com Signed-off-by: Danilo Krummrich --- MAINTAINERS | 2 + rust/bindings/bindings_helper.h | 1 + rust/helpers/drm_gpuvm.c | 20 +++ rust/helpers/helpers.c | 1 + rust/kernel/drm/gpuvm/mod.rs | 260 ++++++++++++++++++++++++++++++++ rust/kernel/drm/mod.rs | 1 + 6 files changed, 285 insertions(+) create mode 100644 rust/helpers/drm_gpuvm.c create mode 100644 rust/kernel/drm/gpuvm/mod.rs diff --git a/MAINTAINERS b/MAINTAINERS index 5c9272622033..fb335dc25f43 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8879,6 +8879,8 @@ S: Supported T: git https://gitlab.freedesktop.org/drm/misc/kernel.git F: drivers/gpu/drm/drm_gpuvm.c F: include/drm/drm_gpuvm.h +F: rust/helpers/drm_gpuvm.c +F: rust/kernel/drm/gpuvm/ DRM LOG M: Jocelyn Falempe diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 446dbeaf0866..1124785e210b 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c new file mode 100644 index 000000000000..18cf104a8bc7 --- /dev/null +++ b/rust/helpers/drm_gpuvm.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +#ifdef CONFIG_DRM_GPUVM + +#include + +__rust_helper +struct drm_gpuvm *rust_helper_drm_gpuvm_get(struct drm_gpuvm *obj) +{ + return drm_gpuvm_get(obj); +} + +__rust_helper +bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, + struct drm_gem_object *obj) +{ + return drm_gpuvm_is_extobj(gpuvm, obj); +} + +#endif // CONFIG_DRM_GPUVM diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 625921e27dfb..4488a87223b9 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -59,6 +59,7 @@ #include "dma.c" #include "dma-resv.c" #include "drm.c" +#include "drm_gpuvm.c" #include "err.c" #include "irq.c" #include "fs.c" diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs new file mode 100644 index 000000000000..1d9138d989b3 --- /dev/null +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +#![cfg(CONFIG_DRM_GPUVM = "y")] + +//! DRM GPUVM in immediate mode +//! +//! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated +//! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM +//! and the GPU's virtual address space has the same state at all times. +//! +//! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) + +use kernel::{ + alloc::AllocError, + bindings, + drm, + drm::gem::IntoGEMObject, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; + +use core::{ + cell::UnsafeCell, + ops::{ + Deref, + Range, // + }, + ptr::NonNull, // +}; + +/// A DRM GPU VA manager. +/// +/// This object is refcounted, but the locations of mapped ranges may only be accessed or changed +/// via the special unique handle [`UniqueRefGpuVm`]. +/// +/// # Invariants +/// +/// * Stored in an allocation managed by the refcount in `self.vm`. +/// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. +/// * Does not contain any sparse `GpuVa` instances. +#[pin_data] +pub struct GpuVm { + #[pin] + vm: Opaque, + /// Accessed only through the [`UniqueRefGpuVm`] reference. + data: UnsafeCell, +} + +// SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will +// drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait. +unsafe impl Send for GpuVm {} +// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel. +unsafe impl Sync for GpuVm {} + +// SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. +unsafe impl AlwaysRefCounted for GpuVm { + fn inc_ref(&self) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. + unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. + unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; + } +} + +impl PartialEq for GpuVm { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl Eq for GpuVm {} + +impl GpuVm { + const fn vtable() -> &'static bindings::drm_gpuvm_ops { + &bindings::drm_gpuvm_ops { + vm_free: Some(Self::vm_free), + op_alloc: None, + op_free: None, + vm_bo_alloc: None, + vm_bo_free: None, + vm_bo_validate: None, + sm_step_map: None, + sm_step_unmap: None, + sm_step_remap: None, + } + } + + /// Creates a GPUVM instance. + #[expect(clippy::new_ret_no_self)] + pub fn new( + name: &'static CStr, + dev: &drm::Device, + r_obj: &T::Object, + range: Range, + reserve_range: Range, + data: T, + ) -> Result, E> + where + E: From, + E: From, + { + let obj = KBox::try_pin_init::( + try_pin_init!(Self { + data: UnsafeCell::new(data), + vm <- Opaque::ffi_init(|vm| { + // SAFETY: These arguments are valid. `vm` is valid until refcount drops to + // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag + // below. + unsafe { + bindings::drm_gpuvm_init( + vm, + name.as_char_ptr(), + bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE + | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, + dev.as_raw(), + r_obj.as_raw(), + range.start, + range.end - range.start, + reserve_range.start, + reserve_range.end - reserve_range.start, + const { Self::vtable() }, + ) + } + }), + }? E), + GFP_KERNEL | __GFP_ZERO, + )?; + // SAFETY: This transfers the initial refcount to the ARef. + let aref = unsafe { + ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( + Pin::into_inner_unchecked(obj), + ))) + }; + // INVARIANT: This reference is unique. + Ok(UniqueRefGpuVm(aref)) + } + + /// Access this [`GpuVm`] from a raw pointer. + /// + /// # Safety + /// + /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm`] that remains + /// valid for at least `'a`. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { + // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm`. Caller ensures the + // pointer is valid for 'a. + unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } + } + + /// Returns a raw pointer to the embedded `struct drm_gpuvm`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { + self.vm.get() + } + + /// The start of the VA space. + #[inline] + pub fn va_start(&self) -> u64 { + // SAFETY: The `mm_start` field is immutable. + unsafe { (*self.as_raw()).mm_start } + } + + /// The length of the GPU's virtual address space. + #[inline] + pub fn va_length(&self) -> u64 { + // SAFETY: The `mm_range` field is immutable. + unsafe { (*self.as_raw()).mm_range } + } + + /// Returns the range of the GPU virtual address space. + #[inline] + pub fn va_range(&self) -> Range { + let start = self.va_start(); + // OVERFLOW: This reconstructs the Range passed to the constructor, so it won't fail. + let end = start + self.va_length(); + Range { start, end } + } + + /// Clean up buffer objects that are no longer used. + #[inline] + pub fn deferred_cleanup(&self) { + // SAFETY: This GPUVM uses immediate mode. + unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } + } + + /// Check if this GEM object is an external object for this GPUVM. + #[inline] + pub fn is_extobj(&self, obj: &T::Object) -> bool { + // SAFETY: We may call this with any GPUVM and GEM object. + unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } + } + + /// Free this GPUVM. + /// + /// # Safety + /// + /// Called when refcount hits zero. + unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { + // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm`. + let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; + // SAFETY: By type invariants we can free it when refcount hits zero. + drop(unsafe { KBox::from_raw(me) }) + } +} + +/// The manager for a GPUVM. +pub trait DriverGpuVm: Sized + Send { + /// Parent `Driver` for this object. + type Driver: drm::Driver; + + /// The kind of GEM object stored in this GPUVM. + type Object: IntoGEMObject; +} + +/// The core of the DRM GPU VA manager. +/// +/// This object is a unique reference to the VM that can access the interval tree and the Rust +/// `data` field. +/// +/// # Invariants +/// +/// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. +pub struct UniqueRefGpuVm(ARef>); + +// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and +// concurrent access to `data` is safe due to the `T: Sync` requirement. +unsafe impl Sync for UniqueRefGpuVm {} + +impl UniqueRefGpuVm { + /// Access the data owned by this `UniqueRefGpuVm` immutably. + #[inline] + pub fn data_ref(&self) -> &T { + // SAFETY: By the type invariants we may access `data`. + unsafe { &*self.0.data.get() } + } + + /// Access the data owned by this `UniqueRefGpuVm` mutably. + #[inline] + pub fn data(&mut self) -> &mut T { + // SAFETY: By the type invariants we may access `data`. + unsafe { &mut *self.0.data.get() } + } +} + +impl Deref for UniqueRefGpuVm { + type Target = GpuVm; + + #[inline] + fn deref(&self) -> &GpuVm { + &self.0 + } +} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index 1b82b6945edf..a4b6c5430198 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -6,6 +6,7 @@ pub mod driver; pub mod file; pub mod gem; +pub mod gpuvm; pub mod ioctl; pub use self::device::Device; From 71a3921111bd05298988fad1c58241db13384ea7 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 9 Apr 2026 15:26:07 +0000 Subject: [PATCH 032/131] rust: gpuvm: add GpuVm::obtain() This provides a mechanism to create (or look up) VMBO instances, which represent the mapping between GPUVM and GEM objects. The GpuVmBoRegistered type can be considered like ARef>, except that no way to increment the refcount is provided. The GpuVmBoAlloc type is more akin to a pre-allocated GpuVmBo, so it's not really a GpuVmBo yet. Its destructor could call drm_gpuvm_bo_destroy_not_in_lists(), but as the type is currently private and never called anywhere, this perf optimization does not need to happen now. Pre-allocating and obtaining the gpuvm_bo object is exposed as a single step. This could theoretically be a problem if one wanted to call drm_gpuvm_bo_obtain_prealloc() during the fence signalling critical path, but that's not a possibility because: 1. Adding the BO to the extobj list requires the resv lock, so it cannot happen during the fence signalling critical path. 2. obtain() requires that the BO is not in the extobj list, so obtain() must be called before adding the BO to the extobj list. Thus, drm_gpuvm_bo_obtain_prealloc() cannot be called during the fence signalling critical path. (For extobjs at least.) Reviewed-by: Daniel Almeida Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20260409-gpuvm-rust-v6-2-b16e6ada7261@google.com Signed-off-by: Danilo Krummrich --- rust/helpers/drm_gpuvm.c | 6 + rust/kernel/drm/gpuvm/mod.rs | 32 ++++- rust/kernel/drm/gpuvm/vm_bo.rs | 242 +++++++++++++++++++++++++++++++++ 3 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 rust/kernel/drm/gpuvm/vm_bo.rs diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c index 18cf104a8bc7..ca959d9a66f6 100644 --- a/rust/helpers/drm_gpuvm.c +++ b/rust/helpers/drm_gpuvm.c @@ -4,6 +4,12 @@ #include +__rust_helper +struct drm_gpuvm_bo *rust_helper_drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) +{ + return drm_gpuvm_bo_get(vm_bo); +} + __rust_helper struct drm_gpuvm *rust_helper_drm_gpuvm_get(struct drm_gpuvm *obj) { diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index 1d9138d989b3..56e02b49a581 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -25,13 +25,20 @@ use core::{ cell::UnsafeCell, + mem::ManuallyDrop, ops::{ Deref, Range, // }, - ptr::NonNull, // + ptr::{ + self, + NonNull, // + }, // }; +mod vm_bo; +pub use self::vm_bo::*; + /// A DRM GPU VA manager. /// /// This object is refcounted, but the locations of mapped ranges may only be accessed or changed @@ -83,8 +90,8 @@ const fn vtable() -> &'static bindings::drm_gpuvm_ops { vm_free: Some(Self::vm_free), op_alloc: None, op_free: None, - vm_bo_alloc: None, - vm_bo_free: None, + vm_bo_alloc: GpuVmBo::::ALLOC_FN, + vm_bo_free: GpuVmBo::::FREE_FN, vm_bo_validate: None, sm_step_map: None, sm_step_unmap: None, @@ -184,6 +191,16 @@ pub fn va_range(&self) -> Range { Range { start, end } } + /// Get or create the [`GpuVmBo`] for this gem object. + #[inline] + pub fn obtain( + &self, + obj: &T::Object, + data: impl PinInit, + ) -> Result>, AllocError> { + Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain()) + } + /// Clean up buffer objects that are no longer used. #[inline] pub fn deferred_cleanup(&self) { @@ -209,6 +226,12 @@ pub fn is_extobj(&self, obj: &T::Object) -> bool { // SAFETY: By type invariants we can free it when refcount hits zero. drop(unsafe { KBox::from_raw(me) }) } + + #[inline] + fn raw_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `r_obj` is immutable and valid for duration of GPUVM. + unsafe { (*(*self.as_raw()).r_obj).resv } + } } /// The manager for a GPUVM. @@ -218,6 +241,9 @@ pub trait DriverGpuVm: Sized + Send { /// The kind of GEM object stored in this GPUVM. type Object: IntoGEMObject; + + /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). + type VmBoData; } /// The core of the DRM GPU VA manager. diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs new file mode 100644 index 000000000000..65f03f93bd21 --- /dev/null +++ b/rust/kernel/drm/gpuvm/vm_bo.rs @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// Represents that a given GEM object has at least one mapping on this [`GpuVm`] instance. +/// +/// Does not assume that GEM lock is held. +/// +/// # Invariants +/// +/// * Allocated with `kmalloc` and refcounted via `inner`. +/// * Is present in the gem list. +#[repr(C)] +#[pin_data] +pub struct GpuVmBo { + #[pin] + inner: Opaque, + #[pin] + data: T::VmBoData, +} + +// SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. +unsafe impl AlwaysRefCounted for GpuVmBo { + fn inc_ref(&self) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. + unsafe { bindings::drm_gpuvm_bo_get(self.inner.get()) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // CAST: `drm_gpuvm_bo` is first field of repr(C) struct. + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. + // This GPUVM instance uses immediate mode, so we may put the refcount using the deferred + // mechanism. + unsafe { bindings::drm_gpuvm_bo_put_deferred(obj.as_ptr().cast()) }; + } +} + +impl PartialEq for GpuVmBo { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl Eq for GpuVmBo {} + +impl GpuVmBo { + /// The function pointer for allocating a GpuVmBo stored in the gpuvm vtable. + /// + /// Allocation is always implemented according to [`Self::vm_bo_alloc`], but it is set to + /// `None` if the default gpuvm behavior is the same as `vm_bo_alloc`. + /// + /// This may be `Some` even if `FREE_FN` is `None`, or vice-versa. + pub(super) const ALLOC_FN: Option *mut bindings::drm_gpuvm_bo> = { + use core::alloc::Layout; + let base = Layout::new::(); + let rust = Layout::new::(); + assert!(base.size() <= rust.size()); + if base.size() != rust.size() || base.align() != rust.align() { + Some(Self::vm_bo_alloc) + } else { + // This causes GPUVM to allocate a `GpuVmBo` with `kzalloc(sizeof(drm_gpuvm_bo))`. + None + } + }; + + /// The function pointer for freeing a GpuVmBo stored in the gpuvm vtable. + /// + /// Freeing is always implemented according to [`Self::vm_bo_free`], but it is set to `None` if + /// the default gpuvm behavior is the same as `vm_bo_free`. + /// + /// This may be `Some` even if `ALLOC_FN` is `None`, or vice-versa. + pub(super) const FREE_FN: Option = { + if core::mem::needs_drop::() { + Some(Self::vm_bo_free) + } else { + // This causes GPUVM to free a `GpuVmBo` with `kfree`. + None + } + }; + + /// Custom function for allocating a `drm_gpuvm_bo`. + /// + /// # Safety + /// + /// Always safe to call. + unsafe extern "C" fn vm_bo_alloc() -> *mut bindings::drm_gpuvm_bo { + let raw_ptr = KBox::::new_uninit(GFP_KERNEL | __GFP_ZERO) + .map(KBox::into_raw) + .unwrap_or(ptr::null_mut()); + + // CAST: `drm_gpuvm_bo` is first field of `Self`. + raw_ptr.cast() + } + + /// Custom function for freeing a `drm_gpuvm_bo`. + /// + /// # Safety + /// + /// The pointer must have been allocated with [`GpuVmBo::ALLOC_FN`], and must not be used after + /// this call. + unsafe extern "C" fn vm_bo_free(ptr: *mut bindings::drm_gpuvm_bo) { + // CAST: `drm_gpuvm_bo` is first field of `Self`. + // SAFETY: + // * The ptr was allocated from kmalloc with the layout of `GpuVmBo`. + // * `ptr->inner` has no destructor. + // * `ptr->data` contains a valid `T::VmBoData` that we can drop. + drop(unsafe { KBox::::from_raw(ptr.cast()) }); + } + + /// Access this [`GpuVmBo`] from a raw pointer. + /// + /// # Safety + /// + /// For the duration of `'a`, the pointer must reference a valid `drm_gpuvm_bo` associated with + /// a [`GpuVm`]. The BO must also be present in the GEM list. + #[inline] + #[expect(dead_code)] + pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm_bo) -> &'a Self { + // SAFETY: `drm_gpuvm_bo` is first field and `repr(C)`. + unsafe { &*ptr.cast() } + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo { + self.inner.get() + } + + /// The [`GpuVm`] that this GEM object is mapped in. + #[inline] + pub fn gpuvm(&self) -> &GpuVm { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { GpuVm::::from_raw((*self.inner.get()).vm) } + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) for these mappings. + #[inline] + pub fn obj(&self) -> &T::Object { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { ::from_raw((*self.inner.get()).obj) } + } + + /// The driver data with this buffer object. + #[inline] + pub fn data(&self) -> &T::VmBoData { + &self.data + } +} + +/// A pre-allocated [`GpuVmBo`] object. +/// +/// # Invariants +/// +/// Points at a `drm_gpuvm_bo` that contains a valid `T::VmBoData`, has a refcount of one, and is +/// absent from any gem, extobj, or evict lists. +pub(super) struct GpuVmBoAlloc(NonNull>); + +impl GpuVmBoAlloc { + /// Create a new pre-allocated [`GpuVmBo`]. + /// + /// It's intentional that the initializer is infallible because `drm_gpuvm_bo_put` will call + /// drop on the data, so we don't have a way to free it when the data is missing. + #[inline] + pub(super) fn new( + gpuvm: &GpuVm, + gem: &T::Object, + value: impl PinInit, + ) -> Result, AllocError> { + // CAST: `GpuVmBoAlloc::vm_bo_alloc` ensures that this memory was allocated with the layout + // of `GpuVmBo`. The type is repr(C), so `container_of` is not required. + // SAFETY: The provided gpuvm and gem ptrs are valid for the duration of this call. + let raw_ptr = unsafe { + bindings::drm_gpuvm_bo_create(gpuvm.as_raw(), gem.as_raw()).cast::>() + }; + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + // SAFETY: `ptr->data` is a valid pinned location. + let Ok(()) = unsafe { value.__pinned_init(&raw mut (*raw_ptr).data) }; + // INVARIANTS: We just created the vm_bo so it's absent from lists, and the data is valid + // as we just initialized it. + Ok(GpuVmBoAlloc(ptr)) + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub(super) fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo { + // SAFETY: The pointer references a valid `drm_gpuvm_bo`. + unsafe { (*self.0.as_ptr()).inner.get() } + } + + /// Look up whether there is an existing [`GpuVmBo`] for this gem object. + /// + /// The caller should not hold the GEM mutex or DMA resv lock. + #[inline] + pub(super) fn obtain(self) -> ARef> { + let me = ManuallyDrop::new(self); + // SAFETY: Valid `drm_gpuvm_bo` not already in the lists. We do not access `me` after this + // call. + let ptr = unsafe { bindings::drm_gpuvm_bo_obtain_prealloc(me.as_raw()) }; + + // SAFETY: `drm_gpuvm_bo_obtain_prealloc` always returns a non-null ptr + let nonnull = unsafe { NonNull::new_unchecked(ptr.cast()) }; + + // INVARIANTS: `drm_gpuvm_bo_obtain_prealloc` ensures that the bo is in the GEM list. + // SAFETY: We received one refcount from `drm_gpuvm_bo_obtain_prealloc`. + let ret = unsafe { ARef::>::from_raw(nonnull) }; + + // Ensure that external objects are in the extobj list. + // + // Note that we must call `extobj_add` even if `ptr != me` to avoid a race condition where + // we could end up using the extobj before the thread with `ptr == me` calls extobj_add. + if ret.gpuvm().is_extobj(ret.obj()) { + let resv_lock = ret.gpuvm().raw_resv(); + // TODO: Use a proper lock guard here once a dma_resv lock abstraction exists. + // SAFETY: The GPUVM is still alive, so its resv lock is too. + unsafe { bindings::dma_resv_lock(resv_lock, ptr::null_mut()) }; + // SAFETY: We hold the GPUVMs resv lock. + unsafe { bindings::drm_gpuvm_bo_extobj_add(ptr) }; + // SAFETY: We took the lock, so we can unlock it. + unsafe { bindings::dma_resv_unlock(resv_lock) }; + } + + ret + } +} + +impl Deref for GpuVmBoAlloc { + type Target = GpuVmBo; + #[inline] + fn deref(&self) -> &GpuVmBo { + // SAFETY: By the type invariants we may deref while `Self` exists. + unsafe { self.0.as_ref() } + } +} + +impl Drop for GpuVmBoAlloc { + #[inline] + fn drop(&mut self) { + // TODO: Call drm_gpuvm_bo_destroy_not_in_lists() directly. + // SAFETY: It's safe to perform a deferred put in any context. + unsafe { bindings::drm_gpuvm_bo_put_deferred(self.as_raw()) }; + } +} From 5540a9c747b3e1914ae68ae89f71c0a779bee5d1 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 9 Apr 2026 15:26:08 +0000 Subject: [PATCH 033/131] rust: gpuvm: add GpuVa struct This struct will be used to keep track of individual mapped ranges in the GPU's virtual memory. Sparse VAs are not yet supported. Co-developed-by: Asahi Lina Signed-off-by: Asahi Lina Co-developed-by: Daniel Almeida Signed-off-by: Daniel Almeida Reviewed-by: Daniel Almeida Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20260409-gpuvm-rust-v6-3-b16e6ada7261@google.com Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gpuvm/mod.rs | 19 +++- rust/kernel/drm/gpuvm/va.rs | 169 +++++++++++++++++++++++++++++++++ rust/kernel/drm/gpuvm/vm_bo.rs | 1 - 3 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 rust/kernel/drm/gpuvm/va.rs diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index 56e02b49a581..78951e8aa5d3 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -11,7 +11,10 @@ //! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) use kernel::{ - alloc::AllocError, + alloc::{ + AllocError, + Flags as AllocFlags, // + }, bindings, drm, drm::gem::IntoGEMObject, @@ -25,9 +28,13 @@ use core::{ cell::UnsafeCell, - mem::ManuallyDrop, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, ops::{ Deref, + DerefMut, Range, // }, ptr::{ @@ -36,6 +43,9 @@ }, // }; +mod va; +pub use self::va::*; + mod vm_bo; pub use self::vm_bo::*; @@ -48,7 +58,7 @@ /// /// * Stored in an allocation managed by the refcount in `self.vm`. /// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. -/// * Does not contain any sparse `GpuVa` instances. +/// * Does not contain any sparse [`GpuVa`] instances. #[pin_data] pub struct GpuVm { #[pin] @@ -242,6 +252,9 @@ pub trait DriverGpuVm: Sized + Send { /// The kind of GEM object stored in this GPUVM. type Object: IntoGEMObject; + /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). + type VaData; + /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). type VmBoData; } diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs new file mode 100644 index 000000000000..227c259f7db9 --- /dev/null +++ b/rust/kernel/drm/gpuvm/va.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +#![expect(dead_code)] +use super::*; + +/// Represents that a range of a GEM object is mapped in this [`GpuVm`] instance. +/// +/// Does not assume that GEM lock is held. +/// +/// # Invariants +/// +/// * This is a valid `drm_gpuva` object that is resident in a [`GpuVm`] instance. +/// * It is associated with a [`GpuVmBo`]. Or in other words, it's not an +/// `gpuvm->kernel_alloc_node` and `DRM_GPUVA_SPARSE` is not set. +/// * The associated [`GpuVmBo`] is part of the GEM list. +#[repr(C)] +#[pin_data] +pub struct GpuVa { + #[pin] + inner: Opaque, + #[pin] + data: T::VaData, +} + +impl PartialEq for GpuVa { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl Eq for GpuVa {} + +impl GpuVa { + /// Access this [`GpuVa`] from a raw pointer. + /// + /// # Safety + /// + /// * For the duration of `'a`, the pointer must reference a valid `drm_gpuva` associated with + /// a [`GpuVm`]. + /// * It must be associated with a [`GpuVmBo`]. + /// * The associated [`GpuVmBo`] is part of the GEM list. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuva) -> &'a Self { + // CAST: `drm_gpuva` is first field and `repr(C)`. + // SAFETY: The safety requirements match the invariants of `GpuVa`. + unsafe { &*ptr.cast() } + } + + /// Returns a raw pointer to underlying C value. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuva { + self.inner.get() + } + + /// Returns the address of this mapping in the GPU virtual address space. + #[inline] + pub fn addr(&self) -> u64 { + // SAFETY: The `va.addr` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).va.addr } + } + + /// Returns the length of this mapping. + #[inline] + pub fn length(&self) -> u64 { + // SAFETY: The `va.range` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).va.range } + } + + /// Returns `addr..addr+length`. + #[inline] + pub fn range(&self) -> Range { + let addr = self.addr(); + addr..addr + self.length() + } + + /// Returns the offset within the GEM object. + #[inline] + pub fn gem_offset(&self) -> u64 { + // SAFETY: The `gem.offset` field of `drm_gpuva` is immutable. + unsafe { (*self.as_raw()).gem.offset } + } + + /// Returns the GEM object. + #[inline] + pub fn obj(&self) -> &T::Object { + // SAFETY: The `gem.obj` field of `drm_gpuva` is immutable. We know that it's not null + // because this VA is associated with a `GpuVmBo`. + unsafe { ::from_raw((*self.as_raw()).gem.obj) } + } + + /// Returns the underlying [`GpuVmBo`] object that backs this [`GpuVa`]. + #[inline] + pub fn vm_bo(&self) -> &GpuVmBo { + // SAFETY: The `vm_bo` field of `drm_gpuva` is immutable. We know that it's not null + // because this VA is associated with a `GpuVmBo`. The BO is in the GEM list by the type + // invariants. + unsafe { GpuVmBo::from_raw((*self.as_raw()).vm_bo) } + } +} + +/// A pre-allocated [`GpuVa`] object. +/// +/// # Invariants +/// +/// The memory is zeroed. +pub struct GpuVaAlloc(KBox>>); + +impl GpuVaAlloc { + /// Pre-allocate a [`GpuVa`] object. + pub fn new(flags: AllocFlags) -> Result, AllocError> { + // INVARIANTS: Memory allocated with __GFP_ZERO. + Ok(GpuVaAlloc(KBox::new_uninit(flags | __GFP_ZERO)?)) + } + + /// Prepare this `drm_gpuva` for insertion into the GPUVM. + #[must_use] + pub(super) fn prepare(mut self, va_data: impl PinInit) -> *mut bindings::drm_gpuva { + let va_ptr = MaybeUninit::as_mut_ptr(&mut self.0); + // SAFETY: The `data` field is pinned. + let Ok(()) = unsafe { va_data.__pinned_init(&raw mut (*va_ptr).data) }; + KBox::into_raw(self.0).cast() + } +} + +/// A [`GpuVa`] object that has been removed. +/// +/// # Invariants +/// +/// The `drm_gpuva` is not resident in the [`GpuVm`]. +pub struct GpuVaRemoved(KBox>); + +impl GpuVaRemoved { + /// Convert a raw pointer into a [`GpuVaRemoved`]. + /// + /// # Safety + /// + /// * Must have been removed from a [`GpuVm`]. + /// * It must not be a `gpuvm->kernel_alloc_node` va. + pub(super) unsafe fn from_raw(ptr: *mut bindings::drm_gpuva) -> Self { + // SAFETY: Since it used to be a VA in a `GpuVm` and it's not a kernel_alloc_node, this + // pointer references a `GpuVa` with a valid `T::VaData`. Since it has been removed, we + // can take ownership of the allocation. + GpuVaRemoved(unsafe { KBox::from_raw(ptr.cast()) }) + } + + /// Take ownership of the VA data. + pub fn into_inner(self) -> T::VaData + where + T::VaData: Unpin, + { + KBox::into_inner(self.0).data + } +} + +impl Deref for GpuVaRemoved { + type Target = T::VaData; + fn deref(&self) -> &T::VaData { + &self.0.data + } +} + +impl DerefMut for GpuVaRemoved +where + T::VaData: Unpin, +{ + fn deref_mut(&mut self) -> &mut T::VaData { + &mut self.0.data + } +} diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs index 65f03f93bd21..05fd7998f4bd 100644 --- a/rust/kernel/drm/gpuvm/vm_bo.rs +++ b/rust/kernel/drm/gpuvm/vm_bo.rs @@ -114,7 +114,6 @@ impl GpuVmBo { /// For the duration of `'a`, the pointer must reference a valid `drm_gpuvm_bo` associated with /// a [`GpuVm`]. The BO must also be present in the GEM list. #[inline] - #[expect(dead_code)] pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm_bo) -> &'a Self { // SAFETY: `drm_gpuvm_bo` is first field and `repr(C)`. unsafe { &*ptr.cast() } From dc3846045f694eef0606697e2304099cba403691 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 9 Apr 2026 15:26:09 +0000 Subject: [PATCH 034/131] rust: gpuvm: add GpuVmCore::sm_unmap() Add the entrypoint for unmapping ranges in the GPUVM, and provide callbacks and VA types for the implementation. Co-developed-by: Asahi Lina Signed-off-by: Asahi Lina Reviewed-by: Daniel Almeida Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20260409-gpuvm-rust-v6-4-b16e6ada7261@google.com Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gpuvm/mod.rs | 30 +++- rust/kernel/drm/gpuvm/sm_ops.rs | 272 ++++++++++++++++++++++++++++++++ rust/kernel/drm/gpuvm/va.rs | 1 - rust/kernel/drm/gpuvm/vm_bo.rs | 8 + 4 files changed, 306 insertions(+), 5 deletions(-) create mode 100644 rust/kernel/drm/gpuvm/sm_ops.rs diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index 78951e8aa5d3..a6436abd0f9c 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -18,6 +18,7 @@ bindings, drm, drm::gem::IntoGEMObject, + error::to_result, prelude::*, sync::aref::{ ARef, @@ -28,6 +29,7 @@ use core::{ cell::UnsafeCell, + marker::PhantomData, mem::{ ManuallyDrop, MaybeUninit, // @@ -43,12 +45,15 @@ }, // }; -mod va; -pub use self::va::*; +mod sm_ops; +pub use self::sm_ops::*; mod vm_bo; pub use self::vm_bo::*; +mod va; +pub use self::va::*; + /// A DRM GPU VA manager. /// /// This object is refcounted, but the locations of mapped ranges may only be accessed or changed @@ -104,8 +109,8 @@ const fn vtable() -> &'static bindings::drm_gpuvm_ops { vm_bo_free: GpuVmBo::::FREE_FN, vm_bo_validate: None, sm_step_map: None, - sm_step_unmap: None, - sm_step_remap: None, + sm_step_unmap: Some(Self::sm_step_unmap), + sm_step_remap: Some(Self::sm_step_remap), } } @@ -257,6 +262,23 @@ pub trait DriverGpuVm: Sized + Send { /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). type VmBoData; + + /// The private data passed to callbacks. + type SmContext<'ctx>; + + /// Indicates that an existing mapping should be removed. + fn sm_step_unmap<'op, 'ctx>( + &mut self, + op: OpUnmap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result, Error>; + + /// Indicates that an existing mapping should be split up. + fn sm_step_remap<'op, 'ctx>( + &mut self, + op: OpRemap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result, Error>; } /// The core of the DRM GPU VA manager. diff --git a/rust/kernel/drm/gpuvm/sm_ops.rs b/rust/kernel/drm/gpuvm/sm_ops.rs new file mode 100644 index 000000000000..05f81c638aef --- /dev/null +++ b/rust/kernel/drm/gpuvm/sm_ops.rs @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +use super::*; + +/// The actual data that gets threaded through the callbacks. +struct SmData<'a, 'ctx, T: DriverGpuVm> { + gpuvm: &'a mut UniqueRefGpuVm, + user_context: &'a mut T::SmContext<'ctx>, +} + +/// Represents an `sm_step_unmap` operation that has not yet been completed. +pub struct OpUnmap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_unmap, + // This ensures that 'op is invariant, so that `OpUnmap<'long, T>` does not + // coerce to `OpUnmap<'short, T>`. This ensures that the user can't return the + // wrong`OpUnmapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpUnmap<'op, T> { + /// Indicates whether this [`GpuVa`] is physically contiguous with the + /// original mapping request. + /// + /// Optionally, if `keep` is set, drivers may keep the actual page table + /// mappings for this `drm_gpuva`, adding the missing page table entries + /// only and update the `drm_gpuvm` accordingly. + pub fn keep(&self) -> bool { + self.op.keep + } + + /// The range being unmapped. + pub fn va(&self) -> &GpuVa { + // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it, + // and it's not sparse by the `GpuVm` type invariants. + unsafe { GpuVa::::from_raw(self.op.va) } + } + + /// Remove the VA. + pub fn remove(self) -> (OpUnmapped<'op, T>, GpuVaRemoved) { + // SAFETY: The op references a valid drm_gpuva in the GPUVM. + unsafe { bindings::drm_gpuva_unmap(self.op) }; + // SAFETY: The va is no longer in the interval tree so we may unlink it. + unsafe { bindings::drm_gpuva_unlink_defer(self.op.va) }; + + // SAFETY: We just removed this va from the `GpuVm`. + let va = unsafe { GpuVaRemoved::from_raw(self.op.va) }; + + ( + OpUnmapped { + _invariant: self._invariant, + }, + va, + ) + } +} + +/// Represents a completed [`OpUnmap`] operation. +pub struct OpUnmapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +/// Represents an `sm_step_remap` operation that has not yet been completed. +pub struct OpRemap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_remap, + // This ensures that 'op is invariant, so that `OpRemap<'long, T>` does not + // coerce to `OpRemap<'short, T>`. This ensures that the user can't return the + // wrong`OpRemapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpRemap<'op, T> { + /// The preceding part of a split mapping. + #[inline] + pub fn prev(&self) -> Option<&OpRemapMapData> { + // SAFETY: We checked for null, so the pointer must be valid. + NonNull::new(self.op.prev).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) }) + } + + /// The subsequent part of a split mapping. + #[inline] + pub fn next(&self) -> Option<&OpRemapMapData> { + // SAFETY: We checked for null, so the pointer must be valid. + NonNull::new(self.op.next).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) }) + } + + /// Indicates whether the `drm_gpuva` being removed is physically contiguous with the original + /// mapping request. + /// + /// Optionally, if `keep` is set, drivers may keep the actual page table mappings for this + /// `drm_gpuva`, adding the missing page table entries only and update the `drm_gpuvm` + /// accordingly. + #[inline] + pub fn keep(&self) -> bool { + // SAFETY: The unmap pointer is always valid. + unsafe { (*self.op.unmap).keep } + } + + /// The range being unmapped. + #[inline] + pub fn va_to_unmap(&self) -> &GpuVa { + // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it, + // and it's not sparse by the `GpuVm` type invariants. + unsafe { GpuVa::::from_raw((*self.op.unmap).va) } + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) whose VA is being remapped. + #[inline] + pub fn obj(&self) -> &T::Object { + self.va_to_unmap().obj() + } + + /// The [`GpuVmBo`] that is being remapped. + #[inline] + pub fn vm_bo(&self) -> &GpuVmBo { + self.va_to_unmap().vm_bo() + } + + /// Update the GPUVM to perform the remapping. + pub fn remap( + self, + va_alloc: [GpuVaAlloc; 2], + prev_data: impl PinInit, + next_data: impl PinInit, + ) -> (OpRemapped<'op, T>, OpRemapRet) { + let [va1, va2] = va_alloc; + + let mut unused_va = None; + let mut prev_ptr = ptr::null_mut(); + let mut next_ptr = ptr::null_mut(); + if self.prev().is_some() { + prev_ptr = va1.prepare(prev_data); + } else { + unused_va = Some(va1); + } + if self.next().is_some() { + next_ptr = va2.prepare(next_data); + } else { + unused_va = Some(va2); + } + + // SAFETY: the pointers are non-null when required + unsafe { bindings::drm_gpuva_remap(prev_ptr, next_ptr, self.op) }; + + let gpuva_guard = self.vm_bo().lock_gpuva(); + if !prev_ptr.is_null() { + // SAFETY: The prev_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still + // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo. + unsafe { bindings::drm_gpuva_link(prev_ptr, self.vm_bo().as_raw()) }; + } + if !next_ptr.is_null() { + // SAFETY: The next_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still + // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo. + unsafe { bindings::drm_gpuva_link(next_ptr, self.vm_bo().as_raw()) }; + } + drop(gpuva_guard); + + // SAFETY: The va is no longer in the interval tree so we may unlink it. + unsafe { bindings::drm_gpuva_unlink_defer((*self.op.unmap).va) }; + + ( + OpRemapped { + _invariant: self._invariant, + }, + OpRemapRet { + // SAFETY: We just removed this va from the `GpuVm`. + unmapped_va: unsafe { GpuVaRemoved::from_raw((*self.op.unmap).va) }, + unused_va, + }, + ) + } +} + +/// Part of an [`OpRemap`] that represents a new mapping. +#[repr(transparent)] +pub struct OpRemapMapData(bindings::drm_gpuva_op_map); + +impl OpRemapMapData { + /// # Safety + /// Must reference a valid `drm_gpuva_op_map` for duration of `'a`. + unsafe fn from_raw<'a>(ptr: NonNull) -> &'a Self { + // SAFETY: ok per safety requirements + unsafe { ptr.cast().as_ref() } + } + + /// The base address of the new mapping. + pub fn addr(&self) -> u64 { + self.0.va.addr + } + + /// The length of the new mapping. + pub fn length(&self) -> u64 { + self.0.va.range + } + + /// The offset within the [`drm_gem_object`](DriverGpuVm::Object). + pub fn gem_offset(&self) -> u64 { + self.0.gem.offset + } +} + +/// Struct containing objects removed or not used by [`OpRemap::remap`]. +pub struct OpRemapRet { + /// The `drm_gpuva` that was removed. + pub unmapped_va: GpuVaRemoved, + /// If the remap did not split the region into two pieces, then the unused `drm_gpuva` is + /// returned here. + pub unused_va: Option>, +} + +/// Represents a completed [`OpRemap`] operation. +pub struct OpRemapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + +impl UniqueRefGpuVm { + /// Remove any mappings in the given region. + /// + /// Internally calls [`DriverGpuVm::sm_step_unmap`] for ranges entirely contained within the + /// given range, and [`DriverGpuVm::sm_step_remap`] for ranges that overlap with the range. + #[inline] + pub fn sm_unmap(&mut self, addr: u64, length: u64, context: &mut T::SmContext<'_>) -> Result { + let gpuvm = self.as_raw(); + let mut p = SmData { + gpuvm: self, + user_context: context, + }; + // SAFETY: + // * raw_request() creates a valid request. + // * The private data is valid to be interpreted as SmData. + to_result(unsafe { bindings::drm_gpuvm_sm_unmap(gpuvm, (&raw mut p).cast(), addr, length) }) + } +} + +impl GpuVm { + /// # Safety + /// Must be called from `sm_unmap` with a pointer to `SmData`. + pub(super) unsafe extern "C" fn sm_step_unmap( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: The caller provides a pointer to `SmData`. + let p = unsafe { &mut *p.cast::>() }; + let op = OpUnmap { + // SAFETY: sm_step_unmap is called with an unmap operation. + op: unsafe { &(*op).__bindgen_anon_1.unmap }, + _invariant: PhantomData, + }; + match p.gpuvm.data().sm_step_unmap(op, p.user_context) { + Ok(OpUnmapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } + + /// # Safety + /// Must be called from `sm_unmap` with a pointer to `SmData`. + pub(super) unsafe extern "C" fn sm_step_remap( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: The caller provides a pointer to `SmData`. + let p = unsafe { &mut *p.cast::>() }; + let op = OpRemap { + // SAFETY: sm_step_remap is called with a remap operation. + op: unsafe { &(*op).__bindgen_anon_1.remap }, + _invariant: PhantomData, + }; + match p.gpuvm.data().sm_step_remap(op, p.user_context) { + Ok(OpRemapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } +} diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs index 227c259f7db9..0b09fe44ab39 100644 --- a/rust/kernel/drm/gpuvm/va.rs +++ b/rust/kernel/drm/gpuvm/va.rs @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 OR MIT -#![expect(dead_code)] use super::*; /// Represents that a range of a GEM object is mapped in this [`GpuVm`] instance. diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs index 05fd7998f4bd..c064ac63897b 100644 --- a/rust/kernel/drm/gpuvm/vm_bo.rs +++ b/rust/kernel/drm/gpuvm/vm_bo.rs @@ -144,6 +144,14 @@ pub fn obj(&self) -> &T::Object { pub fn data(&self) -> &T::VmBoData { &self.data } + + pub(super) fn lock_gpuva(&self) -> crate::sync::MutexGuard<'_, ()> { + // SAFETY: The GEM object is valid. + let ptr = unsafe { &raw mut (*self.obj().as_raw()).gpuva.lock }; + // SAFETY: The GEM object is valid, so the mutex is properly initialized. + let mutex = unsafe { crate::sync::Mutex::from_raw(ptr) }; + mutex.lock() + } } /// A pre-allocated [`GpuVmBo`] object. From 0b715b1e382b3d2e15d71c03a23be5f4333e7894 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 9 Apr 2026 15:26:10 +0000 Subject: [PATCH 035/131] rust: gpuvm: add GpuVmCore::sm_map() Finally also add the operation for creating new mappings. Mapping operations need extra data in the context since they involve a vm_bo coming from the outside. Co-developed-by: Asahi Lina Signed-off-by: Asahi Lina Reviewed-by: Daniel Almeida Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20260409-gpuvm-rust-v6-5-b16e6ada7261@google.com Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gpuvm/mod.rs | 9 +- rust/kernel/drm/gpuvm/sm_ops.rs | 167 +++++++++++++++++++++++++++++++- 2 files changed, 170 insertions(+), 6 deletions(-) diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index a6436abd0f9c..dc755f248143 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -108,7 +108,7 @@ const fn vtable() -> &'static bindings::drm_gpuvm_ops { vm_bo_alloc: GpuVmBo::::ALLOC_FN, vm_bo_free: GpuVmBo::::FREE_FN, vm_bo_validate: None, - sm_step_map: None, + sm_step_map: Some(Self::sm_step_map), sm_step_unmap: Some(Self::sm_step_unmap), sm_step_remap: Some(Self::sm_step_remap), } @@ -266,6 +266,13 @@ pub trait DriverGpuVm: Sized + Send { /// The private data passed to callbacks. type SmContext<'ctx>; + /// Indicates that a new mapping should be created. + fn sm_step_map<'op, 'ctx>( + &mut self, + op: OpMap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result, Error>; + /// Indicates that an existing mapping should be removed. fn sm_step_unmap<'op, 'ctx>( &mut self, diff --git a/rust/kernel/drm/gpuvm/sm_ops.rs b/rust/kernel/drm/gpuvm/sm_ops.rs index 05f81c638aef..69a8e5ab2821 100644 --- a/rust/kernel/drm/gpuvm/sm_ops.rs +++ b/rust/kernel/drm/gpuvm/sm_ops.rs @@ -8,6 +8,108 @@ struct SmData<'a, 'ctx, T: DriverGpuVm> { user_context: &'a mut T::SmContext<'ctx>, } +/// Adds an extra field to `SmData` for `sm_map()` callbacks. +/// +/// # Invariants +/// +/// `self.vm_bo.gpuvm() == self.sm_data.gpuvm`. +#[repr(C)] +struct SmMapData<'a, 'ctx, T: DriverGpuVm> { + sm_data: SmData<'a, 'ctx, T>, + vm_bo: &'a GpuVmBo, +} + +/// The argument for [`UniqueRefGpuVm::sm_map`]. +pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm> { + /// Address in GPU virtual address space. + pub addr: u64, + /// Length of mapping to create. + pub range: u64, + /// Offset in GEM object. + pub gem_offset: u64, + /// The GEM object to map. + pub vm_bo: &'a GpuVmBo, + /// The user-provided context type. + pub context: &'a mut T::SmContext<'ctx>, +} + +impl<'a, 'ctx, T: DriverGpuVm> OpMapRequest<'a, 'ctx, T> { + fn raw_request(&self) -> bindings::drm_gpuvm_map_req { + bindings::drm_gpuvm_map_req { + map: bindings::drm_gpuva_op_map { + va: bindings::drm_gpuva_op_map__bindgen_ty_1 { + addr: self.addr, + range: self.range, + }, + gem: bindings::drm_gpuva_op_map__bindgen_ty_2 { + offset: self.gem_offset, + obj: self.vm_bo.obj().as_raw(), + }, + }, + } + } +} + +/// Represents an `sm_step_map` operation that has not yet been completed. +pub struct OpMap<'op, T: DriverGpuVm> { + op: &'op bindings::drm_gpuva_op_map, + // Since these abstractions are designed for immediate mode, the VM BO needs to be + // pre-allocated, so we always have it available when we reach this point. + vm_bo: &'op GpuVmBo, + // This ensures that 'op is invariant, so that `OpMap<'long, T>` does not + // coerce to `OpMap<'short, T>`. This ensures that the user can't return + // the wrong `OpMapped` value. + _invariant: PhantomData<*mut &'op mut T>, +} + +impl<'op, T: DriverGpuVm> OpMap<'op, T> { + /// The base address of the new mapping. + pub fn addr(&self) -> u64 { + self.op.va.addr + } + + /// The length of the new mapping. + pub fn length(&self) -> u64 { + self.op.va.range + } + + /// The offset within the [`drm_gem_object`](DriverGpuVm::Object). + pub fn gem_offset(&self) -> u64 { + self.op.gem.offset + } + + /// The [`drm_gem_object`](DriverGpuVm::Object) to map. + pub fn obj(&self) -> &T::Object { + // SAFETY: The `obj` pointer is guaranteed to be valid. + unsafe { ::from_raw(self.op.gem.obj) } + } + + /// The [`GpuVmBo`] that the new VA will be associated with. + pub fn vm_bo(&self) -> &GpuVmBo { + self.vm_bo + } + + /// Use the pre-allocated VA to carry out this map operation. + pub fn insert(self, va: GpuVaAlloc, va_data: impl PinInit) -> OpMapped<'op, T> { + let va = va.prepare(va_data); + // SAFETY: By the type invariants we may access the interval tree. + unsafe { bindings::drm_gpuva_map(self.vm_bo.gpuvm().as_raw(), va, self.op) }; + + let _gpuva_guard = self.vm_bo().lock_gpuva(); + // SAFETY: The va is prepared for insertion, and we hold the GEM lock. + unsafe { bindings::drm_gpuva_link(va, self.vm_bo.as_raw()) }; + + OpMapped { + _invariant: self._invariant, + } + } +} + +/// Represents a completed [`OpMap`] operation. +pub struct OpMapped<'op, T> { + _invariant: PhantomData<*mut &'op mut T>, +} + /// Represents an `sm_step_unmap` operation that has not yet been completed. pub struct OpUnmap<'op, T: DriverGpuVm> { op: &'op bindings::drm_gpuva_op_unmap, @@ -213,6 +315,35 @@ pub struct OpRemapped<'op, T> { } impl UniqueRefGpuVm { + /// Create a mapping, removing or remapping anything that overlaps. + /// + /// Internally calls the [`DriverGpuVm`] callbacks similar to [`Self::sm_unmap`], except that + /// the [`DriverGpuVm::sm_step_map`] is called once to create the requested mapping. + #[inline] + pub fn sm_map(&mut self, req: OpMapRequest<'_, '_, T>) -> Result { + if req.vm_bo.gpuvm() != &**self { + return Err(EINVAL); + } + + let gpuvm = self.as_raw(); + let raw_req = req.raw_request(); + // INVARIANT: Checked above that `vm_bo.gpuvm() == self`. + let mut p = SmMapData { + sm_data: SmData { + gpuvm: self, + user_context: req.context, + }, + vm_bo: req.vm_bo, + }; + // SAFETY: + // * raw_request() creates a valid request. + // * The private data is valid to be interpreted as both SmData and SmMapData since the + // first field of SmMapData is SmData. + to_result(unsafe { + bindings::drm_gpuvm_sm_map(gpuvm, (&raw mut p).cast(), &raw const raw_req) + }) + } + /// Remove any mappings in the given region. /// /// Internally calls [`DriverGpuVm::sm_step_unmap`] for ranges entirely contained within the @@ -226,19 +357,45 @@ pub fn sm_unmap(&mut self, addr: u64, length: u64, context: &mut T::SmContext<'_ }; // SAFETY: // * raw_request() creates a valid request. - // * The private data is valid to be interpreted as SmData. + // * The private data is a valid SmData. to_result(unsafe { bindings::drm_gpuvm_sm_unmap(gpuvm, (&raw mut p).cast(), addr, length) }) } } impl GpuVm { /// # Safety - /// Must be called from `sm_unmap` with a pointer to `SmData`. + /// Must be called from `sm_map` with a pointer to `SmMapData`. + pub(super) unsafe extern "C" fn sm_step_map( + op: *mut bindings::drm_gpuva_op, + p: *mut c_void, + ) -> c_int { + // SAFETY: If we reach `sm_step_map` then we were called from `sm_map` which always passes + // an `SmMapData` as private data. + let p = unsafe { &mut *p.cast::>() }; + let op = OpMap { + // SAFETY: sm_step_map is called with a map operation. + op: unsafe { &(*op).__bindgen_anon_1.map }, + vm_bo: p.vm_bo, + _invariant: PhantomData, + }; + match p + .sm_data + .gpuvm + .data() + .sm_step_map(op, p.sm_data.user_context) + { + Ok(OpMapped { .. }) => 0, + Err(err) => err.to_errno(), + } + } + + /// # Safety + /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`. pub(super) unsafe extern "C" fn sm_step_unmap( op: *mut bindings::drm_gpuva_op, p: *mut c_void, ) -> c_int { - // SAFETY: The caller provides a pointer to `SmData`. + // SAFETY: The caller provides a pointer that can be treated as `SmData`. let p = unsafe { &mut *p.cast::>() }; let op = OpUnmap { // SAFETY: sm_step_unmap is called with an unmap operation. @@ -252,12 +409,12 @@ impl GpuVm { } /// # Safety - /// Must be called from `sm_unmap` with a pointer to `SmData`. + /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`. pub(super) unsafe extern "C" fn sm_step_remap( op: *mut bindings::drm_gpuva_op, p: *mut c_void, ) -> c_int { - // SAFETY: The caller provides a pointer to `SmData`. + // SAFETY: The caller provides a pointer that can be treated as `SmData`. let p = unsafe { &mut *p.cast::>() }; let op = OpRemap { // SAFETY: sm_step_remap is called with a remap operation. From 37f748ed0c19e007e7c5677f5d605d6b93841792 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 27 Apr 2026 10:54:51 +0000 Subject: [PATCH 036/131] drm/gpuvm: rust: add RUST_DRM_GPUVM option to Kconfig Since Rust uses GPUVM via the kernel crate, which is built-in, the GPUVM module must also be built-in to use GPUVM from Rust. Adjust the Kconfig settings accordingly. Suggested-by: Danilo Krummrich Signed-off-by: Alice Ryhl Link: https://patch.msgid.link/20260427-gpuvm-config-v1-1-8ece03771f8a@google.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/Kconfig | 7 +++++++ rust/helpers/drm_gpuvm.c | 4 ++-- rust/kernel/drm/gpuvm/mod.rs | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 8f5a8d3012e4..323422861e8f 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -218,6 +218,13 @@ config DRM_GPUVM GPU-VM representation providing helpers to manage a GPUs virtual address space +config RUST_DRM_GPUVM + bool + depends on DRM + select DRM_GPUVM + help + Choose this if you need GPUVM functions in Rust + config DRM_GPUSVM tristate depends on DRM diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c index ca959d9a66f6..4130b6325213 100644 --- a/rust/helpers/drm_gpuvm.c +++ b/rust/helpers/drm_gpuvm.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 or MIT -#ifdef CONFIG_DRM_GPUVM +#ifdef CONFIG_RUST_DRM_GPUVM #include @@ -23,4 +23,4 @@ bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, return drm_gpuvm_is_extobj(gpuvm, obj); } -#endif // CONFIG_DRM_GPUVM +#endif // CONFIG_RUST_DRM_GPUVM diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index dc755f248143..ae58f6f667c1 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR MIT -#![cfg(CONFIG_DRM_GPUVM = "y")] +#![cfg(CONFIG_RUST_DRM_GPUVM)] //! DRM GPUVM in immediate mode //! From 4f3491f6ec85c04535e8108614f34e12c351bbe9 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Fri, 8 May 2026 02:49:25 +0800 Subject: [PATCH 037/131] gpu: nova, nova-core: Rename to kebab-case Driver names must follow kernel kebab-case convention before they are exposed as UAPI via driver_override. Rename the nova-drm module from "Nova" to "nova-drm" and the nova-core module from "NovaCore" to "nova-core". Update NOVA_CORE_MODULE_NAME to match the renamed nova-core module. Suggested-by: Gary Guo Reviewed-by: Gary Guo Reviewed-by: John Hubbard Acked-by: Timur Tabi Closes: https://github.com/Rust-for-Linux/linux/issues/1228 Signed-off-by: Cheng-Yang Chou Link: https://patch.msgid.link/20260507185012.1527139-2-yphbchou0911@gmail.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 2 +- drivers/gpu/drm/nova/nova.rs | 2 +- drivers/gpu/nova-core/nova_core.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index b1af0a099551..f8f7beeccb13 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -36,7 +36,7 @@ pub(crate) struct NovaData { desc: c"Nvidia Graphics", }; -const NOVA_CORE_MODULE_NAME: &CStr = c"NovaCore"; +const NOVA_CORE_MODULE_NAME: &CStr = c"nova-core"; const AUXILIARY_NAME: &CStr = c"nova-drm"; kernel::auxiliary_device_table!( diff --git a/drivers/gpu/drm/nova/nova.rs b/drivers/gpu/drm/nova/nova.rs index 8893e58ee0db..1fd454c7e0df 100644 --- a/drivers/gpu/drm/nova/nova.rs +++ b/drivers/gpu/drm/nova/nova.rs @@ -10,7 +10,7 @@ kernel::module_auxiliary_driver! { type: NovaDriver, - name: "Nova", + name: "nova-drm", authors: ["Danilo Krummrich"], description: "Nova GPU driver", license: "GPL v2", diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 3a609f6937e4..80264094d44b 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -67,7 +67,7 @@ fn init(module: &'static kernel::ThisModule) -> impl PinInit { module! { type: NovaCoreModule, - name: "NovaCore", + name: "nova-core", authors: ["Danilo Krummrich"], description: "Nova Core GPU driver", license: "GPL v2", From 8bfe9d72cf2064f679c4192dba84be79eb70675d Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Fri, 8 May 2026 02:49:26 +0800 Subject: [PATCH 038/131] gpu: nova: Use module names consistently Update nova/Makefile and nova-core/Makefile so that nova-drm.ko and nova-core.ko are produced, matching the module names set in patch 1. Update drm::DriverInfo with the correct driver name and vendor description. Fix Kconfig help text for both drivers and the debugfs directory name in nova-core to match the new module names. Closes: https://github.com/Rust-for-Linux/linux/issues/1228 Signed-off-by: Cheng-Yang Chou Link: https://patch.msgid.link/20260507185012.1527139-3-yphbchou0911@gmail.com [ Change commit subject to "gpu: nova: Use module names consistently"; slightly adjust commit message. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/Kconfig | 2 +- drivers/gpu/drm/nova/Makefile | 3 ++- drivers/gpu/drm/nova/driver.rs | 4 ++-- drivers/gpu/nova-core/Kconfig | 2 +- drivers/gpu/nova-core/Makefile | 3 ++- drivers/gpu/nova-core/nova_core.rs | 2 +- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/nova/Kconfig b/drivers/gpu/drm/nova/Kconfig index a2028b8539d7..ba16c74401f8 100644 --- a/drivers/gpu/drm/nova/Kconfig +++ b/drivers/gpu/drm/nova/Kconfig @@ -14,4 +14,4 @@ config DRM_NOVA This driver is work in progress and may not be functional. - If M is selected, the module will be called nova. + If M is selected, the module will be called nova-drm. diff --git a/drivers/gpu/drm/nova/Makefile b/drivers/gpu/drm/nova/Makefile index 42019bff3173..f8527b2b7b4a 100644 --- a/drivers/gpu/drm/nova/Makefile +++ b/drivers/gpu/drm/nova/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_DRM_NOVA) += nova.o +obj-$(CONFIG_DRM_NOVA) += nova-drm.o +nova-drm-y := nova.o diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index f8f7beeccb13..e3de04f358f0 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -32,8 +32,8 @@ pub(crate) struct NovaData { major: 0, minor: 0, patchlevel: 0, - name: c"nova", - desc: c"Nvidia Graphics", + name: c"nova-drm", + desc: c"NVIDIA Graphics and Compute", }; const NOVA_CORE_MODULE_NAME: &CStr = c"nova-core"; diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig index d8456f8eaa05..f918f69e0599 100644 --- a/drivers/gpu/nova-core/Kconfig +++ b/drivers/gpu/nova-core/Kconfig @@ -14,4 +14,4 @@ config NOVA_CORE This driver is work in progress and may not be functional. - If M is selected, the module will be called nova_core. + If M is selected, the module will be called nova-core. diff --git a/drivers/gpu/nova-core/Makefile b/drivers/gpu/nova-core/Makefile index 2d78c50126e1..4ae544f808f4 100644 --- a/drivers/gpu/nova-core/Makefile +++ b/drivers/gpu/nova-core/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_NOVA_CORE) += nova_core.o +obj-$(CONFIG_NOVA_CORE) += nova-core.o +nova-core-y := nova_core.o diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 80264094d44b..c488058880be 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -52,7 +52,7 @@ struct NovaCoreModule { impl InPlaceModule for NovaCoreModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { - let dir = debugfs::Dir::new(kernel::c_str!("nova_core")); + let dir = debugfs::Dir::new(kernel::c_str!("nova-core")); // SAFETY: We are the only driver code running during init, so there // cannot be any concurrent access to `DEBUGFS_ROOT`. From abb21500e7e5dcf2d1b1a4a02b2ee77b3d5061b6 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 5 May 2026 17:23:07 +0200 Subject: [PATCH 039/131] rust: alloc: add Box::zeroed() Add Box::zeroed() for T: Zeroable types. This allocates with __GFP_ZERO directly, letting the underlying allocator deal with zeroing out the memory compared to Box::new(T::zeroed(), flags). Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260505152400.3905096-2-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kbox.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index bd6da02c7ab8..c824ed6e1523 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -19,6 +19,7 @@ use crate::fmt; use crate::init::InPlaceInit; use crate::page::AsPageIter; +use crate::prelude::*; use crate::types::ForeignOwnable; use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption}; @@ -256,6 +257,27 @@ pub fn new_uninit(flags: Flags) -> Result, A>, AllocError> { Ok(Box(ptr.cast(), PhantomData)) } + /// Creates a new zero-initialized `Box`. + /// + /// New memory is allocated with `A` and the [`__GFP_ZERO`] flag. The allocation may fail, in + /// which case an error is returned. For ZSTs no memory is allocated. + /// + /// # Examples + /// + /// ``` + /// let b = KBox::<[u8; 128]>::zeroed(GFP_KERNEL)?; + /// assert_eq!(*b, [0; 128]); + /// # Ok::<(), Error>(()) + /// ``` + pub fn zeroed(flags: Flags) -> Result + where + T: Zeroable, + { + // SAFETY: `__GFP_ZERO` guarantees the memory is zeroed; `T: Zeroable` guarantees that + // all-zeroes is a valid bit pattern for `T`. + Ok(unsafe { Self::new_uninit(flags | __GFP_ZERO)?.assume_init() }) + } + /// Constructs a new `Pin>`. If `T` does not implement [`Unpin`], then `x` will be /// pinned in memory and can't be moved. #[inline] From fd3b87ff0232f46e1ad53a48609a3853c8757c6c Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 5 May 2026 17:23:08 +0200 Subject: [PATCH 040/131] rust: auxiliary: add registration data to auxiliary devices Add a registration_data pointer to struct auxiliary_device, allowing the registering (parent) driver to attach private data to the device at registration time and retrieve it later when called back by the auxiliary (child) driver. By tying the data to the device's registration, Rust drivers can bind the lifetime of device resources to it, since the auxiliary bus guarantees that the parent driver remains bound while the auxiliary device is bound. On the Rust side, Registration takes ownership of the data via ForeignOwnable. A TypeId is stored alongside the data for runtime type checking, making Device::registration_data() a safe method. Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260505152400.3905096-3-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 10 +- include/linux/auxiliary_bus.h | 4 + rust/kernel/auxiliary.rs | 194 ++++++++++++++++++-------- samples/rust/rust_driver_auxiliary.rs | 40 ++++-- 4 files changed, 173 insertions(+), 75 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 84b0e1703150..8fe484d357f6 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -32,8 +32,7 @@ pub(crate) struct NovaCore { #[pin] pub(crate) gpu: Gpu, - #[pin] - _reg: Devres, + _reg: Devres>, } const BAR0_SIZE: usize = SZ_16M; @@ -96,14 +95,15 @@ fn probe(pdev: &pci::Device, _info: &Self::IdInfo) -> impl PinInit &device::Device { // SAFETY: A bound auxiliary device always has a bound parent device. unsafe { parent.as_bound() } } + + /// Returns a pinned reference to the registration data set by the registering (parent) driver. + /// + /// Returns [`EINVAL`] if `T` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + pub fn registration_data(&self) -> Result> { + // SAFETY: By the type invariant, `self.as_raw()` is a valid `struct auxiliary_device`. + let ptr = unsafe { (*self.as_raw()).registration_data_rust }; + if ptr.is_null() { + dev_warn!( + self.as_ref(), + "No registration data set; parent is not a Rust driver.\n" + ); + return Err(ENOENT); + } + + // SAFETY: `ptr` is non-null and was set via `into_foreign()` in `Registration::new()`; + // `RegistrationData` is `#[repr(C)]` with `type_id` at offset 0, so reading a `TypeId` + // at the start of the allocation is valid regardless of `T`. + let type_id = unsafe { ptr.cast::().read() }; + if type_id != TypeId::of::() { + return Err(EINVAL); + } + + // SAFETY: The `TypeId` check above confirms that the stored type is `T`; `ptr` remains + // valid until `Registration::drop()` calls `from_foreign()`. + let wrapper = unsafe { Pin::>>::borrow(ptr) }; + + // SAFETY: `data` is a structurally pinned field of `RegistrationData`. + Ok(unsafe { wrapper.map_unchecked(|w| &w.data) }) + } } impl Device { @@ -326,87 +365,132 @@ unsafe impl Send for Device {} // (i.e. `Device) are thread safe. unsafe impl Sync for Device {} +/// Wrapper that stores a [`TypeId`] alongside the registration data for runtime type checking. +#[repr(C)] +#[pin_data] +struct RegistrationData { + type_id: TypeId, + #[pin] + data: T, +} + /// The registration of an auxiliary device. /// /// This type represents the registration of a [`struct auxiliary_device`]. When its parent device /// is unbound, the corresponding auxiliary device will be unregistered from the system. /// +/// The type parameter `T` is the type of the registration data owned by the registering (parent) +/// driver. It can be accessed by the auxiliary driver through +/// [`Device::registration_data()`]. +/// /// # Invariants /// -/// `self.0` always holds a valid pointer to an initialized and registered -/// [`struct auxiliary_device`]. -pub struct Registration(NonNull); +/// `self.adev` always holds a valid pointer to an initialized and registered +/// [`struct auxiliary_device`] whose `registration_data_rust` field points to a +/// valid `Pin>>`. +pub struct Registration { + adev: NonNull, + _data: PhantomData, +} -impl Registration { - /// Create and register a new auxiliary device. - pub fn new<'a>( - parent: &'a device::Device, - name: &'a CStr, +impl Registration { + /// Create and register a new auxiliary device with the given registration data. + /// + /// The `data` is owned by the registration and can be accessed through the auxiliary device + /// via [`Device::registration_data()`]. + pub fn new( + parent: &device::Device, + name: &CStr, id: u32, - modname: &'a CStr, - ) -> impl PinInit, Error> + 'a { - pin_init::pin_init_scope(move || { - let boxed = KBox::new(Opaque::::zeroed(), GFP_KERNEL)?; - let adev = boxed.get(); + modname: &CStr, + data: impl PinInit, + ) -> Result> + where + Error: From, + { + let data = KBox::pin_init::( + try_pin_init!(RegistrationData { + type_id: TypeId::of::(), + data <- data, + }), + GFP_KERNEL, + )?; - // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization. - unsafe { - (*adev).dev.parent = parent.as_raw(); - (*adev).dev.release = Some(Device::release); - (*adev).name = name.as_char_ptr(); - (*adev).id = id; - } + let boxed: KBox> = KBox::zeroed(GFP_KERNEL)?; + let adev = boxed.get(); - // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, - // which has not been initialized yet. - unsafe { bindings::auxiliary_device_init(adev) }; + // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization. + unsafe { + (*adev).dev.parent = parent.as_raw(); + (*adev).dev.release = Some(Device::release); + (*adev).name = name.as_char_ptr(); + (*adev).id = id; + (*adev).registration_data_rust = data.into_foreign(); + } - // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be - // freed by `Device::release` when the last reference to the `struct auxiliary_device` - // is dropped. - let _ = KBox::into_raw(boxed); + // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, + // which has not been initialized yet. + unsafe { bindings::auxiliary_device_init(adev) }; - // SAFETY: - // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which - // has been initialized, - // - `modname.as_char_ptr()` is a NULL terminated string. - let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) }; - if ret != 0 { - // SAFETY: `adev` is guaranteed to be a valid pointer to a - // `struct auxiliary_device`, which has been initialized. - unsafe { bindings::auxiliary_device_uninit(adev) }; + // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be + // freed by `Device::release` when the last reference to the `struct auxiliary_device` + // is dropped. + let _ = KBox::into_raw(boxed); - return Err(Error::from_errno(ret)); - } + // SAFETY: + // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which + // has been initialized, + // - `modname.as_char_ptr()` is a NULL terminated string. + let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) }; + if ret != 0 { + // SAFETY: `registration_data` was set above via `into_foreign()`. + drop(unsafe { + Pin::>>::from_foreign((*adev).registration_data_rust) + }); - // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is - // called, which happens in `Self::drop()`. - Ok(Devres::new( - parent, - // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated - // successfully. - Self(unsafe { NonNull::new_unchecked(adev) }), - )) - }) + // SAFETY: `adev` is guaranteed to be a valid pointer to a + // `struct auxiliary_device`, which has been initialized. + unsafe { bindings::auxiliary_device_uninit(adev) }; + + return Err(Error::from_errno(ret)); + } + + // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is + // called, which happens in `Self::drop()`. + let reg = Self { + // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated + // successfully. + adev: unsafe { NonNull::new_unchecked(adev) }, + _data: PhantomData, + }; + + Devres::new::(parent, reg) } } -impl Drop for Registration { +impl Drop for Registration { fn drop(&mut self) { - // SAFETY: By the type invariant of `Self`, `self.0.as_ptr()` is a valid registered + // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered // `struct auxiliary_device`. - unsafe { bindings::auxiliary_device_delete(self.0.as_ptr()) }; + unsafe { bindings::auxiliary_device_delete(self.adev.as_ptr()) }; + + // SAFETY: `registration_data` was set in `new()` via `into_foreign()`. + drop(unsafe { + Pin::>>::from_foreign( + (*self.adev.as_ptr()).registration_data_rust, + ) + }); // This drops the reference we acquired through `auxiliary_device_init()`. // - // SAFETY: By the type invariant of `Self`, `self.0.as_ptr()` is a valid registered + // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered // `struct auxiliary_device`. - unsafe { bindings::auxiliary_device_uninit(self.0.as_ptr()) }; + unsafe { bindings::auxiliary_device_uninit(self.adev.as_ptr()) }; } } // SAFETY: A `Registration` of a `struct auxiliary_device` can be released from any thread. -unsafe impl Send for Registration {} +unsafe impl Send for Registration {} // SAFETY: `Registration` does not expose any methods or fields that need synchronization. -unsafe impl Sync for Registration {} +unsafe impl Sync for Registration {} diff --git a/samples/rust/rust_driver_auxiliary.rs b/samples/rust/rust_driver_auxiliary.rs index 5c5a5105a3ff..319ef734c02b 100644 --- a/samples/rust/rust_driver_auxiliary.rs +++ b/samples/rust/rust_driver_auxiliary.rs @@ -17,8 +17,6 @@ InPlaceModule, // }; -use core::any::TypeId; - const MODULE_NAME: &CStr = ::NAME; const AUXILIARY_NAME: &CStr = c"auxiliary"; @@ -49,13 +47,13 @@ fn probe(adev: &auxiliary::Device, _info: &Self::IdInfo) -> impl PinInit, - #[pin] - _reg1: Devres, + _reg0: Devres>, + _reg1: Devres>, } kernel::pci_device_table!( @@ -71,10 +69,21 @@ impl pci::Driver for ParentDriver { const ID_TABLE: pci::IdTable = &PCI_TABLE; fn probe(pdev: &pci::Device, _info: &Self::IdInfo) -> impl PinInit { - try_pin_init!(Self { - private: TypeId::of::(), - _reg0 <- auxiliary::Registration::new(pdev.as_ref(), AUXILIARY_NAME, 0, MODULE_NAME), - _reg1 <- auxiliary::Registration::new(pdev.as_ref(), AUXILIARY_NAME, 1, MODULE_NAME), + Ok(Self { + _reg0: auxiliary::Registration::new( + pdev.as_ref(), + AUXILIARY_NAME, + 0, + MODULE_NAME, + Data { index: 0 }, + )?, + _reg1: auxiliary::Registration::new( + pdev.as_ref(), + AUXILIARY_NAME, + 1, + MODULE_NAME, + Data { index: 1 }, + )?, }) } } @@ -83,7 +92,8 @@ impl ParentDriver { fn connect(adev: &auxiliary::Device) -> Result { let dev = adev.parent(); let pdev: &pci::Device = dev.try_into()?; - let drvdata = dev.drvdata::()?; + + let data = adev.registration_data::()?; dev_info!( dev, @@ -95,8 +105,8 @@ fn connect(adev: &auxiliary::Device) -> Result { dev_info!( dev, - "We have access to the private data of {:?}.\n", - drvdata.private + "Connected to auxiliary device with index {}.\n", + data.index ); Ok(()) From 95ade775c4ab9b9b3d7cfa2d45283e93fbfa4e7a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 5 May 2026 17:23:09 +0200 Subject: [PATCH 041/131] rust: driver core: remove drvdata() and driver_type When drvdata() was introduced in commit 6f61a2637abe ("rust: device: introduce Device::drvdata()"), its commit message already noted that a direct accessor to the driver's bus device private data is not commonly required -- bus callbacks provide access through &self, and other entry points (IRQs, workqueues, IOCTLs, etc.) carry their own private data. The sole motivation for drvdata() was inter-driver interaction -- an auxiliary driver deriving the parent's bus device private data from the parent device. However, drvdata() exposes the driver's bus device private data beyond the driver's own scope. This creates ordering constraints; for instance drvdata may not be set yet when the first caller of drvdata() can appear. It also forces the driver's bus device private data to outlive all registrations that access it, which causes unnecessary complications. Private data should be private to the entity that issues it, i.e. bus device private data belongs to bus callbacks, class device private data to class callbacks, IRQ private data to the IRQ handler, etc. With registration-private data now available through the auxiliary bus, there is no remaining user of drvdata(), thus remove it. Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260505152400.3905096-4-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/base/base.h | 16 ------------ rust/kernel/device.rs | 60 ------------------------------------------- 2 files changed, 76 deletions(-) diff --git a/drivers/base/base.h b/drivers/base/base.h index 30b416588617..a19f4cda2c83 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -86,18 +86,6 @@ struct driver_private { }; #define to_driver(obj) container_of(obj, struct driver_private, kobj) -#ifdef CONFIG_RUST -/** - * struct driver_type - Representation of a Rust driver type. - */ -struct driver_type { - /** - * @id: Representation of core::any::TypeId. - */ - u8 id[16]; -} __packed; -#endif - /** * struct device_private - structure to hold the private to the driver core * portions of the device structure. @@ -115,7 +103,6 @@ struct driver_type { * dev_err_probe() for later retrieval via debugfs * @device: pointer back to the struct device that this structure is * associated with. - * @driver_type: The type of the bound Rust driver. * @dead: This device is currently either in the process of or has been * removed from the system. Any asynchronous events scheduled for this * device should exit without taking any action. @@ -132,9 +119,6 @@ struct device_private { const struct device_driver *async_driver; char *deferred_probe_reason; struct device *device; -#ifdef CONFIG_RUST - struct driver_type driver_type; -#endif u8 dead:1; }; #define to_device_private_parent(obj) \ diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 6d5396a43ebe..fd50399aadea 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -15,16 +15,12 @@ }, // }; use core::{ - any::TypeId, marker::PhantomData, ptr, // }; pub mod property; -// Assert that we can `read()` / `write()` a `TypeId` instance from / into `struct driver_type`. -static_assert!(core::mem::size_of::() >= core::mem::size_of::()); - /// The core representation of a device in the kernel's driver model. /// /// This structure represents the Rust abstraction for a C `struct device`. A [`Device`] can either @@ -206,29 +202,12 @@ pub unsafe fn as_bound(&self) -> &Device { } impl Device { - fn set_type_id(&self) { - // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. - let private = unsafe { (*self.as_raw()).p }; - - // SAFETY: For a bound device (implied by the `CoreInternal` device context), `private` is - // guaranteed to be a valid pointer to a `struct device_private`. - let driver_type = unsafe { &raw mut (*private).driver_type }; - - // SAFETY: `driver_type` is valid for (unaligned) writes of a `TypeId`. - unsafe { - driver_type - .cast::() - .write_unaligned(TypeId::of::()) - }; - } - /// Store a pointer to the bound driver's private data. pub fn set_drvdata(&self, data: impl PinInit) -> Result { let data = KBox::pin_init(data, GFP_KERNEL)?; // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. unsafe { bindings::dev_set_drvdata(self.as_raw(), data.into_foreign().cast()) }; - self.set_type_id::(); Ok(()) } @@ -292,45 +271,6 @@ unsafe fn drvdata_unchecked(&self) -> Pin<&T> { // in `into_foreign()`. unsafe { Pin::>::borrow(ptr.cast()) } } - - fn match_type_id(&self) -> Result { - // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. - let private = unsafe { (*self.as_raw()).p }; - - // SAFETY: For a bound device, `private` is guaranteed to be a valid pointer to a - // `struct device_private`. - let driver_type = unsafe { &raw mut (*private).driver_type }; - - // SAFETY: - // - `driver_type` is valid for (unaligned) reads of a `TypeId`. - // - A bound device guarantees that `driver_type` contains a valid `TypeId` value. - let type_id = unsafe { driver_type.cast::().read_unaligned() }; - - if type_id != TypeId::of::() { - return Err(EINVAL); - } - - Ok(()) - } - - /// Access a driver's private data. - /// - /// Returns a pinned reference to the driver's private data or [`EINVAL`] if it doesn't match - /// the asserted type `T`. - pub fn drvdata(&self) -> Result> { - // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. - if unsafe { bindings::dev_get_drvdata(self.as_raw()) }.is_null() { - return Err(ENOENT); - } - - self.match_type_id::()?; - - // SAFETY: - // - The above check of `dev_get_drvdata()` guarantees that we are called after - // `set_drvdata()`. - // - We've just checked that the type of the driver's private data is in fact `T`. - Ok(unsafe { self.drvdata_unchecked() }) - } } impl Device { From fc7c1054b6f983ae2f3e100a24cc87908ae9f4b7 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:19 +0900 Subject: [PATCH 042/131] gpu: nova-core: vbios: stop scanning at BIOS_MAX_SCAN_LEN Current code lets `current_offset` go to `BIOS_MAX_SCAN_LEN` which is one byte too far. Fixes: 6fda04e7f0cd ("gpu: nova-core: vbios: Add base support for VBIOS construction and iteration") Reviewed-by: Joel Fernandes Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-1-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 6bcfb6c5cf44..7bec81a37340 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -272,7 +272,7 @@ fn next(&mut self) -> Option { return None; } - if self.current_offset > BIOS_MAX_SCAN_LEN { + if self.current_offset >= BIOS_MAX_SCAN_LEN { dev_err!(self.dev, "Error: exceeded BIOS scan limit, stopping scan\n"); return None; } From 7a1d09e477b6496f13f92b84ed9b2eab191b3366 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:20 +0900 Subject: [PATCH 043/131] gpu: nova-core: vbios: use checked arithmetic for bios image range end `read_bios_image_at_offset` is called with a length from the VBIOS header, so we should be more defensive here and use checked arithmetic. Fixes: 6fda04e7f0cd ("gpu: nova-core: vbios: Add base support for VBIOS construction and iteration") Reviewed-by: Joel Fernandes Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-2-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 7bec81a37340..180928433766 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -238,8 +238,8 @@ fn read_bios_image_at_offset( len: usize, context: &str, ) -> Result { - let data_len = self.data.len(); - if offset + len > data_len { + let end = offset.checked_add(len).ok_or(EINVAL)?; + if end > self.data.len() { self.read_more_at_offset(offset, len).inspect_err(|e| { dev_err!( self.dev, @@ -250,7 +250,7 @@ fn read_bios_image_at_offset( })?; } - BiosImage::new(self.dev, &self.data[offset..offset + len]).inspect_err(|err| { + BiosImage::new(self.dev, &self.data[offset..end]).inspect_err(|err| { dev_err!( self.dev, "Failed to {} at offset {:#x}: {:?}\n", From 33f1402bcfa6cfd85fa265ce6fa5c6bb7d981c6d Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:21 +0900 Subject: [PATCH 044/131] gpu: nova-core: vbios: avoid reading too far in read_more_at_offset Fix bug where `read_more_at_offset` would unnecessarily read more data. This happens when the window to read has some part cached and some part not. It would read `len` bytes instead of just the uncached portion, which could read past `BIOS_MAX_SCAN_LEN`. Fixes: 6fda04e7f0cd ("gpu: nova-core: vbios: Add base support for VBIOS construction and iteration") Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-3-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 180928433766..79eb01dabc6f 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -185,8 +185,13 @@ fn new(dev: &'a device::Device, bar0: &'a Bar0) -> Result { /// Read bytes from the ROM at the current end of the data vector. fn read_more(&mut self, len: usize) -> Result { - let current_len = self.data.len(); - let start = ROM_OFFSET + current_len; + let start = self.data.len(); + let end = start + len; + + if end > BIOS_MAX_SCAN_LEN { + dev_err!(self.dev, "Error: exceeded BIOS scan limit.\n"); + return Err(EINVAL); + } // Ensure length is a multiple of 4 for 32-bit reads if len % core::mem::size_of::() != 0 { @@ -200,9 +205,9 @@ fn read_more(&mut self, len: usize) -> Result { self.data.reserve(len, GFP_KERNEL)?; // Read ROM data bytes and push directly to `data`. - for addr in (start..start + len).step_by(core::mem::size_of::()) { + for addr in (start..end).step_by(core::mem::size_of::()) { // Read 32-bit word from the VBIOS ROM - let word = self.bar0.try_read32(addr)?; + let word = self.bar0.try_read32(ROM_OFFSET + addr)?; // Convert the `u32` to a 4 byte array and push each byte. word.to_ne_bytes() @@ -215,17 +220,9 @@ fn read_more(&mut self, len: usize) -> Result { /// Read bytes at a specific offset, filling any gap. fn read_more_at_offset(&mut self, offset: usize, len: usize) -> Result { - if offset > BIOS_MAX_SCAN_LEN { - dev_err!(self.dev, "Error: exceeded BIOS scan limit.\n"); - return Err(EINVAL); - } + let end = offset.checked_add(len).ok_or(EINVAL)?; - // If `offset` is beyond current data size, fill the gap first. - let current_len = self.data.len(); - let gap_bytes = offset.saturating_sub(current_len); - - // Now read the requested bytes at the offset. - self.read_more(gap_bytes + len) + self.read_more(end.saturating_sub(self.data.len())) } /// Read a BIOS image at a specific offset and create a [`BiosImage`] from it. From 237c252be0db616c93e4984369db7e74bb797564 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:22 +0900 Subject: [PATCH 045/131] gpu: nova-core: vbios: read BitToken using FromBytes If `header.token_size` is smaller than `BitToken`, then we currently can read past the end of `image.base.data`. Use checked arithmetic for computing offsets and simplify reading it in using `FromBytes`. Fixes: dc70c6ae2441 ("gpu: nova-core: vbios: Add support to look up PMU table in FWSEC") Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-4-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 37 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 79eb01dabc6f..2ff67273fdff 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -486,7 +486,7 @@ fn new(data: &[u8]) -> Result { /// BIT Token Entry: Records in the BIT table followed by the BIT header. #[derive(Debug, Clone, Copy)] -#[expect(dead_code)] +#[repr(C)] struct BitToken { /// 00h: Token identifier id: u8, @@ -498,6 +498,9 @@ struct BitToken { data_offset: u16, } +// SAFETY: all bit patterns are valid for `BitToken`. +unsafe impl FromBytes for BitToken {} + // Define the token ID for the Falcon data const BIT_TOKEN_ID_FALCON_DATA: u8 = 0x70; @@ -505,32 +508,28 @@ impl BitToken { /// Find a BIT token entry by BIT ID in a PciAtBiosImage fn from_id(image: &PciAtBiosImage, token_id: u8) -> Result { let header = &image.bit_header; + let entry_size = usize::from(header.token_size); // Offset to the first token entry let tokens_start = image.bit_offset + usize::from(header.header_size); for i in 0..usize::from(header.token_entries) { - let entry_offset = tokens_start + (i * usize::from(header.token_size)); + let entry_offset = i + .checked_mul(entry_size) + .and_then(|offset| tokens_start.checked_add(offset)) + .ok_or(EINVAL)?; + let entry = image + .base + .data + .get(entry_offset..) + .and_then(|data| data.get(..entry_size)) + .ok_or(EINVAL)?; - // Make sure we don't go out of bounds - if entry_offset + usize::from(header.token_size) > image.base.data.len() { - return Err(EINVAL); - } + let (token, _) = BitToken::from_bytes_copy_prefix(entry).ok_or(EINVAL)?; // Check if this token has the requested ID - if image.base.data[entry_offset] == token_id { - return Ok(BitToken { - id: image.base.data[entry_offset], - data_version: image.base.data[entry_offset + 1], - data_size: u16::from_le_bytes([ - image.base.data[entry_offset + 2], - image.base.data[entry_offset + 3], - ]), - data_offset: u16::from_le_bytes([ - image.base.data[entry_offset + 4], - image.base.data[entry_offset + 5], - ]), - }); + if token.id == token_id { + return Ok(token); } } From 7c62d0b006527efc5fb4609b555c65674c819603 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:23 +0900 Subject: [PATCH 046/131] gpu: nova-core: vbios: use checked ops and accesses in `FwSecBiosImage::ucode` Use checked arithmetic and access for extracting the microcode since the offsets are firmware derived. Fixes: 47c4846e4319 ("gpu: nova-core: vbios: Add support for FWSEC ucode extraction") Reviewed-by: Joel Fernandes Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-5-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 2ff67273fdff..c62d918a3041 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -1110,16 +1110,18 @@ pub(crate) fn header(&self) -> Result { /// Get the ucode data as a byte slice pub(crate) fn ucode(&self, desc: &FalconUCodeDesc) -> Result<&[u8]> { - let falcon_ucode_offset = self.falcon_ucode_offset; + let size = usize::from_safe_cast( + desc.imem_load_size() + .checked_add(desc.dmem_load_size()) + .ok_or(ERANGE)?, + ); // The ucode data follows the descriptor. - let ucode_data_offset = falcon_ucode_offset + desc.size(); - let size = usize::from_safe_cast(desc.imem_load_size() + desc.dmem_load_size()); - - // Get the data slice, checking bounds in a single operation. self.base .data - .get(ucode_data_offset..ucode_data_offset + size) + .get(self.falcon_ucode_offset..) + .and_then(|data| data.get(desc.size()..)) + .and_then(|data| data.get(..size)) .ok_or(ERANGE) .inspect_err(|_| { dev_err!( From 25ad950b4ee37f7b42e006f508a793b7c38fcc12 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:24 +0900 Subject: [PATCH 047/131] gpu: nova-core: vbios: use checked access in `FwSecBiosImage::header` Use checked access in `FwSecBiosImage::header` for getting the header version since the value is firmware derived. Fixes: 47c4846e4319 ("gpu: nova-core: vbios: Add support for FWSEC ucode extraction") Reviewed-by: Joel Fernandes Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-6-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index c62d918a3041..48a46684e279 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -1077,17 +1077,14 @@ fn build(self) -> Result { impl FwSecBiosImage { /// Get the FwSec header ([`FalconUCodeDesc`]). pub(crate) fn header(&self) -> Result { - // Get the falcon ucode offset that was found in setup_falcon_data. - let falcon_ucode_offset = self.falcon_ucode_offset; + let data = self + .base + .data + .get(self.falcon_ucode_offset..) + .ok_or(EINVAL)?; - // Read the first 4 bytes to get the version. - let hdr_bytes: [u8; 4] = self.base.data[falcon_ucode_offset..falcon_ucode_offset + 4] - .try_into() - .map_err(|_| EINVAL)?; - let hdr = u32::from_le_bytes(hdr_bytes); - let ver = (hdr & 0xff00) >> 8; - - let data = self.base.data.get(falcon_ucode_offset..).ok_or(EINVAL)?; + // Read the version byte from the header. + let ver = data.get(1).copied().ok_or(EINVAL)?; match ver { 2 => { let v2 = FalconUCodeDescV2::from_bytes_copy_prefix(data) From 051ae1b21ff7a3cc27522b0b3b56e277b62c1207 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:25 +0900 Subject: [PATCH 048/131] gpu: nova-core: vbios: use checked accesses in `setup_falcon_data` Use checked arithmetic for `ucode_offset` in `setup_falcon_data`. This prevents a malformed firmware from causing a panic. Fixes: dc70c6ae2441 ("gpu: nova-core: vbios: Add support to look up PMU table in FWSEC") Reviewed-by: Joel Fernandes Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-7-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 48a46684e279..871f455bb720 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -1036,14 +1036,15 @@ fn setup_falcon_data( .find_entry_by_type(FALCON_UCODE_ENTRY_APPID_FWSEC_PROD) { Ok(entry) => { - let mut ucode_offset = usize::from_safe_cast(entry.data); - ucode_offset -= pci_at_image.base.data.len(); - if ucode_offset < first_fwsec.base.data.len() { - dev_err!(self.base.dev, "Falcon Ucode offset not in second Fwsec.\n"); - return Err(EINVAL); - } - ucode_offset -= first_fwsec.base.data.len(); - self.falcon_ucode_offset = Some(ucode_offset); + self.falcon_ucode_offset = Some( + usize::from_safe_cast(entry.data) + .checked_sub(pci_at_image.base.data.len()) + .and_then(|o| o.checked_sub(first_fwsec.base.data.len())) + .ok_or(EINVAL) + .inspect_err(|_| { + dev_err!(self.base.dev, "Falcon Ucode offset not in second Fwsec.\n"); + })?, + ); } Err(e) => { dev_err!( From 56f7c0b3800e730df0f290e0f36d7f0d664cb66b Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:26 +0900 Subject: [PATCH 049/131] gpu: nova-core: vbios: drop unused falcon_data_offset from FwSecBiosBuilder This is unused, so we can remove it. Reviewed-by: Joel Fernandes Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-8-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 871f455bb720..8a0e16e6c9e8 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -338,7 +338,6 @@ pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { Ok(BiosImageType::FwSec) => { let fwsec = FwSecBiosBuilder { base: image, - falcon_data_offset: None, pmu_lookup_table: None, falcon_ucode_offset: None, }; @@ -712,8 +711,6 @@ struct FwSecBiosBuilder { /// Once FwSecBiosBuilder is constructed, the `falcon_ucode_offset` will be copied into a new /// [`FwSecBiosImage`]. /// - /// The offset of the Falcon data from the start of Fwsec image. - falcon_data_offset: Option, /// The [`PmuLookupTable`] starts at the offset of the falcon data pointer. pmu_lookup_table: Option, /// The offset of the Falcon ucode. @@ -1015,8 +1012,6 @@ fn setup_falcon_data( offset -= first_fwsec.base.data.len(); } - self.falcon_data_offset = Some(offset); - if pmu_in_first_fwsec { self.pmu_lookup_table = Some(PmuLookupTable::new( &self.base.dev, From 8cf15cf2641be8a786e4c7b5bd1f7f5cf640889f Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:27 +0900 Subject: [PATCH 050/131] gpu: nova-core: vbios: keep PmuLookupTable local in setup_falcon_data This does not need to be stored in `FwSecBiosBuilder` so we can remove it from there, and just create and use it locally in `setup_falcon_data`. Reviewed-by: Joel Fernandes Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-9-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 8a0e16e6c9e8..cadc6dcffefb 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -338,7 +338,6 @@ pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { Ok(BiosImageType::FwSec) => { let fwsec = FwSecBiosBuilder { base: image, - pmu_lookup_table: None, falcon_ucode_offset: None, }; if first_fwsec_image.is_none() { @@ -711,8 +710,6 @@ struct FwSecBiosBuilder { /// Once FwSecBiosBuilder is constructed, the `falcon_ucode_offset` will be copied into a new /// [`FwSecBiosImage`]. /// - /// The [`PmuLookupTable`] starts at the offset of the falcon data pointer. - pmu_lookup_table: Option, /// The offset of the Falcon ucode. falcon_ucode_offset: Option, } @@ -1012,24 +1009,14 @@ fn setup_falcon_data( offset -= first_fwsec.base.data.len(); } - if pmu_in_first_fwsec { - self.pmu_lookup_table = Some(PmuLookupTable::new( - &self.base.dev, - &first_fwsec.base.data[offset..], - )?); + let pmu_lookup_data = if pmu_in_first_fwsec { + &first_fwsec.base.data[offset..] } else { - self.pmu_lookup_table = Some(PmuLookupTable::new( - &self.base.dev, - &self.base.data[offset..], - )?); - } + self.base.data.get(offset..).ok_or(EINVAL)? + }; + let pmu_lookup_table = PmuLookupTable::new(&self.base.dev, pmu_lookup_data)?; - match self - .pmu_lookup_table - .as_ref() - .ok_or(EINVAL)? - .find_entry_by_type(FALCON_UCODE_ENTRY_APPID_FWSEC_PROD) - { + match pmu_lookup_table.find_entry_by_type(FALCON_UCODE_ENTRY_APPID_FWSEC_PROD) { Ok(entry) => { self.falcon_ucode_offset = Some( usize::from_safe_cast(entry.data) From b2a48fc068ea67729a822c3b85c3a535a01da793 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:28 +0900 Subject: [PATCH 051/131] gpu: nova-core: vbios: compute FWSEC-relative Falcon data offset Push the computation of the falcon data offset into a helper function. The subtraction to create the offset should be checked, and by doing this the check can be folded into the existing check in `falcon_data_ptr`. Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Reviewed-by: Joel Fernandes Link: https://patch.msgid.link/20260525-fix-vbios-v5-10-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 48 ++++++++++++++-------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index cadc6dcffefb..ca101b2b6095 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -846,33 +846,29 @@ fn get_bit_token(&self, token_id: u8) -> Result { BitToken::from_id(self, token_id) } - /// Find the Falcon data pointer structure in the [`PciAtBiosImage`]. + /// Find the Falcon data offset from the start of the FWSEC region. /// - /// This is just a 4 byte structure that contains a pointer to the Falcon data in the FWSEC - /// image. - fn falcon_data_ptr(&self) -> Result { + /// The BIT table contains a 4-byte pointer to the Falcon data. Testing shows this pointer + /// treats the PCI-AT and FWSEC images as logically contiguous even when an EFI image sits in + /// between them, so subtract the PCI-AT image size here to convert it to a FWSEC-relative + /// offset. + fn falcon_data_offset(&self) -> Result { let token = self.get_bit_token(BIT_TOKEN_ID_FALCON_DATA)?; - - // Make sure we don't go out of bounds - if usize::from(token.data_offset) + 4 > self.base.data.len() { - return Err(EINVAL); - } - - // read the 4 bytes at the offset specified in the token let offset = usize::from(token.data_offset); - let bytes: [u8; 4] = self.base.data[offset..offset + 4].try_into().map_err(|_| { - dev_err!(self.base.dev, "Failed to convert data slice to array\n"); - EINVAL - })?; - let data_ptr = u32::from_le_bytes(bytes); + // Read the 4-byte falcon data pointer at the offset specified in the token. + let data = &self.base.data; + let (ptr, _) = data + .get(offset..) + .and_then(u32::from_bytes_copy_prefix) + .ok_or(EINVAL)?; - if (usize::from_safe_cast(data_ptr)) < self.base.data.len() { - dev_err!(self.base.dev, "Falcon data pointer out of bounds\n"); - return Err(EINVAL); - } - - Ok(data_ptr) + usize::from_safe_cast(ptr) + .checked_sub(data.len()) + .ok_or(EINVAL) + .inspect_err(|_| { + dev_err!(self.base.dev, "Falcon data pointer out of bounds\n"); + }) } } @@ -989,15 +985,9 @@ fn setup_falcon_data( pci_at_image: &PciAtBiosImage, first_fwsec: &FwSecBiosBuilder, ) -> Result { - let mut offset = usize::from_safe_cast(pci_at_image.falcon_data_ptr()?); + let mut offset = pci_at_image.falcon_data_offset()?; let mut pmu_in_first_fwsec = false; - // The falcon data pointer assumes that the PciAt and FWSEC images - // are contiguous in memory. However, testing shows the EFI image sits in - // between them. So calculate the offset from the end of the PciAt image - // rather than the start of it. Compensate. - offset -= pci_at_image.base.data.len(); - // The offset is now from the start of the first Fwsec image, however // the offset points to a location in the second Fwsec image. Since // the fwsec images are contiguous, subtract the length of the first Fwsec From 99e110a36885ffc91c711e84d3121160a5c5eb77 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:29 +0900 Subject: [PATCH 052/131] gpu: nova-core: vbios: simplify setup_falcon_data The code first computes `pmu_in_first_fwsec` or adjusts the offset and then uses it in a branch just once to get the correct source for the PMU table. This can be simplified to a single branch while also avoiding the mutation of `offset`. Also, adjust the code after this to keep the success case non-nested. Reviewed-by: Joel Fernandes Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-11-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 54 +++++++++++++++------------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index ca101b2b6095..470e0e2a81ab 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -985,48 +985,40 @@ fn setup_falcon_data( pci_at_image: &PciAtBiosImage, first_fwsec: &FwSecBiosBuilder, ) -> Result { - let mut offset = pci_at_image.falcon_data_offset()?; - let mut pmu_in_first_fwsec = false; + let offset = pci_at_image.falcon_data_offset()?; - // The offset is now from the start of the first Fwsec image, however - // the offset points to a location in the second Fwsec image. Since - // the fwsec images are contiguous, subtract the length of the first Fwsec - // image from the offset to get the offset to the start of the second - // Fwsec image. - if offset < first_fwsec.base.data.len() { - pmu_in_first_fwsec = true; + // The offset is from the start of the first FwSec image, but it + // may point into the second FwSec image. Treat the two FwSec images + // as contiguous here and subtract the first image length when the + // target lies in the second one. + let pmu_lookup_data = if offset < first_fwsec.base.data.len() { + first_fwsec.base.data.get(offset..) } else { - offset -= first_fwsec.base.data.len(); + self.base.data.get(offset - first_fwsec.base.data.len()..) } + .ok_or(EINVAL)?; - let pmu_lookup_data = if pmu_in_first_fwsec { - &first_fwsec.base.data[offset..] - } else { - self.base.data.get(offset..).ok_or(EINVAL)? - }; let pmu_lookup_table = PmuLookupTable::new(&self.base.dev, pmu_lookup_data)?; - match pmu_lookup_table.find_entry_by_type(FALCON_UCODE_ENTRY_APPID_FWSEC_PROD) { - Ok(entry) => { - self.falcon_ucode_offset = Some( - usize::from_safe_cast(entry.data) - .checked_sub(pci_at_image.base.data.len()) - .and_then(|o| o.checked_sub(first_fwsec.base.data.len())) - .ok_or(EINVAL) - .inspect_err(|_| { - dev_err!(self.base.dev, "Falcon Ucode offset not in second Fwsec.\n"); - })?, - ); - } - Err(e) => { + let entry = pmu_lookup_table + .find_entry_by_type(FALCON_UCODE_ENTRY_APPID_FWSEC_PROD) + .inspect_err(|e| { dev_err!( self.base.dev, "PmuLookupTableEntry not found, error: {:?}\n", e ); - return Err(EINVAL); - } - } + })?; + + let falcon_ucode_offset = usize::from_safe_cast(entry.data) + .checked_sub(pci_at_image.base.data.len()) + .and_then(|o| o.checked_sub(first_fwsec.base.data.len())) + .ok_or(EINVAL) + .inspect_err(|_| { + dev_err!(self.base.dev, "Falcon Ucode offset not in second Fwsec.\n"); + })?; + + self.falcon_ucode_offset = Some(falcon_ucode_offset); Ok(()) } From c22095fddad76e1d5387292589b4d225bc1b2ba1 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:30 +0900 Subject: [PATCH 053/131] gpu: nova-core: vbios: read PMU lookup entries using FromBytes This simplifies the construction of `PmuLookupTableEntry` and is allowed now that the driver can assume it is little endian. Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-12-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 470e0e2a81ab..987eb1948314 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -897,19 +897,8 @@ struct PmuLookupTableEntry { data: u32, } -impl PmuLookupTableEntry { - fn new(data: &[u8]) -> Result { - if data.len() < core::mem::size_of::() { - return Err(EINVAL); - } - - Ok(PmuLookupTableEntry { - application_id: data[0], - target_id: data[1], - data: u32::from_le_bytes(data[2..6].try_into().map_err(|_| EINVAL)?), - }) - } -} +// SAFETY: all bit patterns are valid for `PmuLookupTableEntry`. +unsafe impl FromBytes for PmuLookupTableEntry {} #[repr(C)] struct PmuLookupTableHeader { @@ -963,7 +952,13 @@ fn lookup_index(&self, idx: u8) -> Result { } let index = (usize::from(idx)) * usize::from(self.header.entry_len); - PmuLookupTableEntry::new(&self.table_data[index..]) + let (entry, _) = self + .table_data + .get(index..) + .and_then(PmuLookupTableEntry::from_bytes_copy_prefix) + .ok_or(EINVAL)?; + + Ok(entry) } // find entry by type value From 620e7ac19664eb1226b5ea226177bc6df91683db Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:31 +0900 Subject: [PATCH 054/131] gpu: nova-core: vbios: store PMU lookup entries in a KVVec The current code copies the data into a KVec and parses it on demand. We can simplify the code by storing the parsed entries. Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-13-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 56 +++++++++++----------------------- 1 file changed, 18 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 987eb1948314..1ffaf0ef56a7 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -917,8 +917,7 @@ unsafe impl FromBytes for PmuLookupTableHeader {} /// The table of entries is pointed to by the falcon data pointer in the BIT table, and is used to /// locate the Falcon Ucode. struct PmuLookupTable { - header: PmuLookupTableHeader, - table_data: KVec, + entries: KVVec, } impl PmuLookupTable { @@ -929,48 +928,29 @@ fn new(dev: &device::Device, data: &[u8]) -> Result { let entry_len = usize::from(header.entry_len); let entry_count = usize::from(header.entry_count); - let required_bytes = header_len + (entry_count * entry_len); + let data = data + .get(header_len..header_len + entry_count * entry_len) + .ok_or(EINVAL) + .inspect_err(|_| { + dev_err!(dev, "PmuLookupTable data length less than required\n"); + })?; - if data.len() < required_bytes { - dev_err!(dev, "PmuLookupTable data length less than required\n"); - return Err(EINVAL); + let mut entries = KVVec::with_capacity(entry_count, GFP_KERNEL)?; + for i in 0..entry_count { + let (entry, _) = PmuLookupTableEntry::from_bytes_copy_prefix(&data[i * entry_len..]) + .ok_or(EINVAL)?; + entries.push(entry, GFP_KERNEL)?; } - // Create a copy of only the table data - let table_data = { - let mut ret = KVec::new(); - ret.extend_from_slice(&data[header_len..required_bytes], GFP_KERNEL)?; - ret - }; - - Ok(PmuLookupTable { header, table_data }) - } - - fn lookup_index(&self, idx: u8) -> Result { - if idx >= self.header.entry_count { - return Err(EINVAL); - } - - let index = (usize::from(idx)) * usize::from(self.header.entry_len); - let (entry, _) = self - .table_data - .get(index..) - .and_then(PmuLookupTableEntry::from_bytes_copy_prefix) - .ok_or(EINVAL)?; - - Ok(entry) + Ok(PmuLookupTable { entries }) } // find entry by type value - fn find_entry_by_type(&self, entry_type: u8) -> Result { - for i in 0..self.header.entry_count { - let entry = self.lookup_index(i)?; - if entry.application_id == entry_type { - return Ok(entry); - } - } - - Err(EINVAL) + fn find_entry_by_type(&self, entry_type: u8) -> Result<&PmuLookupTableEntry> { + self.entries + .iter() + .find(|entry| entry.application_id == entry_type) + .ok_or(EINVAL) } } From 7e545bed7b1a2b2918052fe012945aafd49e64e7 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:32 +0900 Subject: [PATCH 055/131] gpu: nova-core: vbios: construct `FwSecBiosImage` directly from BIOS images `FwSecBiosBuilder` now only contains `falcon_ucode_offset` which just gets passed directly into `FwSecBiosImage`. Remove `FwSecBiosBuilder` and construct `FwSecBiosImage` directly, as a simplification. Reviewed-by: Joel Fernandes Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-14-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 88 +++++++++++----------------------- 1 file changed, 29 insertions(+), 59 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 1ffaf0ef56a7..9c5281e7a0d3 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -315,8 +315,8 @@ impl Vbios { pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { // Images to extract from iteration let mut pci_at_image: Option = None; - let mut first_fwsec_image: Option = None; - let mut second_fwsec_image: Option = None; + let mut first_fwsec_image: Option = None; + let mut second_fwsec_image: Option = None; // Parse all VBIOS images in the ROM for image_result in VbiosIterator::new(dev, bar0)? { @@ -336,14 +336,10 @@ pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { pci_at_image = Some(PciAtBiosImage::try_from(image)?); } Ok(BiosImageType::FwSec) => { - let fwsec = FwSecBiosBuilder { - base: image, - falcon_ucode_offset: None, - }; if first_fwsec_image.is_none() { - first_fwsec_image = Some(fwsec); + first_fwsec_image = Some(image); } else { - second_fwsec_image = Some(fwsec); + second_fwsec_image = Some(image); } } _ => { @@ -353,15 +349,13 @@ pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { } // Using all the images, setup the falcon data pointer in Fwsec. - if let (Some(mut second), Some(first), Some(pci_at)) = + if let (Some(second), Some(first), Some(pci_at)) = (second_fwsec_image, first_fwsec_image, pci_at_image) { - second - .setup_falcon_data(&pci_at, &first) + let fwsec_image = FwSecBiosImage::new(pci_at, first, second) .inspect_err(|e| dev_err!(dev, "Falcon data setup failed: {:?}\n", e))?; - Ok(Vbios { - fwsec_image: second.build()?, - }) + + Ok(Vbios { fwsec_image }) } else { dev_err!( dev, @@ -702,18 +696,6 @@ struct NbsiBiosImage { // NBSI-specific fields can be added here in the future. } -struct FwSecBiosBuilder { - base: BiosImage, - /// These are temporary fields that are used during the construction of the - /// [`FwSecBiosBuilder`]. - /// - /// Once FwSecBiosBuilder is constructed, the `falcon_ucode_offset` will be copied into a new - /// [`FwSecBiosImage`]. - /// - /// The offset of the Falcon ucode. - falcon_ucode_offset: Option, -} - /// The [`FwSecBiosImage`] structure contains the PMU table and the Falcon Ucode. /// /// The PMU table contains voltage/frequency tables as well as a pointer to the Falcon Ucode. @@ -954,32 +936,33 @@ fn find_entry_by_type(&self, entry_type: u8) -> Result<&PmuLookupTableEntry> { } } -impl FwSecBiosBuilder { - fn setup_falcon_data( - &mut self, - pci_at_image: &PciAtBiosImage, - first_fwsec: &FwSecBiosBuilder, - ) -> Result { +impl FwSecBiosImage { + /// Build the final `FwSecBiosImage` from the PCI-AT and FWSEC BIOS images. + fn new( + pci_at_image: PciAtBiosImage, + first_fwsec: BiosImage, + second_fwsec: BiosImage, + ) -> Result { let offset = pci_at_image.falcon_data_offset()?; // The offset is from the start of the first FwSec image, but it // may point into the second FwSec image. Treat the two FwSec images // as contiguous here and subtract the first image length when the // target lies in the second one. - let pmu_lookup_data = if offset < first_fwsec.base.data.len() { - first_fwsec.base.data.get(offset..) + let pmu_lookup_data = if offset < first_fwsec.data.len() { + first_fwsec.data.get(offset..) } else { - self.base.data.get(offset - first_fwsec.base.data.len()..) + second_fwsec.data.get(offset - first_fwsec.data.len()..) } .ok_or(EINVAL)?; - let pmu_lookup_table = PmuLookupTable::new(&self.base.dev, pmu_lookup_data)?; + let pmu_lookup_table = PmuLookupTable::new(&second_fwsec.dev, pmu_lookup_data)?; let entry = pmu_lookup_table .find_entry_by_type(FALCON_UCODE_ENTRY_APPID_FWSEC_PROD) .inspect_err(|e| { dev_err!( - self.base.dev, + second_fwsec.dev, "PmuLookupTableEntry not found, error: {:?}\n", e ); @@ -987,34 +970,21 @@ fn setup_falcon_data( let falcon_ucode_offset = usize::from_safe_cast(entry.data) .checked_sub(pci_at_image.base.data.len()) - .and_then(|o| o.checked_sub(first_fwsec.base.data.len())) + .and_then(|o| o.checked_sub(first_fwsec.data.len())) .ok_or(EINVAL) .inspect_err(|_| { - dev_err!(self.base.dev, "Falcon Ucode offset not in second Fwsec.\n"); + dev_err!( + second_fwsec.dev, + "Falcon Ucode offset not in second Fwsec.\n" + ); })?; - self.falcon_ucode_offset = Some(falcon_ucode_offset); - Ok(()) + Ok(FwSecBiosImage { + base: second_fwsec, + falcon_ucode_offset, + }) } - /// Build the final FwSecBiosImage from this builder - fn build(self) -> Result { - let ret = FwSecBiosImage { - base: self.base, - falcon_ucode_offset: self.falcon_ucode_offset.ok_or(EINVAL)?, - }; - - if cfg!(debug_assertions) { - // Print the desc header for debugging - let desc = ret.header()?; - dev_dbg!(ret.base.dev, "PmuLookupTableEntry desc: {:#?}\n", desc); - } - - Ok(ret) - } -} - -impl FwSecBiosImage { /// Get the FwSec header ([`FalconUCodeDesc`]). pub(crate) fn header(&self) -> Result { let data = self From 433730a61f13c77ab981db469983c4ea198e8895 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:33 +0900 Subject: [PATCH 056/131] gpu: nova-core: vbios: use the first PCI-AT image Currently, PCI-AT takes the final image if multiple exist. Use the first one instead, to match nouveau behaviour. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-15-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 9c5281e7a0d3..82811e42e858 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -333,7 +333,10 @@ pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { // Convert to a specific image type match BiosImageType::try_from(image.pcir.code_type) { Ok(BiosImageType::PciAt) => { - pci_at_image = Some(PciAtBiosImage::try_from(image)?); + // Silently ignore any extra PCI-AT images. + if pci_at_image.is_none() { + pci_at_image = Some(PciAtBiosImage::try_from(image)?); + } } Ok(BiosImageType::FwSec) => { if first_fwsec_image.is_none() { From 43e7bef8c05471fd8f44dec17994ef247092df26 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:34 +0900 Subject: [PATCH 057/131] gpu: nova-core: vbios: use single logical block for the FWSEC section Currently, FWSEC takes the first image and the last image. Treat the first FWSEC image and all following image data as one logical block for building the final FWSEC image. This avoids explicitly tracking two FWSEC images. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-16-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 99 +++++++++++++--------------------- 1 file changed, 37 insertions(+), 62 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 82811e42e858..5266e15793cf 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -84,7 +84,7 @@ struct VbiosIterator<'a> { /// progressively extends. It is used so that we do not re-read any contents that are already /// read as we use the cumulative length read so far, and re-read any gaps as we extend the /// length. - data: KVec, + data: KVVec, /// Current offset of the [`Iterator`]. current_offset: usize, /// Indicate whether the last image has been found. @@ -177,7 +177,7 @@ fn new(dev: &'a device::Device, bar0: &'a Bar0) -> Result { Ok(Self { dev, bar0, - data: KVec::new(), + data: KVVec::new(), current_offset: vbios_rom_offset(dev, bar0)?, last_found: false, }) @@ -315,8 +315,7 @@ impl Vbios { pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { // Images to extract from iteration let mut pci_at_image: Option = None; - let mut first_fwsec_image: Option = None; - let mut second_fwsec_image: Option = None; + let mut fwsec_section: Option> = None; // Parse all VBIOS images in the ROM for image_result in VbiosIterator::new(dev, bar0)? { @@ -330,6 +329,13 @@ pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { image.is_last() ); + // Once we have found the first FWSEC image, grab all data after that as the FWSEC + // section. This is indexed as one logical block to build the final FWSEC image. + if let Some(data) = fwsec_section.as_mut() { + data.extend_from_slice(&image.data, GFP_KERNEL)?; + continue; + } + // Convert to a specific image type match BiosImageType::try_from(image.pcir.code_type) { Ok(BiosImageType::PciAt) => { @@ -338,13 +344,7 @@ pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { pci_at_image = Some(PciAtBiosImage::try_from(image)?); } } - Ok(BiosImageType::FwSec) => { - if first_fwsec_image.is_none() { - first_fwsec_image = Some(image); - } else { - second_fwsec_image = Some(image); - } - } + Ok(BiosImageType::FwSec) => fwsec_section = Some(image.data), _ => { // Ignore other image types or unknown types } @@ -352,10 +352,8 @@ pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { } // Using all the images, setup the falcon data pointer in Fwsec. - if let (Some(second), Some(first), Some(pci_at)) = - (second_fwsec_image, first_fwsec_image, pci_at_image) - { - let fwsec_image = FwSecBiosImage::new(pci_at, first, second) + if let (Some(pci_at), Some(fwsec_section)) = (pci_at_image, fwsec_section) { + let fwsec_image = FwSecBiosImage::new(dev, pci_at, fwsec_section) .inspect_err(|e| dev_err!(dev, "Falcon data setup failed: {:?}\n", e))?; Ok(Vbios { fwsec_image }) @@ -703,7 +701,10 @@ struct NbsiBiosImage { /// /// The PMU table contains voltage/frequency tables as well as a pointer to the Falcon Ucode. pub(crate) struct FwSecBiosImage { - base: BiosImage, + /// Used for logging. + dev: ARef, + /// FWSEC data. + data: KVVec, /// The offset of the Falcon ucode. falcon_ucode_offset: usize, } @@ -713,8 +714,6 @@ pub(crate) struct FwSecBiosImage { /// A BiosImage struct is embedded into all image types and implements common operations. #[expect(dead_code)] struct BiosImage { - /// Used for logging. - dev: ARef, /// PCI ROM Expansion Header rom_header: PciRomHeader, /// PCI Data Structure @@ -722,7 +721,7 @@ struct BiosImage { /// NVIDIA PCI Data Extension (optional) npde: Option, /// Image data (includes ROM header and PCIR) - data: KVec, + data: KVVec, } impl BiosImage { @@ -795,11 +794,10 @@ fn new(dev: &device::Device, data: &[u8]) -> Result { let npde = NpdeStruct::find_in_data(dev, data, &rom_header, &pcir); // Create a copy of the data. - let mut data_copy = KVec::new(); + let mut data_copy = KVVec::new(); data_copy.extend_from_slice(data, GFP_KERNEL)?; Ok(BiosImage { - dev: dev.into(), rom_header, pcir, npde, @@ -837,7 +835,7 @@ fn get_bit_token(&self, token_id: u8) -> Result { /// treats the PCI-AT and FWSEC images as logically contiguous even when an EFI image sits in /// between them, so subtract the PCI-AT image size here to convert it to a FWSEC-relative /// offset. - fn falcon_data_offset(&self) -> Result { + fn falcon_data_offset(&self, dev: &device::Device) -> Result { let token = self.get_bit_token(BIT_TOKEN_ID_FALCON_DATA)?; let offset = usize::from(token.data_offset); @@ -852,7 +850,7 @@ fn falcon_data_offset(&self) -> Result { .checked_sub(data.len()) .ok_or(EINVAL) .inspect_err(|_| { - dev_err!(self.base.dev, "Falcon data pointer out of bounds\n"); + dev_err!(dev, "Falcon data pointer out of bounds\n"); }) } } @@ -942,59 +940,38 @@ fn find_entry_by_type(&self, entry_type: u8) -> Result<&PmuLookupTableEntry> { impl FwSecBiosImage { /// Build the final `FwSecBiosImage` from the PCI-AT and FWSEC BIOS images. fn new( + dev: &device::Device, pci_at_image: PciAtBiosImage, - first_fwsec: BiosImage, - second_fwsec: BiosImage, + data: KVVec, ) -> Result { - let offset = pci_at_image.falcon_data_offset()?; + let offset = pci_at_image.falcon_data_offset(dev)?; - // The offset is from the start of the first FwSec image, but it - // may point into the second FwSec image. Treat the two FwSec images - // as contiguous here and subtract the first image length when the - // target lies in the second one. - let pmu_lookup_data = if offset < first_fwsec.data.len() { - first_fwsec.data.get(offset..) - } else { - second_fwsec.data.get(offset - first_fwsec.data.len()..) - } - .ok_or(EINVAL)?; - - let pmu_lookup_table = PmuLookupTable::new(&second_fwsec.dev, pmu_lookup_data)?; + let pmu_lookup_data = data.get(offset..).ok_or(EINVAL)?; + let pmu_lookup_table = PmuLookupTable::new(dev, pmu_lookup_data)?; let entry = pmu_lookup_table .find_entry_by_type(FALCON_UCODE_ENTRY_APPID_FWSEC_PROD) .inspect_err(|e| { - dev_err!( - second_fwsec.dev, - "PmuLookupTableEntry not found, error: {:?}\n", - e - ); + dev_err!(dev, "PmuLookupTableEntry not found, error: {:?}\n", e); })?; let falcon_ucode_offset = usize::from_safe_cast(entry.data) .checked_sub(pci_at_image.base.data.len()) - .and_then(|o| o.checked_sub(first_fwsec.data.len())) .ok_or(EINVAL) .inspect_err(|_| { - dev_err!( - second_fwsec.dev, - "Falcon Ucode offset not in second Fwsec.\n" - ); + dev_err!(dev, "Falcon Ucode offset not in Fwsec.\n"); })?; Ok(FwSecBiosImage { - base: second_fwsec, + dev: dev.into(), + data, falcon_ucode_offset, }) } /// Get the FwSec header ([`FalconUCodeDesc`]). pub(crate) fn header(&self) -> Result { - let data = self - .base - .data - .get(self.falcon_ucode_offset..) - .ok_or(EINVAL)?; + let data = self.data.get(self.falcon_ucode_offset..).ok_or(EINVAL)?; // Read the version byte from the header. let ver = data.get(1).copied().ok_or(EINVAL)?; @@ -1012,7 +989,7 @@ pub(crate) fn header(&self) -> Result { Ok(FalconUCodeDesc::V3(v3)) } _ => { - dev_err!(self.base.dev, "invalid fwsec firmware version: {:?}\n", ver); + dev_err!(self.dev, "invalid fwsec firmware version: {:?}\n", ver); Err(EINVAL) } } @@ -1027,15 +1004,14 @@ pub(crate) fn ucode(&self, desc: &FalconUCodeDesc) -> Result<&[u8]> { ); // The ucode data follows the descriptor. - self.base - .data + self.data .get(self.falcon_ucode_offset..) .and_then(|data| data.get(desc.size()..)) .and_then(|data| data.get(..size)) .ok_or(ERANGE) .inspect_err(|_| { dev_err!( - self.base.dev, + self.dev, "fwsec ucode data not contained within BIOS bounds\n" ) }) @@ -1053,9 +1029,9 @@ pub(crate) fn sigs(&self, desc: &FalconUCodeDesc) -> Result<&[Bcrt30Rsa3kSignatu let sigs_size = sigs_count * core::mem::size_of::(); // Make sure the data is within bounds. - if sigs_data_offset + sigs_size > self.base.data.len() { + if sigs_data_offset + sigs_size > self.data.len() { dev_err!( - self.base.dev, + self.dev, "fwsec signatures data not contained within BIOS bounds\n" ); return Err(ERANGE); @@ -1065,8 +1041,7 @@ pub(crate) fn sigs(&self, desc: &FalconUCodeDesc) -> Result<&[Bcrt30Rsa3kSignatu // sizeof::()` is within the bounds of `data`. Ok(unsafe { core::slice::from_raw_parts( - self.base - .data + self.data .as_ptr() .add(sigs_data_offset) .cast::(), From e8baefdffd4e131138e5c17ae3c5b2b2a907ba3f Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:35 +0900 Subject: [PATCH 058/131] gpu: nova-core: vbios: use let-else in Vbios::new Improve readability by moving the success path outside of a nested branch. Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-17-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 5266e15793cf..52e33fdd4f5d 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -352,18 +352,18 @@ pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { } // Using all the images, setup the falcon data pointer in Fwsec. - if let (Some(pci_at), Some(fwsec_section)) = (pci_at_image, fwsec_section) { - let fwsec_image = FwSecBiosImage::new(dev, pci_at, fwsec_section) - .inspect_err(|e| dev_err!(dev, "Falcon data setup failed: {:?}\n", e))?; - - Ok(Vbios { fwsec_image }) - } else { + let (Some(pci_at), Some(fwsec_section)) = (pci_at_image, fwsec_section) else { dev_err!( dev, "Missing required images for falcon data setup, skipping\n" ); - Err(EINVAL) - } + return Err(EINVAL); + }; + + let fwsec_image = FwSecBiosImage::new(dev, pci_at, fwsec_section) + .inspect_err(|e| dev_err!(dev, "Falcon data setup failed: {:?}\n", e))?; + + Ok(Vbios { fwsec_image }) } pub(crate) fn fwsec_image(&self) -> &FwSecBiosImage { From 84eb369da61320152a7bbeb245469311cf31205f Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:36 +0900 Subject: [PATCH 059/131] gpu: nova-core: vbios: remove unnecessary fields in PciRomHeader Remove unnecessary fields in PciRomHeader. This allows a simplification to use `FromBytes` instead of reading fields piecemeal. A lot of these checks were redundant as well since it checks the size of the `data` first in `BiosImage`. Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-18-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 68 ++++++++-------------------------- 1 file changed, 16 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 52e33fdd4f5d..7399f2d087a9 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -533,67 +533,38 @@ fn from_id(image: &PciAtBiosImage, token_id: u8) -> Result { /// PCI ROM Expansion Header as defined in PCI Firmware Specification. /// -/// This is header is at the beginning of every image in the set of images in the ROM. It contains -/// a pointer to the PCI Data Structure which describes the image. For "NBSI" images (NoteBook -/// System Information), the ROM header deviates from the standard and contains an offset to the -/// NBSI image however we do not yet parse that in this module and keep it for future reference. +/// This header is at the beginning of every image in the set of images in the ROM. It contains a +/// pointer to the PCI Data Structure which describes the image. #[derive(Debug, Clone, Copy)] -#[expect(dead_code)] +#[repr(C)] struct PciRomHeader { /// 00h: Signature (0xAA55) signature: u16, - /// 02h: Reserved bytes for processor architecture unique data (20 bytes) - reserved: [u8; 20], - /// 16h: NBSI Data Offset (NBSI-specific, offset from header to NBSI image) - nbsi_data_offset: Option, + /// 02h: Reserved bytes for processor architecture unique data (22 bytes) + reserved: [u8; 22], /// 18h: Pointer to PCI Data Structure (offset from start of ROM image) pci_data_struct_offset: u16, - /// 1Ah: Size of block (this is NBSI-specific) - size_of_block: Option, } +// SAFETY: all bit patterns are valid for `PciRomHeader`. +unsafe impl FromBytes for PciRomHeader {} + impl PciRomHeader { fn new(dev: &device::Device, data: &[u8]) -> Result { - if data.len() < 26 { - // Need at least 26 bytes to read pciDataStrucPtr and sizeOfBlock. - return Err(EINVAL); - } - - let signature = u16::from_le_bytes([data[0], data[1]]); + let (rom_header, _) = PciRomHeader::from_bytes_copy_prefix(data) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "Not enough data for ROM header\n"))?; // Check for valid ROM signatures. - match signature { + match rom_header.signature { 0xAA55 | 0x4E56 => {} _ => { - dev_err!(dev, "ROM signature unknown {:#x}\n", signature); + dev_err!(dev, "ROM signature unknown {:#x}\n", rom_header.signature); return Err(EINVAL); } } - // Read the pointer to the PCI Data Structure at offset 0x18. - let pci_data_struct_ptr = u16::from_le_bytes([data[24], data[25]]); - - // Try to read optional fields if enough data. - let mut size_of_block = None; - let mut nbsi_data_offset = None; - - if data.len() >= 30 { - // Read size_of_block at offset 0x1A. - size_of_block = Some(u32::from_le_bytes([data[26], data[27], data[28], data[29]])); - } - - // For NBSI images, try to read the nbsiDataOffset at offset 0x16. - if data.len() >= 24 { - nbsi_data_offset = Some(u16::from_le_bytes([data[22], data[23]])); - } - - Ok(PciRomHeader { - signature, - reserved: [0u8; 20], - pci_data_struct_offset: pci_data_struct_ptr, - size_of_block, - nbsi_data_offset, - }) + Ok(rom_header) } } @@ -712,9 +683,9 @@ pub(crate) struct FwSecBiosImage { /// BIOS Image structure containing various headers and reference fields to all BIOS images. /// /// A BiosImage struct is embedded into all image types and implements common operations. -#[expect(dead_code)] struct BiosImage { /// PCI ROM Expansion Header + #[expect(dead_code)] rom_header: PciRomHeader, /// PCI Data Structure pcir: PcirStruct, @@ -759,15 +730,8 @@ fn is_last(&self) -> bool { /// Creates a new BiosImage from raw byte data. fn new(dev: &device::Device, data: &[u8]) -> Result { - // Ensure we have enough data for the ROM header. - if data.len() < 26 { - dev_err!(dev, "Not enough data for ROM header\n"); - return Err(EINVAL); - } - // Parse the ROM header. - let rom_header = PciRomHeader::new(dev, &data[0..26]) - .inspect_err(|e| dev_err!(dev, "Failed to create PciRomHeader: {:?}\n", e))?; + let rom_header = PciRomHeader::new(dev, data)?; // Get the PCI Data Structure using the pointer from the ROM header. let pcir_offset = usize::from(rom_header.pci_data_struct_offset); From 91a8ec505e0961a67fc763d3fd80bf98e7cf08d5 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:37 +0900 Subject: [PATCH 060/131] gpu: nova-core: vbios: drop unused image wrappers These are unused currently, and it is probably sufficient to just check the type of BIOS image in the future. Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-19-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 7399f2d087a9..5401702acee4 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -656,18 +656,6 @@ struct PciAtBiosImage { bit_offset: usize, } -#[expect(dead_code)] -struct EfiBiosImage { - base: BiosImage, - // EFI-specific fields can be added here in the future. -} - -#[expect(dead_code)] -struct NbsiBiosImage { - base: BiosImage, - // NBSI-specific fields can be added here in the future. -} - /// The [`FwSecBiosImage`] structure contains the PMU table and the Falcon Ucode. /// /// The PMU table contains voltage/frequency tables as well as a pointer to the Falcon Ucode. From 16c41263240eac53031fc35cb2139339a7922ff0 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:38 +0900 Subject: [PATCH 061/131] gpu: nova-core: vbios: drop redundant TryFrom import This is unused. Reviewed-by: John Hubbard Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-20-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 5401702acee4..6492f158d11e 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -2,8 +2,6 @@ //! VBIOS extraction and parsing. -use core::convert::TryFrom; - use kernel::{ device, io::Io, From c70fe8b2bacf6208263b728a742538f74c5da1ec Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:39 +0900 Subject: [PATCH 062/131] gpu: nova-core: vbios: move constants and functions to be associated Move constants and functions to be inside the impls of the types they are related to. This makes it more obvious what each type and value is for. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-21-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- Documentation/gpu/nova/core/vbios.rst | 2 +- drivers/gpu/nova-core/vbios.rs | 220 +++++++++++++------------- 2 files changed, 113 insertions(+), 109 deletions(-) diff --git a/Documentation/gpu/nova/core/vbios.rst b/Documentation/gpu/nova/core/vbios.rst index a4fe63422ede..9d3379ccfb30 100644 --- a/Documentation/gpu/nova/core/vbios.rst +++ b/Documentation/gpu/nova/core/vbios.rst @@ -232,7 +232,7 @@ Falcon data in the VBIOS which contains the PMU lookup table. This lookup table used to find the required Falcon ucode based on an application ID. The location of the PMU lookup table is found by scanning the BIT (`BIOS Information Table`_) -tokens for a token with the id `BIT_TOKEN_ID_FALCON_DATA` (0x70) which indicates the +tokens for a token with the Falcon data token id (0x70) which indicates the offset of the same from the start of the VBIOS image. Unfortunately, the offset does not account for the EFI image located between the PciAt and FwSec images. The `vbios.rs` code compensates for this with appropriate arithmetic. diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index 6492f158d11e..b14f9ebdc68f 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -27,16 +27,6 @@ num::FromSafeCast, }; -/// The offset of the VBIOS ROM in the BAR0 space. -const ROM_OFFSET: usize = 0x300000; -/// The maximum length of the VBIOS ROM to scan into. -const BIOS_MAX_SCAN_LEN: usize = 0x100000; -/// The size to read ahead when parsing initial BIOS image headers. -const BIOS_READ_AHEAD_SIZE: usize = 1024; -/// The bit in the last image indicator byte for the PCI Data Structure that -/// indicates the last image. Bit 0-6 are reserved, bit 7 is last image bit. -const LAST_IMAGE_BIT_MASK: u8 = 0x80; - /// BIOS Image Type from PCI Data Structure code_type field. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u8)] @@ -65,14 +55,6 @@ fn try_from(code: u8) -> Result { } } -// PMU lookup table entry types. Used to locate PMU table entries -// in the Fwsec image, corresponding to falcon ucodes. -#[expect(dead_code)] -const FALCON_UCODE_ENTRY_APPID_FIRMWARE_SEC_LIC: u8 = 0x05; -#[expect(dead_code)] -const FALCON_UCODE_ENTRY_APPID_FWSEC_DBG: u8 = 0x45; -const FALCON_UCODE_ENTRY_APPID_FWSEC_PROD: u8 = 0x85; - /// Vbios Reader for constructing the VBIOS data. struct VbiosIterator<'a> { dev: &'a device::Device, @@ -89,94 +71,99 @@ struct VbiosIterator<'a> { last_found: bool, } -// IFR Header in VBIOS. +impl<'a> VbiosIterator<'a> { + /// The offset of the VBIOS ROM in the BAR0 space. + const ROM_OFFSET: usize = 0x300000; + /// The maximum length of the VBIOS ROM to scan into. + const BIOS_MAX_SCAN_LEN: usize = 0x100000; + /// The size to read ahead when parsing initial BIOS image headers. + const BIOS_READ_AHEAD_SIZE: usize = 1024; -register! { - pub(crate) NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 { - 31:0 signature; - } -} - -register! { - pub(crate) NV_PBUS_IFR_FMT_FIXED1(u32) @ 0x300004 { - 30:16 fixed_data_size; - 15:8 version => u8; - } -} - -register! { - pub(crate) NV_PBUS_IFR_FMT_FIXED2(u32) @ 0x300008 { - 19:0 total_data_size; - } -} - -/// Return the byte offset where the PCI Expansion ROM images begin in the GPU's ROM. -/// -/// The GPU's ROM may begin with an Init-from-ROM (IFR) header that precedes -/// the PCI Expansion ROM images (VBIOS). When present, the PROM shadow -/// method must parse this header to determine the offset where the PCI ROM -/// images actually begin, and adjust all subsequent reads accordingly. -/// -/// On most GPUs this is not needed because the IFR microcode has already -/// applied the ROM offset so that PROM reads transparently skip the header. -/// On GA100, for some reason, the IFR offset is not applied to PROM -/// reads. Therefore, the search for the PCI expansion must skip the IFR -/// header, if found. -fn vbios_rom_offset(dev: &device::Device, bar0: &Bar0) -> Result { - /// IFR signature. - const NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE: u32 = u32::from_le_bytes(*b"NVGI"); - /// ROM directory signature. - const NV_ROM_DIRECTORY_IDENTIFIER: u32 = u32::from_le_bytes(*b"RFRD"); - /// Offset of the NV_PMGR_ROM_ADDR_OFFSET register in IFR Extended section. - const IFR_SW_EXT_ROM_ADDR_OFFSET: usize = 4; - /// Size of Redundant Firmware Flash Status section. - const RFW_FLASH_STATUS_SIZE: usize = SZ_4K; - /// Offset in the ROM Directory of the PCI Option ROM offset - const PCI_OPTION_ROM_OFFSET: usize = 8; - - let signature = bar0.read(NV_PBUS_IFR_FMT_FIXED0).signature(); - - if signature == NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE { - let fixed1 = bar0.read(NV_PBUS_IFR_FMT_FIXED1); - - match fixed1.version() { - 1 | 2 => { - let fixed_data_size = usize::from(fixed1.fixed_data_size()); - let pmgr_rom_addr_offset = fixed_data_size + IFR_SW_EXT_ROM_ADDR_OFFSET; - bar0.try_read32(ROM_OFFSET + pmgr_rom_addr_offset) - .map(usize::from_safe_cast) - } - 3 => { - let fixed2 = bar0.read(NV_PBUS_IFR_FMT_FIXED2); - let total_data_size = usize::from(fixed2.total_data_size()); - let flash_status_offset = - usize::from_safe_cast(bar0.try_read32(ROM_OFFSET + total_data_size)?); - let dir_offset = flash_status_offset + RFW_FLASH_STATUS_SIZE; - let dir_sig = bar0.try_read32(ROM_OFFSET + dir_offset)?; - if dir_sig != NV_ROM_DIRECTORY_IDENTIFIER { - dev_err!(dev, "could not find IFR ROM directory\n"); - return Err(EINVAL); - } - bar0.try_read32(ROM_OFFSET + dir_offset + PCI_OPTION_ROM_OFFSET) - .map(usize::from_safe_cast) - } - _ => { - dev_err!(dev, "unsupported IFR header version {}\n", fixed1.version()); - Err(EINVAL) + /// Return the byte offset where the PCI Expansion ROM images begin in the GPU's ROM. + /// + /// The GPU's ROM may begin with an Init-from-ROM (IFR) header that precedes the PCI Expansion + /// ROM images (VBIOS). When present, the PROM shadow method must parse this header to determine + /// the offset where the PCI ROM images actually begin, and adjust all subsequent reads + /// accordingly. + /// + /// On most GPUs this is not needed because the IFR microcode has already applied the ROM offset + /// so that PROM reads transparently skip the header. On GA100, for some reason, the IFR offset + /// is not applied to PROM reads. Therefore, the search for the PCI expansion must skip the IFR + /// header, if found. + fn rom_offset(dev: &device::Device, bar0: &Bar0) -> Result { + // IFR Header in VBIOS. + register! { + NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 { + 31:0 signature; } } - } else { - Ok(0) - } -} -impl<'a> VbiosIterator<'a> { + register! { + NV_PBUS_IFR_FMT_FIXED1(u32) @ 0x300004 { + 30:16 fixed_data_size; + 15:8 version => u8; + } + } + + register! { + NV_PBUS_IFR_FMT_FIXED2(u32) @ 0x300008 { + 19:0 total_data_size; + } + } + + /// IFR signature. + const NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE: u32 = u32::from_le_bytes(*b"NVGI"); + /// ROM directory signature. + const NV_ROM_DIRECTORY_IDENTIFIER: u32 = u32::from_le_bytes(*b"RFRD"); + /// Offset of the NV_PMGR_ROM_ADDR_OFFSET register in IFR Extended section. + const IFR_SW_EXT_ROM_ADDR_OFFSET: usize = 4; + /// Size of Redundant Firmware Flash Status section. + const RFW_FLASH_STATUS_SIZE: usize = SZ_4K; + /// Offset in the ROM Directory of the PCI Option ROM offset. + const PCI_OPTION_ROM_OFFSET: usize = 8; + + let signature = bar0.read(NV_PBUS_IFR_FMT_FIXED0).signature(); + + if signature == NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE { + let fixed1 = bar0.read(NV_PBUS_IFR_FMT_FIXED1); + + match fixed1.version() { + 1 | 2 => { + let fixed_data_size = usize::from(fixed1.fixed_data_size()); + let pmgr_rom_addr_offset = fixed_data_size + IFR_SW_EXT_ROM_ADDR_OFFSET; + bar0.try_read32(Self::ROM_OFFSET + pmgr_rom_addr_offset) + .map(usize::from_safe_cast) + } + 3 => { + let fixed2 = bar0.read(NV_PBUS_IFR_FMT_FIXED2); + let total_data_size = usize::from(fixed2.total_data_size()); + let flash_status_offset = + usize::from_safe_cast(bar0.try_read32(Self::ROM_OFFSET + total_data_size)?); + let dir_offset = flash_status_offset + RFW_FLASH_STATUS_SIZE; + let dir_sig = bar0.try_read32(Self::ROM_OFFSET + dir_offset)?; + if dir_sig != NV_ROM_DIRECTORY_IDENTIFIER { + dev_err!(dev, "could not find IFR ROM directory\n"); + return Err(EINVAL); + } + bar0.try_read32(Self::ROM_OFFSET + dir_offset + PCI_OPTION_ROM_OFFSET) + .map(usize::from_safe_cast) + } + _ => { + dev_err!(dev, "unsupported IFR header version {}\n", fixed1.version()); + Err(EINVAL) + } + } + } else { + Ok(0) + } + } + fn new(dev: &'a device::Device, bar0: &'a Bar0) -> Result { Ok(Self { dev, bar0, data: KVVec::new(), - current_offset: vbios_rom_offset(dev, bar0)?, + current_offset: Self::rom_offset(dev, bar0)?, last_found: false, }) } @@ -186,7 +173,7 @@ fn read_more(&mut self, len: usize) -> Result { let start = self.data.len(); let end = start + len; - if end > BIOS_MAX_SCAN_LEN { + if end > Self::BIOS_MAX_SCAN_LEN { dev_err!(self.dev, "Error: exceeded BIOS scan limit.\n"); return Err(EINVAL); } @@ -205,7 +192,7 @@ fn read_more(&mut self, len: usize) -> Result { // Read ROM data bytes and push directly to `data`. for addr in (start..end).step_by(core::mem::size_of::()) { // Read 32-bit word from the VBIOS ROM - let word = self.bar0.try_read32(ROM_OFFSET + addr)?; + let word = self.bar0.try_read32(Self::ROM_OFFSET + addr)?; // Convert the `u32` to a 4 byte array and push each byte. word.to_ne_bytes() @@ -267,7 +254,7 @@ fn next(&mut self) -> Option { return None; } - if self.current_offset >= BIOS_MAX_SCAN_LEN { + if self.current_offset >= Self::BIOS_MAX_SCAN_LEN { dev_err!(self.dev, "Error: exceeded BIOS scan limit, stopping scan\n"); return None; } @@ -275,7 +262,7 @@ fn next(&mut self) -> Option { // Parse image headers first to get image size. let image_size = match self.read_bios_image_at_offset( self.current_offset, - BIOS_READ_AHEAD_SIZE, + Self::BIOS_READ_AHEAD_SIZE, "parse initial BIOS image headers", ) { Ok(image) => image.image_size_bytes(), @@ -403,6 +390,9 @@ struct PcirStruct { unsafe impl FromBytes for PcirStruct {} impl PcirStruct { + /// The bit in `last_image` that indicates the last image. + const LAST_IMAGE_BIT_MASK: u8 = 0x80; + fn new(dev: &device::Device, data: &[u8]) -> Result { let (pcir, _) = PcirStruct::from_bytes_copy_prefix(data).ok_or(EINVAL)?; @@ -426,7 +416,7 @@ fn new(dev: &device::Device, data: &[u8]) -> Result { /// Check if this is the last image in the ROM. fn is_last(&self) -> bool { - self.last_image & LAST_IMAGE_BIT_MASK != 0 + self.last_image & Self::LAST_IMAGE_BIT_MASK != 0 } /// Calculate image size in bytes from 512-byte blocks. @@ -492,10 +482,10 @@ struct BitToken { // SAFETY: all bit patterns are valid for `BitToken`. unsafe impl FromBytes for BitToken {} -// Define the token ID for the Falcon data -const BIT_TOKEN_ID_FALCON_DATA: u8 = 0x70; - impl BitToken { + /// BIT token ID for Falcon data. + const ID_FALCON_DATA: u8 = 0x70; + /// Find a BIT token entry by BIT ID in a PciAtBiosImage fn from_id(image: &PciAtBiosImage, token_id: u8) -> Result { let header = &image.bit_header; @@ -591,6 +581,9 @@ struct NpdeStruct { unsafe impl FromBytes for NpdeStruct {} impl NpdeStruct { + /// The bit in `last_image` that indicates the last image. + const LAST_IMAGE_BIT_MASK: u8 = 0x80; + fn new(dev: &device::Device, data: &[u8]) -> Option { let (npde, _) = NpdeStruct::from_bytes_copy_prefix(data)?; @@ -614,7 +607,7 @@ fn new(dev: &device::Device, data: &[u8]) -> Option { /// Check if this is the last image in the ROM. fn is_last(&self) -> bool { - self.last_image & LAST_IMAGE_BIT_MASK != 0 + self.last_image & Self::LAST_IMAGE_BIT_MASK != 0 } /// Calculate image size in bytes from 512-byte blocks. @@ -786,7 +779,7 @@ fn get_bit_token(&self, token_id: u8) -> Result { /// between them, so subtract the PCI-AT image size here to convert it to a FWSEC-relative /// offset. fn falcon_data_offset(&self, dev: &device::Device) -> Result { - let token = self.get_bit_token(BIT_TOKEN_ID_FALCON_DATA)?; + let token = self.get_bit_token(BitToken::ID_FALCON_DATA)?; let offset = usize::from(token.data_offset); // Read the 4-byte falcon data pointer at the offset specified in the token. @@ -833,6 +826,17 @@ struct PmuLookupTableEntry { // SAFETY: all bit patterns are valid for `PmuLookupTableEntry`. unsafe impl FromBytes for PmuLookupTableEntry {} +impl PmuLookupTableEntry { + /// PMU lookup table application ID for firmware security license ucode. + #[expect(dead_code)] + const APPID_FIRMWARE_SEC_LIC: u8 = 0x05; + /// PMU lookup table application ID for debug FWSEC ucode. + #[expect(dead_code)] + const APPID_FWSEC_DBG: u8 = 0x45; + /// PMU lookup table application ID for production FWSEC ucode. + const APPID_FWSEC_PROD: u8 = 0x85; +} + #[repr(C)] struct PmuLookupTableHeader { version: u8, @@ -900,7 +904,7 @@ fn new( let pmu_lookup_table = PmuLookupTable::new(dev, pmu_lookup_data)?; let entry = pmu_lookup_table - .find_entry_by_type(FALCON_UCODE_ENTRY_APPID_FWSEC_PROD) + .find_entry_by_type(PmuLookupTableEntry::APPID_FWSEC_PROD) .inspect_err(|e| { dev_err!(dev, "PmuLookupTableEntry not found, error: {:?}\n", e); })?; From 2cf1840b0fa7637b6731fd554529f8d57ea34c04 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Mon, 25 May 2026 22:57:40 +0900 Subject: [PATCH 063/131] gpu: nova-core: vbios: remove unused rom_header field This is only used during construction, so we can remove it. Signed-off-by: Eliot Courtney Link: https://patch.msgid.link/20260525-fix-vbios-v5-22-e5e455251537@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/vbios.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index b14f9ebdc68f..c0bc1008ed75 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -663,9 +663,6 @@ pub(crate) struct FwSecBiosImage { /// /// A BiosImage struct is embedded into all image types and implements common operations. struct BiosImage { - /// PCI ROM Expansion Header - #[expect(dead_code)] - rom_header: PciRomHeader, /// PCI Data Structure pcir: PcirStruct, /// NVIDIA PCI Data Extension (optional) @@ -741,7 +738,6 @@ fn new(dev: &device::Device, data: &[u8]) -> Result { data_copy.extend_from_slice(data, GFP_KERNEL)?; Ok(BiosImage { - rom_header, pcir, npde, data: data_copy, From e566a9e17f3774c962b6d2522750f227f027edc6 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:20:48 +0200 Subject: [PATCH 064/131] rust: pci: use 'static lifetime for PCI BAR resource names pci_request_region() stores the name pointer directly in struct resource; use &'static CStr to ensure the pointer remains valid even if the Bar is leaked. Cc: stable@vger.kernel.org Reported-by: Sashiko Closes: https://lore.kernel.org/all/20260522004943.CDA7C1F000E9@smtp.kernel.org/ Fixes: 3c2e31d717ac ("rust: pci: move I/O infrastructure to separate file") Reviewed-by: Alexandre Courbot Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-2-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/pci/io.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index ae78676c927f..3ce21482b079 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -153,7 +153,7 @@ pub struct Bar { } impl Bar { - pub(super) fn new(pdev: &Device, num: u32, name: &CStr) -> Result { + pub(super) fn new(pdev: &Device, num: u32, name: &'static CStr) -> Result { let len = pdev.resource_len(num)?; if len == 0 { return Err(ENOMEM); @@ -252,7 +252,7 @@ impl Device { pub fn iomap_region_sized<'a, const SIZE: usize>( &'a self, bar: u32, - name: &'a CStr, + name: &'static CStr, ) -> impl PinInit>, Error> + 'a { Devres::new(self.as_ref(), Bar::::new(self, bar, name)) } @@ -261,7 +261,7 @@ pub fn iomap_region_sized<'a, const SIZE: usize>( pub fn iomap_region<'a>( &'a self, bar: u32, - name: &'a CStr, + name: &'static CStr, ) -> impl PinInit, Error> + 'a { self.iomap_region_sized::<0>(bar, name) } From e9df918d61e08f4281c3bcd42486f1505f396b1d Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 25 May 2026 22:20:49 +0200 Subject: [PATCH 065/131] rust: alloc: remove `'static` bound on `ForeignOwnable` The `'static` bound is currently necessary because there's no restriction on the lifetime of the GAT. Add a `Self: 'a` bound to restrict possible lifetimes on `Borrowed` and `BorrowedMut`, and lift the `'static` requirement. Reviewed-by: Alexandre Courbot Reviewed-by: Greg Kroah-Hartman Signed-off-by: Gary Guo Acked-by: Miguel Ojeda Link: https://patch.msgid.link/20260525202921.124698-3-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/alloc/kbox.rs | 24 ++++++++++++++++++------ rust/kernel/types.rs | 8 ++++++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index c824ed6e1523..2f8c16473c2c 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -477,7 +477,7 @@ fn try_init(init: impl Init, flags: Flags) -> Result // SAFETY: The pointer returned by `into_foreign` comes from a well aligned // pointer to `T` allocated by `A`. -unsafe impl ForeignOwnable for Box +unsafe impl ForeignOwnable for Box where A: Allocator, { @@ -487,8 +487,14 @@ unsafe impl ForeignOwnable for Box core::mem::align_of::() }; - type Borrowed<'a> = &'a T; - type BorrowedMut<'a> = &'a mut T; + type Borrowed<'a> + = &'a T + where + Self: 'a; + type BorrowedMut<'a> + = &'a mut T + where + Self: 'a; fn into_foreign(self) -> *mut c_void { Box::into_raw(self).cast() @@ -516,13 +522,19 @@ unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a mut T { // SAFETY: The pointer returned by `into_foreign` comes from a well aligned // pointer to `T` allocated by `A`. -unsafe impl ForeignOwnable for Pin> +unsafe impl ForeignOwnable for Pin> where A: Allocator, { const FOREIGN_ALIGN: usize = as ForeignOwnable>::FOREIGN_ALIGN; - type Borrowed<'a> = Pin<&'a T>; - type BorrowedMut<'a> = Pin<&'a mut T>; + type Borrowed<'a> + = Pin<&'a T> + where + Self: 'a; + type BorrowedMut<'a> + = Pin<&'a mut T> + where + Self: 'a; fn into_foreign(self) -> *mut c_void { // SAFETY: We are still treating the box as pinned. diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 4329d3c2c2e5..9cf9f869d195 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -27,10 +27,14 @@ pub unsafe trait ForeignOwnable: Sized { const FOREIGN_ALIGN: usize; /// Type used to immutably borrow a value that is currently foreign-owned. - type Borrowed<'a>; + type Borrowed<'a> + where + Self: 'a; /// Type used to mutably borrow a value that is currently foreign-owned. - type BorrowedMut<'a>; + type BorrowedMut<'a> + where + Self: 'a; /// Converts a Rust-owned object to a foreign-owned one. /// From c8a43666bade4683640dc835f92cd456d29cee55 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 25 May 2026 22:20:50 +0200 Subject: [PATCH 066/131] rust: driver: move 'static bounds to constructor With the ForeignOwnable lifetime change, the 'static bound is no longer necessary on the drvdata methods or bus adapter impls. Move it to the Registration constructor instead. Reviewed-by: Alexandre Courbot Reviewed-by: Greg Kroah-Hartman Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-4-dakr@kernel.org Co-developed-by: Danilo Krummrich Signed-off-by: Danilo Krummrich --- rust/kernel/auxiliary.rs | 6 +++--- rust/kernel/device.rs | 8 ++++---- rust/kernel/driver.rs | 7 +++++-- rust/kernel/i2c.rs | 8 ++++---- rust/kernel/pci.rs | 6 +++--- rust/kernel/platform.rs | 8 ++++---- rust/kernel/usb.rs | 6 +++--- 7 files changed, 26 insertions(+), 23 deletions(-) diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index 19aec94aa95b..35b44d194f67 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -44,7 +44,7 @@ // - `T` is the type of the driver's device private data. // - `struct auxiliary_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. -unsafe impl driver::DriverLayout for Adapter { +unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::auxiliary_driver; type DriverData = T; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); @@ -52,7 +52,7 @@ unsafe impl driver::DriverLayout for Adapter { // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if // a preceding call to `register` has been successful. -unsafe impl driver::RegistrationOps for Adapter { +unsafe impl driver::RegistrationOps for Adapter { unsafe fn register( adrv: &Opaque, name: &'static CStr, @@ -78,7 +78,7 @@ unsafe fn unregister(adrv: &Opaque) { } } -impl Adapter { +impl Adapter { extern "C" fn probe_callback( adev: *mut bindings::auxiliary_device, id: *const bindings::auxiliary_device_id, diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index fd50399aadea..5df8fa108a52 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -203,7 +203,7 @@ pub unsafe fn as_bound(&self) -> &Device { impl Device { /// Store a pointer to the bound driver's private data. - pub fn set_drvdata(&self, data: impl PinInit) -> Result { + pub fn set_drvdata(&self, data: impl PinInit) -> Result { let data = KBox::pin_init(data, GFP_KERNEL)?; // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. @@ -218,7 +218,7 @@ pub fn set_drvdata(&self, data: impl PinInit) -> Result { /// /// - The type `T` must match the type of the `ForeignOwnable` previously stored by /// [`Device::set_drvdata`]. - pub(crate) unsafe fn drvdata_obtain(&self) -> Option>> { + pub(crate) unsafe fn drvdata_obtain(&self) -> Option>> { // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; @@ -244,7 +244,7 @@ pub(crate) unsafe fn drvdata_obtain(&self) -> Option>> { /// device is fully unbound. /// - The type `T` must match the type of the `ForeignOwnable` previously stored by /// [`Device::set_drvdata`]. - pub unsafe fn drvdata_borrow(&self) -> Pin<&T> { + pub unsafe fn drvdata_borrow(&self) -> Pin<&T> { // SAFETY: `drvdata_unchecked()` has the exact same safety requirements as the ones // required by this method. unsafe { self.drvdata_unchecked() } @@ -260,7 +260,7 @@ impl Device { /// the device is fully unbound. /// - The type `T` must match the type of the `ForeignOwnable` previously stored by /// [`Device::set_drvdata`]. - unsafe fn drvdata_unchecked(&self) -> Pin<&T> { + unsafe fn drvdata_unchecked(&self) -> Pin<&T> { // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index 36de8098754d..c8406dc4da60 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -181,7 +181,7 @@ unsafe impl Sync for Registration {} // any thread, so `Registration` is `Send`. unsafe impl Send for Registration {} -impl Registration { +impl Registration { extern "C" fn post_unbind_callback(dev: *mut bindings::device) { // SAFETY: The driver core only ever calls the post unbind callback with a valid pointer to // a `struct device`. @@ -215,7 +215,10 @@ fn callbacks_attach(drv: &Opaque) { } /// Creates a new instance of the registration object. - pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit { + pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit + where + T: 'static, + { try_pin_init!(Self { reg <- Opaque::try_ffi_init(|ptr: *mut T::DriverType| { // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write. diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs index 7b908f0c5a58..4ccee4ba4f23 100644 --- a/rust/kernel/i2c.rs +++ b/rust/kernel/i2c.rs @@ -96,7 +96,7 @@ macro_rules! i2c_device_table { // - `T` is the type of the driver's device private data. // - `struct i2c_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. -unsafe impl driver::DriverLayout for Adapter { +unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::i2c_driver; type DriverData = T; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); @@ -104,7 +104,7 @@ unsafe impl driver::DriverLayout for Adapter { // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if // a preceding call to `register` has been successful. -unsafe impl driver::RegistrationOps for Adapter { +unsafe impl driver::RegistrationOps for Adapter { unsafe fn register( idrv: &Opaque, name: &'static CStr, @@ -151,7 +151,7 @@ unsafe fn unregister(idrv: &Opaque) { } } -impl Adapter { +impl Adapter { extern "C" fn probe_callback(idev: *mut bindings::i2c_client) -> kernel::ffi::c_int { // SAFETY: The I2C bus only ever calls the probe callback with a valid pointer to a // `struct i2c_client`. @@ -222,7 +222,7 @@ fn i2c_id_info(dev: &I2cClient) -> Option<&'static ::Id } } -impl driver::Adapter for Adapter { +impl driver::Adapter for Adapter { type IdInfo = T::IdInfo; fn of_id_table() -> Option> { diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index af74ddff6114..17a33819dc0a 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -62,7 +62,7 @@ // - `T` is the type of the driver's device private data. // - `struct pci_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. -unsafe impl driver::DriverLayout for Adapter { +unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::pci_driver; type DriverData = T; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); @@ -70,7 +70,7 @@ unsafe impl driver::DriverLayout for Adapter { // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if // a preceding call to `register` has been successful. -unsafe impl driver::RegistrationOps for Adapter { +unsafe impl driver::RegistrationOps for Adapter { unsafe fn register( pdrv: &Opaque, name: &'static CStr, @@ -96,7 +96,7 @@ unsafe fn unregister(pdrv: &Opaque) { } } -impl Adapter { +impl Adapter { extern "C" fn probe_callback( pdev: *mut bindings::pci_dev, id: *const bindings::pci_device_id, diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 8917d4ee499f..c7a3dcdde3b1 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -48,7 +48,7 @@ // - `T` is the type of the driver's device private data. // - `struct platform_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. -unsafe impl driver::DriverLayout for Adapter { +unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::platform_driver; type DriverData = T; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); @@ -56,7 +56,7 @@ unsafe impl driver::DriverLayout for Adapter { // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if // a preceding call to `register` has been successful. -unsafe impl driver::RegistrationOps for Adapter { +unsafe impl driver::RegistrationOps for Adapter { unsafe fn register( pdrv: &Opaque, name: &'static CStr, @@ -91,7 +91,7 @@ unsafe fn unregister(pdrv: &Opaque) { } } -impl Adapter { +impl Adapter { extern "C" fn probe_callback(pdev: *mut bindings::platform_device) -> kernel::ffi::c_int { // SAFETY: The platform bus only ever calls the probe callback with a valid pointer to a // `struct platform_device`. @@ -124,7 +124,7 @@ extern "C" fn remove_callback(pdev: *mut bindings::platform_device) { } } -impl driver::Adapter for Adapter { +impl driver::Adapter for Adapter { type IdInfo = T::IdInfo; fn of_id_table() -> Option> { diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs index 9c17a672cd27..3f62da585281 100644 --- a/rust/kernel/usb.rs +++ b/rust/kernel/usb.rs @@ -39,7 +39,7 @@ // - `T` is the type of the driver's device private data. // - `struct usb_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. -unsafe impl driver::DriverLayout for Adapter { +unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::usb_driver; type DriverData = T; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); @@ -47,7 +47,7 @@ unsafe impl driver::DriverLayout for Adapter { // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if // a preceding call to `register` has been successful. -unsafe impl driver::RegistrationOps for Adapter { +unsafe impl driver::RegistrationOps for Adapter { unsafe fn register( udrv: &Opaque, name: &'static CStr, @@ -73,7 +73,7 @@ unsafe fn unregister(udrv: &Opaque) { } } -impl Adapter { +impl Adapter { extern "C" fn probe_callback( intf: *mut bindings::usb_interface, id: *const bindings::usb_device_id, From 7fdffdda630ee61ae0e09ef8f1ace52bbf70e2b0 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:20:51 +0200 Subject: [PATCH 067/131] rust: driver: decouple driver private data from driver type Add a type Data<'bound> associated type to all bus driver traits, decoupling the driver's bus device private data type from the driver struct itself. In the context of adding a 'bound lifetime, making this an associated type has the advantage that it allows us to avoid a driver trait global lifetime and it avoids the need for ForLt for bus device private data; both of which make the subsequent implementation by buses much simpler. All existing drivers and doc examples set type Data = Self to preserve the current behavior. Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Reviewed-by: Greg Kroah-Hartman Link: https://patch.msgid.link/20260525202921.124698-5-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/cpufreq/rcpufreq_dt.rs | 1 + drivers/gpu/drm/nova/driver.rs | 1 + drivers/gpu/drm/tyr/driver.rs | 1 + drivers/gpu/nova-core/driver.rs | 1 + drivers/pwm/pwm_th1520.rs | 1 + rust/kernel/auxiliary.rs | 18 +++++++++------ rust/kernel/cpufreq.rs | 1 + rust/kernel/driver.rs | 24 +++++++++++--------- rust/kernel/i2c.rs | 32 +++++++++++++++------------ rust/kernel/io/mem.rs | 2 ++ rust/kernel/pci.rs | 21 +++++++++++------- rust/kernel/platform.rs | 20 ++++++++++------- rust/kernel/usb.rs | 20 ++++++++++------- samples/rust/rust_debugfs.rs | 1 + samples/rust/rust_dma.rs | 1 + samples/rust/rust_driver_auxiliary.rs | 2 ++ samples/rust/rust_driver_i2c.rs | 1 + samples/rust/rust_driver_pci.rs | 1 + samples/rust/rust_driver_platform.rs | 1 + samples/rust/rust_driver_usb.rs | 1 + samples/rust/rust_i2c_client.rs | 1 + samples/rust/rust_soc.rs | 1 + 22 files changed, 98 insertions(+), 55 deletions(-) diff --git a/drivers/cpufreq/rcpufreq_dt.rs b/drivers/cpufreq/rcpufreq_dt.rs index f17bf64c22e2..b7eeb2730eb0 100644 --- a/drivers/cpufreq/rcpufreq_dt.rs +++ b/drivers/cpufreq/rcpufreq_dt.rs @@ -201,6 +201,7 @@ fn register_em(policy: &mut cpufreq::Policy) { impl platform::Driver for CPUFreqDTDriver { type IdInfo = (); + type Data = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe( diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index b1af0a099551..08136ec0bccb 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -51,6 +51,7 @@ pub(crate) struct NovaData { impl auxiliary::Driver for NovaDriver { type IdInfo = (); + type Data = Self; const ID_TABLE: auxiliary::IdTable = &AUX_TABLE; fn probe(adev: &auxiliary::Device, _info: &Self::IdInfo) -> impl PinInit { diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 279710b36a10..c81bf217743d 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -91,6 +91,7 @@ fn issue_soft_reset(dev: &Device, iomem: &Devres) -> Result { impl platform::Driver for TyrPlatformDriverData { type IdInfo = (); + type Data = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe( diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 8fe484d357f6..699e27046c93 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -74,6 +74,7 @@ pub(crate) struct NovaCore { impl pci::Driver for NovaCore { type IdInfo = (); + type Data = Self; const ID_TABLE: pci::IdTable = &PCI_TABLE; fn probe(pdev: &pci::Device, _info: &Self::IdInfo) -> impl PinInit { diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs index ddd44a5ce497..07795910a0b5 100644 --- a/drivers/pwm/pwm_th1520.rs +++ b/drivers/pwm/pwm_th1520.rs @@ -316,6 +316,7 @@ fn drop(self: Pin<&mut Self>) { impl platform::Driver for Th1520PwmPlatformDriver { type IdInfo = (); + type Data = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe( diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index 35b44d194f67..4e83f9e27d78 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -41,12 +41,12 @@ // SAFETY: // - `bindings::auxiliary_driver` is a C type declared as `repr(C)`. -// - `T` is the type of the driver's device private data. +// - `T::Data` is the type of the driver's device private data. // - `struct auxiliary_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::auxiliary_driver; - type DriverData = T; + type DriverData<'bound> = T::Data; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } @@ -111,8 +111,8 @@ extern "C" fn remove_callback(adev: *mut bindings::auxiliary_device) { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { adev.as_ref().drvdata_borrow::() }; + // and stored a `Pin>`. + let data = unsafe { adev.as_ref().drvdata_borrow::() }; T::unbind(adev, data); } @@ -202,13 +202,17 @@ pub trait Driver { /// type IdInfo: 'static = (); type IdInfo: 'static; + /// The type of the driver's bus device private data. + type Data: Send; + /// The table of device ids supported by the driver. const ID_TABLE: IdTable; /// Auxiliary driver probe. /// /// Called when an auxiliary device is matches a corresponding driver. - fn probe(dev: &Device, id_info: &Self::IdInfo) -> impl PinInit; + fn probe(dev: &Device, id_info: &Self::IdInfo) + -> impl PinInit; /// Auxiliary driver unbind. /// @@ -219,8 +223,8 @@ pub trait Driver { /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O /// operations to gracefully tear down the device. /// - /// Otherwise, release operations for driver resources should be performed in `Self::drop`. - fn unbind(dev: &Device, this: Pin<&Self>) { + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind(dev: &Device, this: Pin<&Self::Data>) { let _ = (dev, this); } } diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index d8d26870bea2..50dd2a2c3e81 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -888,6 +888,7 @@ fn register_em(_policy: &mut Policy) { /// /// impl platform::Driver for SampleDriver { /// type IdInfo = (); +/// type Data = Self; /// const OF_ID_TABLE: Option> = None; /// /// fn probe( diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index c8406dc4da60..5fd1cfd64e93 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -13,10 +13,13 @@ //! The main driver interface is defined by a bus specific driver trait. For instance: //! //! ```ignore -//! pub trait Driver: Send { +//! pub trait Driver { //! /// The type holding information about each device ID supported by the driver. //! type IdInfo: 'static; //! +//! /// The type of the driver's bus device private data. +//! type Data: Send; +//! //! /// The table of OF device ids supported by the driver. //! const OF_ID_TABLE: Option> = None; //! @@ -24,10 +27,11 @@ //! const ACPI_ID_TABLE: Option> = None; //! //! /// Driver probe. -//! fn probe(dev: &Device, id_info: &Self::IdInfo) -> impl PinInit; +//! fn probe(dev: &Device, id_info: &Self::IdInfo) +//! -> impl PinInit; //! //! /// Driver unbind (optional). -//! fn unbind(dev: &Device, this: Pin<&Self>) { +//! fn unbind(dev: &Device, this: Pin<&Self::Data>) { //! let _ = (dev, this); //! } //! } @@ -42,9 +46,9 @@ )] #![cfg_attr(CONFIG_PCI, doc = "* [`pci::Driver`](kernel::pci::Driver)")] //! -//! The `probe()` callback should return a `impl PinInit`, i.e. the driver's private -//! data. The bus abstraction should store the pointer in the corresponding bus device. The generic -//! [`Device`] infrastructure provides common helpers for this purpose on its +//! The `probe()` callback should return a `impl PinInit`, i.e. the driver's +//! private data. The bus abstraction should store the pointer in the corresponding bus device. The +//! generic [`Device`] infrastructure provides common helpers for this purpose on its //! [`Device`] implementation. //! //! All driver callbacks should provide a reference to the driver's private data. Once the driver @@ -118,8 +122,8 @@ pub unsafe trait DriverLayout { /// The specific driver type embedding a `struct device_driver`. type DriverType: Default; - /// The type of the driver's device private data. - type DriverData; + /// The type of the driver's bus device private data. + type DriverData<'bound>; /// Byte offset of the embedded `struct device_driver` within `DriverType`. /// @@ -193,8 +197,8 @@ extern "C" fn post_unbind_callback(dev: *mut bindings::device) { // driver's device private data. // // SAFETY: By the safety requirements of the `Driver` trait, `T::DriverData` is the - // driver's device private data type. - drop(unsafe { dev.drvdata_obtain::() }); + // driver's bus device private data type. + drop(unsafe { dev.drvdata_obtain::>() }); } /// Attach generic `struct device_driver` callbacks. diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs index 4ccee4ba4f23..bfd081518615 100644 --- a/rust/kernel/i2c.rs +++ b/rust/kernel/i2c.rs @@ -93,12 +93,12 @@ macro_rules! i2c_device_table { // SAFETY: // - `bindings::i2c_driver` is a C type declared as `repr(C)`. -// - `T` is the type of the driver's device private data. +// - `T::Data` is the type of the driver's device private data. // - `struct i2c_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::i2c_driver; - type DriverData = T; + type DriverData<'bound> = T::Data; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } @@ -176,8 +176,8 @@ extern "C" fn remove_callback(idev: *mut bindings::i2c_client) { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { idev.as_ref().drvdata_borrow::() }; + // and stored a `Pin>`. + let data = unsafe { idev.as_ref().drvdata_borrow::() }; T::unbind(idev, data); } @@ -188,8 +188,8 @@ extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) { // SAFETY: `shutdown_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { idev.as_ref().drvdata_borrow::() }; + // and stored a `Pin>`. + let data = unsafe { idev.as_ref().drvdata_borrow::() }; T::shutdown(idev, data); } @@ -294,6 +294,7 @@ macro_rules! module_i2c_driver { /// /// impl i2c::Driver for MyDriver { /// type IdInfo = (); +/// type Data = Self; /// const I2C_ID_TABLE: Option> = Some(&I2C_TABLE); /// const OF_ID_TABLE: Option> = Some(&OF_TABLE); /// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); @@ -301,15 +302,15 @@ macro_rules! module_i2c_driver { /// fn probe( /// _idev: &i2c::I2cClient, /// _id_info: Option<&Self::IdInfo>, -/// ) -> impl PinInit { +/// ) -> impl PinInit { /// Err(ENODEV) /// } /// -/// fn shutdown(_idev: &i2c::I2cClient, this: Pin<&Self>) { +/// fn shutdown(_idev: &i2c::I2cClient, this: Pin<&Self::Data>) { /// } /// } ///``` -pub trait Driver: Send { +pub trait Driver { /// The type holding information about each device id supported by the driver. // TODO: Use `associated_type_defaults` once stabilized: // @@ -318,6 +319,9 @@ pub trait Driver: Send { // ``` type IdInfo: 'static; + /// The type of the driver's bus device private data. + type Data: Send; + /// The table of device ids supported by the driver. const I2C_ID_TABLE: Option> = None; @@ -334,7 +338,7 @@ pub trait Driver: Send { fn probe( dev: &I2cClient, id_info: Option<&Self::IdInfo>, - ) -> impl PinInit; + ) -> impl PinInit; /// I2C driver shutdown. /// @@ -346,8 +350,8 @@ fn probe( /// /// This callback is distinct from final resource cleanup, as the driver instance remains valid /// after it returns. Any deallocation or teardown of driver-owned resources should instead be - /// handled in `Self::drop`. - fn shutdown(dev: &I2cClient, this: Pin<&Self>) { + /// handled in `Drop`. + fn shutdown(dev: &I2cClient, this: Pin<&Self::Data>) { let _ = (dev, this); } @@ -360,8 +364,8 @@ fn shutdown(dev: &I2cClient, this: Pin<&Self>) { /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O /// operations to gracefully tear down the device. /// - /// Otherwise, release operations for driver resources should be performed in `Self::drop`. - fn unbind(dev: &I2cClient, this: Pin<&Self>) { + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind(dev: &I2cClient, this: Pin<&Self::Data>) { let _ = (dev, this); } } diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 7dc78d547f7a..e136b676d372 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -62,6 +62,7 @@ pub(crate) unsafe fn new(device: &'a Device, resource: &'a Resource) -> S /// /// impl platform::Driver for SampleDriver { /// # type IdInfo = (); + /// # type Data = Self; /// /// fn probe( /// pdev: &platform::Device, @@ -126,6 +127,7 @@ pub fn iomap_exclusive_sized( /// /// impl platform::Driver for SampleDriver { /// # type IdInfo = (); + /// # type Data = Self; /// /// fn probe( /// pdev: &platform::Device, diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 17a33819dc0a..c743f2abb62f 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -59,12 +59,12 @@ // SAFETY: // - `bindings::pci_driver` is a C type declared as `repr(C)`. -// - `T` is the type of the driver's device private data. +// - `T::Data` is the type of the driver's device private data. // - `struct pci_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::pci_driver; - type DriverData = T; + type DriverData<'bound> = T::Data; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } @@ -129,8 +129,8 @@ extern "C" fn remove_callback(pdev: *mut bindings::pci_dev) { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { pdev.as_ref().drvdata_borrow::() }; + // and stored a `Pin>`. + let data = unsafe { pdev.as_ref().drvdata_borrow::() }; T::unbind(pdev, data); } @@ -279,6 +279,7 @@ macro_rules! pci_device_table { /// /// impl pci::Driver for MyDriver { /// type IdInfo = (); +/// type Data = Self; /// const ID_TABLE: pci::IdTable = &PCI_TABLE; /// /// fn probe( @@ -291,7 +292,7 @@ macro_rules! pci_device_table { ///``` /// Drivers must implement this trait in order to get a PCI driver registered. Please refer to the /// `Adapter` documentation for an example. -pub trait Driver: Send { +pub trait Driver { /// The type holding information about each device id supported by the driver. // TODO: Use `associated_type_defaults` once stabilized: // @@ -300,6 +301,9 @@ pub trait Driver: Send { // ``` type IdInfo: 'static; + /// The type of the driver's bus device private data. + type Data: Send; + /// The table of device ids supported by the driver. const ID_TABLE: IdTable; @@ -307,7 +311,8 @@ pub trait Driver: Send { /// /// Called when a new pci device is added or discovered. Implementers should /// attempt to initialize the device here. - fn probe(dev: &Device, id_info: &Self::IdInfo) -> impl PinInit; + fn probe(dev: &Device, id_info: &Self::IdInfo) + -> impl PinInit; /// PCI driver unbind. /// @@ -318,8 +323,8 @@ pub trait Driver: Send { /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O /// operations to gracefully tear down the device. /// - /// Otherwise, release operations for driver resources should be performed in `Self::drop`. - fn unbind(dev: &Device, this: Pin<&Self>) { + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind(dev: &Device, this: Pin<&Self::Data>) { let _ = (dev, this); } } diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index c7a3dcdde3b1..975b22ffe5db 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -45,12 +45,12 @@ // SAFETY: // - `bindings::platform_driver` is a C type declared as `repr(C)`. -// - `T` is the type of the driver's device private data. +// - `T::Data` is the type of the driver's device private data. // - `struct platform_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::platform_driver; - type DriverData = T; + type DriverData<'bound> = T::Data; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } @@ -117,8 +117,8 @@ extern "C" fn remove_callback(pdev: *mut bindings::platform_device) { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { pdev.as_ref().drvdata_borrow::() }; + // and stored a `Pin>`. + let data = unsafe { pdev.as_ref().drvdata_borrow::() }; T::unbind(pdev, data); } @@ -192,6 +192,7 @@ macro_rules! module_platform_driver { /// /// impl platform::Driver for MyDriver { /// type IdInfo = (); +/// type Data = Self; /// const OF_ID_TABLE: Option> = Some(&OF_TABLE); /// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); /// @@ -203,7 +204,7 @@ macro_rules! module_platform_driver { /// } /// } ///``` -pub trait Driver: Send { +pub trait Driver { /// The type holding driver private data about each device id supported by the driver. // TODO: Use associated_type_defaults once stabilized: // @@ -212,6 +213,9 @@ pub trait Driver: Send { // ``` type IdInfo: 'static; + /// The type of the driver's bus device private data. + type Data: Send; + /// The table of OF device ids supported by the driver. const OF_ID_TABLE: Option> = None; @@ -225,7 +229,7 @@ pub trait Driver: Send { fn probe( dev: &Device, id_info: Option<&Self::IdInfo>, - ) -> impl PinInit; + ) -> impl PinInit; /// Platform driver unbind. /// @@ -236,8 +240,8 @@ fn probe( /// `&Device` or `&Device` reference. For instance, drivers may try to perform I/O /// operations to gracefully tear down the device. /// - /// Otherwise, release operations for driver resources should be performed in `Self::drop`. - fn unbind(dev: &Device, this: Pin<&Self>) { + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind(dev: &Device, this: Pin<&Self::Data>) { let _ = (dev, this); } } diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs index 3f62da585281..88721970afcb 100644 --- a/rust/kernel/usb.rs +++ b/rust/kernel/usb.rs @@ -36,12 +36,12 @@ // SAFETY: // - `bindings::usb_driver` is a C type declared as `repr(C)`. -// - `T` is the type of the driver's device private data. +// - `T::Data` is the type of the driver's device private data. // - `struct usb_driver` embeds a `struct device_driver`. // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::usb_driver; - type DriverData = T; + type DriverData<'bound> = T::Data; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } @@ -109,8 +109,8 @@ extern "C" fn disconnect_callback(intf: *mut bindings::usb_interface) { // SAFETY: `disconnect_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { dev.drvdata_borrow::() }; + // and stored a `Pin>`. + let data = unsafe { dev.drvdata_borrow::() }; T::disconnect(intf, data); } @@ -287,23 +287,27 @@ macro_rules! usb_device_table { /// /// impl usb::Driver for MyDriver { /// type IdInfo = (); +/// type Data = Self; /// const ID_TABLE: usb::IdTable = &USB_TABLE; /// /// fn probe( /// _interface: &usb::Interface, /// _id: &usb::DeviceId, /// _info: &Self::IdInfo, -/// ) -> impl PinInit { +/// ) -> impl PinInit { /// Err(ENODEV) /// } /// -/// fn disconnect(_interface: &usb::Interface, _data: Pin<&Self>) {} +/// fn disconnect(_interface: &usb::Interface, _data: Pin<&Self::Data>) {} /// } ///``` pub trait Driver { /// The type holding information about each one of the device ids supported by the driver. type IdInfo: 'static; + /// The type of the driver's bus device private data. + type Data: Send; + /// The table of device ids supported by the driver. const ID_TABLE: IdTable; @@ -315,12 +319,12 @@ fn probe( interface: &Interface, id: &DeviceId, id_info: &Self::IdInfo, - ) -> impl PinInit; + ) -> impl PinInit; /// USB driver disconnect. /// /// Called when the USB interface is about to be unbound from this driver. - fn disconnect(interface: &Interface, data: Pin<&Self>); + fn disconnect(interface: &Interface, data: Pin<&Self::Data>); } /// A USB interface. diff --git a/samples/rust/rust_debugfs.rs b/samples/rust/rust_debugfs.rs index 0963efe19f93..478c4f693deb 100644 --- a/samples/rust/rust_debugfs.rs +++ b/samples/rust/rust_debugfs.rs @@ -117,6 +117,7 @@ fn from_str(s: &str) -> Result { impl platform::Driver for RustDebugFs { type IdInfo = (); + type Data = Self; const OF_ID_TABLE: Option> = None; const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index 129bb4b39c04..e583c6b8390a 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -58,6 +58,7 @@ unsafe impl kernel::transmute::FromBytes for MyStruct {} impl pci::Driver for DmaSampleDriver { type IdInfo = (); + type Data = Self; const ID_TABLE: pci::IdTable = &PCI_TABLE; fn probe(pdev: &pci::Device, _info: &Self::IdInfo) -> impl PinInit { diff --git a/samples/rust/rust_driver_auxiliary.rs b/samples/rust/rust_driver_auxiliary.rs index 319ef734c02b..61d5bf2e8c0d 100644 --- a/samples/rust/rust_driver_auxiliary.rs +++ b/samples/rust/rust_driver_auxiliary.rs @@ -31,6 +31,7 @@ impl auxiliary::Driver for AuxiliaryDriver { type IdInfo = (); + type Data = Self; const ID_TABLE: auxiliary::IdTable = &AUX_TABLE; @@ -65,6 +66,7 @@ struct ParentDriver { impl pci::Driver for ParentDriver { type IdInfo = (); + type Data = Self; const ID_TABLE: pci::IdTable = &PCI_TABLE; diff --git a/samples/rust/rust_driver_i2c.rs b/samples/rust/rust_driver_i2c.rs index 6be79f9e9fb5..8269f1798611 100644 --- a/samples/rust/rust_driver_i2c.rs +++ b/samples/rust/rust_driver_i2c.rs @@ -35,6 +35,7 @@ impl i2c::Driver for SampleDriver { type IdInfo = u32; + type Data = Self; const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); const I2C_ID_TABLE: Option> = Some(&I2C_TABLE); diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs index 47d3e84fab63..f43c6a660b39 100644 --- a/samples/rust/rust_driver_pci.rs +++ b/samples/rust/rust_driver_pci.rs @@ -140,6 +140,7 @@ fn config_space(pdev: &pci::Device) { impl pci::Driver for SampleDriver { type IdInfo = TestIndex; + type Data = Self; const ID_TABLE: pci::IdTable = &PCI_TABLE; diff --git a/samples/rust/rust_driver_platform.rs b/samples/rust/rust_driver_platform.rs index f2229d176fb9..6505902f8200 100644 --- a/samples/rust/rust_driver_platform.rs +++ b/samples/rust/rust_driver_platform.rs @@ -101,6 +101,7 @@ struct SampleDriver { impl platform::Driver for SampleDriver { type IdInfo = Info; + type Data = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); diff --git a/samples/rust/rust_driver_usb.rs b/samples/rust/rust_driver_usb.rs index ab72e99e1274..5942e4b01fd8 100644 --- a/samples/rust/rust_driver_usb.rs +++ b/samples/rust/rust_driver_usb.rs @@ -26,6 +26,7 @@ struct SampleDriver { impl usb::Driver for SampleDriver { type IdInfo = (); + type Data = Self; const ID_TABLE: usb::IdTable = &USB_TABLE; fn probe( diff --git a/samples/rust/rust_i2c_client.rs b/samples/rust/rust_i2c_client.rs index 8d2c12e535b0..5956b647294d 100644 --- a/samples/rust/rust_i2c_client.rs +++ b/samples/rust/rust_i2c_client.rs @@ -106,6 +106,7 @@ struct SampleDriver { impl platform::Driver for SampleDriver { type IdInfo = (); + type Data = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); diff --git a/samples/rust/rust_soc.rs b/samples/rust/rust_soc.rs index 8079c1c48416..a5e72582f4a2 100644 --- a/samples/rust/rust_soc.rs +++ b/samples/rust/rust_soc.rs @@ -37,6 +37,7 @@ struct SampleSocDriver { impl platform::Driver for SampleSocDriver { type IdInfo = (); + type Data = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); From be31fcf5af751815457102575b816a2bd31b4562 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:20:52 +0200 Subject: [PATCH 068/131] rust: driver core: drop drvdata before devres release Move the post_unbind_rust callback before devres_release_all() in device_unbind_cleanup(). With drvdata() removed, the driver's bus device private data is only accessible by the owning driver itself. It is hence safe to drop the driver's bus device private data before devres actions are released. This reordering is the key enabler for Higher-Ranked Lifetime Types (HRT) in Rust device drivers -- it allows driver structs to hold direct references to devres-managed resources, because the bus device private data (and with it all such references) is guaranteed to be dropped while the underlying devres resources are still alive. Without this change, devres resources would be freed first, leaving the driver's bus device private data with dangling references during its destructor. Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Reviewed-by: Greg Kroah-Hartman Link: https://patch.msgid.link/20260525202921.124698-6-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/base/dd.c | 2 +- include/linux/device/driver.h | 4 ++-- rust/kernel/driver.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/base/dd.c b/drivers/base/dd.c index 1dc1e3528043..73801b40a416 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -595,9 +595,9 @@ static DEVICE_ATTR_RW(state_synced); static void device_unbind_cleanup(struct device *dev) { - devres_release_all(dev); if (dev->driver->p_cb.post_unbind_rust) dev->driver->p_cb.post_unbind_rust(dev); + devres_release_all(dev); arch_teardown_dma_ops(dev); kfree(dev->dma_range_map); dev->dma_range_map = NULL; diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index bbc67ec513ed..38e9a4679447 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -123,8 +123,8 @@ struct device_driver { struct driver_private *p; struct { /* - * Called after remove() and after all devres entries have been - * processed. This is a Rust only callback. + * Called after remove() but before devres entries are released. + * This is a Rust only callback. */ void (*post_unbind_rust)(struct device *dev); } p_cb; diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index 5fd1cfd64e93..a95dafaa9d68 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -193,8 +193,8 @@ extern "C" fn post_unbind_callback(dev: *mut bindings::device) { // INVARIANT: `dev` is valid for the duration of the `post_unbind_callback()`. let dev = unsafe { &*dev.cast::>() }; - // `remove()` and all devres callbacks have been completed at this point, hence drop the - // driver's device private data. + // `remove()` has been completed at this point; devres resources are still valid and will + // be released after the driver's bus device private data is dropped. // // SAFETY: By the safety requirements of the `Driver` trait, `T::DriverData` is the // driver's bus device private data type. From 0b9a29c3a4e2a1f0d0b73ad4c4a4175212bacf21 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:20:53 +0200 Subject: [PATCH 069/131] rust: pci: implement Sync for Device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement Sync for Device in addition to Device. Device uses the same underlying struct pci_dev as Device; Bound is a zero-sized type-state marker that does not affect thread safety. This is needed for drivers to store &'bound pci::Device in their private data while remaining Send. Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Reviewed-by: Greg Kroah-Hartman Acked-by: Uwe Kleine-König Tested-by: Dirk Behme Link: https://patch.msgid.link/20260525202921.124698-7-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/pci.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index c743f2abb62f..d214a861375d 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -528,3 +528,7 @@ unsafe impl Send for Device {} // SAFETY: `Device` can be shared among threads because all methods of `Device` // (i.e. `Device) are thread safe. unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct pci_dev` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} From a89111c00b68ff78cae7981a67601571e365d13b Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:20:54 +0200 Subject: [PATCH 070/131] rust: platform: implement Sync for Device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement Sync for Device in addition to Device. Device uses the same underlying struct platform_device as Device; Bound is a zero-sized type-state marker that does not affect thread safety. This is needed for drivers to store &'bound platform::Device in their private data while remaining Send. Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Reviewed-by: Greg Kroah-Hartman Acked-by: Uwe Kleine-König Tested-by: Dirk Behme Link: https://patch.msgid.link/20260525202921.124698-8-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/platform.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 975b22ffe5db..106a5ed57ea6 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -565,3 +565,7 @@ unsafe impl Send for Device {} // SAFETY: `Device` can be shared among threads because all methods of `Device` // (i.e. `Device) are thread safe. unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct platform_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} From 5bbcefe8db74fe07681a9ded7c13362e3eaae54a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:20:55 +0200 Subject: [PATCH 071/131] rust: auxiliary: implement Sync for Device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement Sync for Device in addition to Device. Device uses the same underlying struct auxiliary_device as Device; Bound is a zero-sized type-state marker that does not affect thread safety. This is needed for drivers to store &'bound auxiliary::Device in their private data while remaining Send. Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Reviewed-by: Greg Kroah-Hartman Acked-by: Uwe Kleine-König Tested-by: Dirk Behme Link: https://patch.msgid.link/20260525202921.124698-9-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/auxiliary.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index 4e83f9e27d78..df2c97423dcc 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -369,6 +369,10 @@ unsafe impl Send for Device {} // (i.e. `Device) are thread safe. unsafe impl Sync for Device {} +// SAFETY: Same as `Device` -- the underlying `struct auxiliary_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} + /// Wrapper that stores a [`TypeId`] alongside the registration data for runtime type checking. #[repr(C)] #[pin_data] From 3bb1655192aeed68b761891eabdc97d9f2f7fc38 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:20:56 +0200 Subject: [PATCH 072/131] rust: usb: implement Sync for Device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement Sync for Device in addition to Device. Device uses the same underlying struct usb_device as Device; Bound is a zero-sized type-state marker that does not affect thread safety. This is needed for drivers to store &'bound usb::Device in their private data while remaining Send. Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Reviewed-by: Greg Kroah-Hartman Acked-by: Uwe Kleine-König Tested-by: Dirk Behme Link: https://patch.msgid.link/20260525202921.124698-10-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/usb.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs index 88721970afcb..6c917d8fa883 100644 --- a/rust/kernel/usb.rs +++ b/rust/kernel/usb.rs @@ -468,6 +468,10 @@ unsafe impl Send for Device {} // allow any mutation through a shared reference. unsafe impl Sync for Device {} +// SAFETY: Same as `Device` -- the underlying `struct usb_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} + /// Declares a kernel module that exposes a single USB driver. /// /// # Examples From de12e48a1be3e9edc0f8bc6e37bad8f7b6f32d54 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:20:57 +0200 Subject: [PATCH 073/131] rust: device: implement Sync for Device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement Sync for Device in addition to Device. Device uses the same underlying struct device as Device; Bound is a zero-sized type-state marker that does not affect thread safety. This is needed for types that hold &'bound Device, such as io::mem::IoMem, to be Send. Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Reviewed-by: Greg Kroah-Hartman Acked-by: Uwe Kleine-König Tested-by: Dirk Behme Link: https://patch.msgid.link/20260525202921.124698-11-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/device.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 5df8fa108a52..c4486f4b8c40 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -467,6 +467,10 @@ unsafe impl Send for Device {} // synchronization in `struct device`. unsafe impl Sync for Device {} +// SAFETY: Same as `Device` -- the underlying `struct device` is the same; `Bound` is a +// zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} + /// Marker trait for the context or scope of a bus specific device. /// /// [`DeviceContext`] is a marker trait for types representing the context of a bus specific From 24799831d631239ff21ea1bf7feee832df48b81f Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:20:58 +0200 Subject: [PATCH 074/131] rust: device: make Core and CoreInternal lifetime-parameterized Device references in probe callbacks are scoped to the callback, not the full binding duration. Add a lifetime parameter to Core and CoreInternal to accurately represent this in the type system. Suggested-by: Gary Guo Reviewed-by: Greg Kroah-Hartman Reviewed-by: Alexandre Courbot Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-12-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/cpufreq/rcpufreq_dt.rs | 2 +- drivers/gpu/drm/nova/driver.rs | 5 ++- drivers/gpu/drm/tyr/driver.rs | 2 +- drivers/gpu/nova-core/driver.rs | 4 +-- drivers/gpu/nova-core/gpu.rs | 2 +- drivers/pwm/pwm_th1520.rs | 2 +- rust/kernel/auxiliary.rs | 12 ++++--- rust/kernel/cpufreq.rs | 2 +- rust/kernel/device.rs | 49 +++++++++++++++++++++------ rust/kernel/devres.rs | 2 +- rust/kernel/dma.rs | 2 +- rust/kernel/driver.rs | 6 ++-- rust/kernel/i2c.rs | 16 ++++----- rust/kernel/io/mem.rs | 4 +-- rust/kernel/pci.rs | 20 ++++++----- rust/kernel/pci/id.rs | 2 +- rust/kernel/platform.rs | 12 +++---- rust/kernel/usb.rs | 16 ++++----- samples/rust/rust_debugfs.rs | 4 +-- samples/rust/rust_dma.rs | 2 +- samples/rust/rust_driver_auxiliary.rs | 7 ++-- samples/rust/rust_driver_i2c.rs | 6 ++-- samples/rust/rust_driver_pci.rs | 4 +-- samples/rust/rust_driver_platform.rs | 2 +- samples/rust/rust_driver_usb.rs | 8 ++--- samples/rust/rust_i2c_client.rs | 4 +-- samples/rust/rust_soc.rs | 2 +- 27 files changed, 118 insertions(+), 81 deletions(-) diff --git a/drivers/cpufreq/rcpufreq_dt.rs b/drivers/cpufreq/rcpufreq_dt.rs index b7eeb2730eb0..5e0b224f6699 100644 --- a/drivers/cpufreq/rcpufreq_dt.rs +++ b/drivers/cpufreq/rcpufreq_dt.rs @@ -205,7 +205,7 @@ impl platform::Driver for CPUFreqDTDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe( - pdev: &platform::Device, + pdev: &platform::Device>, _id_info: Option<&Self::IdInfo>, ) -> impl PinInit { cpufreq::Registration::::new_foreign_owned(pdev.as_ref())?; diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 08136ec0bccb..7605348af994 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -54,7 +54,10 @@ impl auxiliary::Driver for NovaDriver { type Data = Self; const ID_TABLE: auxiliary::IdTable = &AUX_TABLE; - fn probe(adev: &auxiliary::Device, _info: &Self::IdInfo) -> impl PinInit { + fn probe( + adev: &auxiliary::Device>, + _info: &Self::IdInfo, + ) -> impl PinInit { let data = try_pin_init!(NovaData { adev: adev.into() }); let drm = drm::Device::::new(adev.as_ref(), data)?; diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index c81bf217743d..001727f44fc8 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -95,7 +95,7 @@ impl platform::Driver for TyrPlatformDriverData { const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe( - pdev: &platform::Device, + pdev: &platform::Device>, _info: Option<&Self::IdInfo>, ) -> impl PinInit { let core_clk = Clk::get(pdev.as_ref(), Some(c"core"))?; diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 699e27046c93..13c5ff15e87f 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -77,7 +77,7 @@ impl pci::Driver for NovaCore { type Data = Self; const ID_TABLE: pci::IdTable = &PCI_TABLE; - fn probe(pdev: &pci::Device, _info: &Self::IdInfo) -> impl PinInit { + fn probe(pdev: &pci::Device>, _info: &Self::IdInfo) -> impl PinInit { pin_init::pin_init_scope(move || { dev_dbg!(pdev, "Probe Nova Core GPU driver.\n"); @@ -109,7 +109,7 @@ fn probe(pdev: &pci::Device, _info: &Self::IdInfo) -> impl PinInit, this: Pin<&Self>) { + fn unbind(pdev: &pci::Device>, this: Pin<&Self>) { this.gpu.unbind(pdev.as_ref()); } } diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 0f6fe9a1b955..4ffb506342a9 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -278,7 +278,7 @@ pub(crate) fn new<'a>( /// Called when the corresponding [`Device`](device::Device) is unbound. /// /// Note: This method must only be called from `Driver::unbind`. - pub(crate) fn unbind(&self, dev: &device::Device) { + pub(crate) fn unbind(&self, dev: &device::Device>) { kernel::warn_on!(self .bar .access(dev) diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs index 07795910a0b5..df83a4a9a507 100644 --- a/drivers/pwm/pwm_th1520.rs +++ b/drivers/pwm/pwm_th1520.rs @@ -320,7 +320,7 @@ impl platform::Driver for Th1520PwmPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe( - pdev: &platform::Device, + pdev: &platform::Device>, _id_info: Option<&Self::IdInfo>, ) -> impl PinInit { let dev = pdev.as_ref(); diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index df2c97423dcc..6d504b0933d5 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -87,7 +87,7 @@ extern "C" fn probe_callback( // `struct auxiliary_device`. // // INVARIANT: `adev` is valid for the duration of `probe_callback()`. - let adev = unsafe { &*adev.cast::>() }; + let adev = unsafe { &*adev.cast::>>() }; // SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id` // and does not add additional invariants, so it's safe to transmute. @@ -107,7 +107,7 @@ extern "C" fn remove_callback(adev: *mut bindings::auxiliary_device) { // `struct auxiliary_device`. // // INVARIANT: `adev` is valid for the duration of `remove_callback()`. - let adev = unsafe { &*adev.cast::>() }; + let adev = unsafe { &*adev.cast::>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called @@ -211,8 +211,10 @@ pub trait Driver { /// Auxiliary driver probe. /// /// Called when an auxiliary device is matches a corresponding driver. - fn probe(dev: &Device, id_info: &Self::IdInfo) - -> impl PinInit; + fn probe( + dev: &Device>, + id_info: &Self::IdInfo, + ) -> impl PinInit; /// Auxiliary driver unbind. /// @@ -224,7 +226,7 @@ fn probe(dev: &Device, id_info: &Self::IdInfo) /// operations to gracefully tear down the device. /// /// Otherwise, release operations for driver resources should be performed in `Drop`. - fn unbind(dev: &Device, this: Pin<&Self::Data>) { + fn unbind(dev: &Device>, this: Pin<&Self::Data>) { let _ = (dev, this); } } diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 50dd2a2c3e81..0df518fa1d77 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -892,7 +892,7 @@ fn register_em(_policy: &mut Policy) { /// const OF_ID_TABLE: Option> = None; /// /// fn probe( -/// pdev: &platform::Device, +/// pdev: &platform::Device>, /// _id_info: Option<&Self::IdInfo>, /// ) -> impl PinInit { /// cpufreq::Registration::::new_foreign_owned(pdev.as_ref())?; diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index c4486f4b8c40..645afc49a27d 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -201,7 +201,7 @@ pub unsafe fn as_bound(&self) -> &Device { } } -impl Device { +impl<'a> Device> { /// Store a pointer to the bound driver's private data. pub fn set_drvdata(&self, data: impl PinInit) -> Result { let data = KBox::pin_init(data, GFP_KERNEL)?; @@ -511,7 +511,7 @@ pub trait DeviceContext: private::Sealed {} /// callback it appears in. It is intended to be used for synchronization purposes. Bus device /// implementations can implement methods for [`Device`], such that they can only be called /// from bus callbacks. -pub struct Core; +pub struct Core<'a>(PhantomData<&'a ()>); /// Semantically the same as [`Core`], but reserved for internal usage of the corresponding bus /// abstraction. @@ -522,7 +522,7 @@ pub trait DeviceContext: private::Sealed {} /// /// This context mainly exists to share generic [`Device`] infrastructure that should only be called /// from bus callbacks with bus abstractions, but without making them accessible for drivers. -pub struct CoreInternal; +pub struct CoreInternal<'a>(PhantomData<&'a ()>); /// The [`Bound`] context is the [`DeviceContext`] of a bus specific device when it is guaranteed to /// be bound to a driver. @@ -546,14 +546,14 @@ mod private { pub trait Sealed {} impl Sealed for super::Bound {} - impl Sealed for super::Core {} - impl Sealed for super::CoreInternal {} + impl<'a> Sealed for super::Core<'a> {} + impl<'a> Sealed for super::CoreInternal<'a> {} impl Sealed for super::Normal {} } impl DeviceContext for Bound {} -impl DeviceContext for Core {} -impl DeviceContext for CoreInternal {} +impl<'a> DeviceContext for Core<'a> {} +impl<'a> DeviceContext for CoreInternal<'a> {} impl DeviceContext for Normal {} impl AsRef> for Device { @@ -603,6 +603,22 @@ unsafe fn from_device(dev: &Device) -> &Self #[doc(hidden)] #[macro_export] macro_rules! __impl_device_context_deref { + (unsafe { $device:ident, <$lt:lifetime> $src:ty => $dst:ty }) => { + impl<$lt> ::core::ops::Deref for $device<$src> { + type Target = $device<$dst>; + + fn deref(&self) -> &Self::Target { + let ptr: *const Self = self; + + // CAST: `$device<$src>` and `$device<$dst>` transparently wrap the same type by the + // safety requirement of the macro. + let ptr = ptr.cast::(); + + // SAFETY: `ptr` was derived from `&self`. + unsafe { &*ptr } + } + } + }; (unsafe { $device:ident, $src:ty => $dst:ty }) => { impl ::core::ops::Deref for $device<$src> { type Target = $device<$dst>; @@ -635,14 +651,14 @@ macro_rules! impl_device_context_deref { // `__impl_device_context_deref!`. ::kernel::__impl_device_context_deref!(unsafe { $device, - $crate::device::CoreInternal => $crate::device::Core + <'a> $crate::device::CoreInternal<'a> => $crate::device::Core<'a> }); // SAFETY: This macro has the exact same safety requirement as // `__impl_device_context_deref!`. ::kernel::__impl_device_context_deref!(unsafe { $device, - $crate::device::Core => $crate::device::Bound + <'a> $crate::device::Core<'a> => $crate::device::Bound }); // SAFETY: This macro has the exact same safety requirement as @@ -657,6 +673,13 @@ macro_rules! impl_device_context_deref { #[doc(hidden)] #[macro_export] macro_rules! __impl_device_context_into_aref { + (<$lt:lifetime> $src:ty, $device:tt) => { + impl<$lt> ::core::convert::From<&$device<$src>> for $crate::sync::aref::ARef<$device> { + fn from(dev: &$device<$src>) -> Self { + (&**dev).into() + } + } + }; ($src:ty, $device:tt) => { impl ::core::convert::From<&$device<$src>> for $crate::sync::aref::ARef<$device> { fn from(dev: &$device<$src>) -> Self { @@ -671,8 +694,12 @@ fn from(dev: &$device<$src>) -> Self { #[macro_export] macro_rules! impl_device_context_into_aref { ($device:tt) => { - ::kernel::__impl_device_context_into_aref!($crate::device::CoreInternal, $device); - ::kernel::__impl_device_context_into_aref!($crate::device::Core, $device); + ::kernel::__impl_device_context_into_aref!( + <'a> $crate::device::CoreInternal<'a>, $device + ); + ::kernel::__impl_device_context_into_aref!( + <'a> $crate::device::Core<'a>, $device + ); ::kernel::__impl_device_context_into_aref!($crate::device::Bound, $device); }; } diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 9e5f93aed20c..fd4633f977f6 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -304,7 +304,7 @@ pub fn device(&self) -> &Device { /// pci, // /// }; /// - /// fn from_core(dev: &pci::Device, devres: Devres>) -> Result { + /// fn from_core(dev: &pci::Device>, devres: Devres>) -> Result { /// let bar = devres.access(dev.as_ref())?; /// /// let _ = bar.read32(0x0); diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 4995ee5dc689..8f97916e0688 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -47,7 +47,7 @@ /// where the underlying bus is DMA capable, such as: #[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] /// * [`platform::Device`](::kernel::platform::Device) -pub trait Device: AsRef> { +pub trait Device<'a>: AsRef>> { /// Set up the device's DMA streaming addressing capabilities. /// /// This method is usually called once from `probe()` as soon as the device capabilities are diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index a95dafaa9d68..558fdef4a1c6 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -27,11 +27,11 @@ //! const ACPI_ID_TABLE: Option> = None; //! //! /// Driver probe. -//! fn probe(dev: &Device, id_info: &Self::IdInfo) +//! fn probe(dev: &Device>, id_info: &Self::IdInfo) //! -> impl PinInit; //! //! /// Driver unbind (optional). -//! fn unbind(dev: &Device, this: Pin<&Self::Data>) { +//! fn unbind(dev: &Device>, this: Pin<&Self::Data>) { //! let _ = (dev, this); //! } //! } @@ -191,7 +191,7 @@ extern "C" fn post_unbind_callback(dev: *mut bindings::device) { // a `struct device`. // // INVARIANT: `dev` is valid for the duration of the `post_unbind_callback()`. - let dev = unsafe { &*dev.cast::>() }; + let dev = unsafe { &*dev.cast::>>() }; // `remove()` has been completed at this point; devres resources are still valid and will // be released after the driver's bus device private data is dropped. diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs index bfd081518615..50feade0fb58 100644 --- a/rust/kernel/i2c.rs +++ b/rust/kernel/i2c.rs @@ -157,7 +157,7 @@ extern "C" fn probe_callback(idev: *mut bindings::i2c_client) -> kernel::ffi::c_ // `struct i2c_client`. // // INVARIANT: `idev` is valid for the duration of `probe_callback()`. - let idev = unsafe { &*idev.cast::>() }; + let idev = unsafe { &*idev.cast::>>() }; let info = Self::i2c_id_info(idev).or_else(|| ::id_info(idev.as_ref())); @@ -172,7 +172,7 @@ extern "C" fn probe_callback(idev: *mut bindings::i2c_client) -> kernel::ffi::c_ extern "C" fn remove_callback(idev: *mut bindings::i2c_client) { // SAFETY: `idev` is a valid pointer to a `struct i2c_client`. - let idev = unsafe { &*idev.cast::>() }; + let idev = unsafe { &*idev.cast::>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called @@ -184,7 +184,7 @@ extern "C" fn remove_callback(idev: *mut bindings::i2c_client) { extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) { // SAFETY: `shutdown_callback` is only ever called for a valid `idev` - let idev = unsafe { &*idev.cast::>() }; + let idev = unsafe { &*idev.cast::>>() }; // SAFETY: `shutdown_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called @@ -300,13 +300,13 @@ macro_rules! module_i2c_driver { /// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); /// /// fn probe( -/// _idev: &i2c::I2cClient, +/// _idev: &i2c::I2cClient>, /// _id_info: Option<&Self::IdInfo>, /// ) -> impl PinInit { /// Err(ENODEV) /// } /// -/// fn shutdown(_idev: &i2c::I2cClient, this: Pin<&Self::Data>) { +/// fn shutdown(_idev: &i2c::I2cClient>, this: Pin<&Self::Data>) { /// } /// } ///``` @@ -336,7 +336,7 @@ pub trait Driver { /// Called when a new i2c client is added or discovered. /// Implementers should attempt to initialize the client here. fn probe( - dev: &I2cClient, + dev: &I2cClient>, id_info: Option<&Self::IdInfo>, ) -> impl PinInit; @@ -351,7 +351,7 @@ fn probe( /// This callback is distinct from final resource cleanup, as the driver instance remains valid /// after it returns. Any deallocation or teardown of driver-owned resources should instead be /// handled in `Drop`. - fn shutdown(dev: &I2cClient, this: Pin<&Self::Data>) { + fn shutdown(dev: &I2cClient>, this: Pin<&Self::Data>) { let _ = (dev, this); } @@ -365,7 +365,7 @@ fn shutdown(dev: &I2cClient, this: Pin<&Self::Data>) { /// operations to gracefully tear down the device. /// /// Otherwise, release operations for driver resources should be performed in `Drop`. - fn unbind(dev: &I2cClient, this: Pin<&Self::Data>) { + fn unbind(dev: &I2cClient>, this: Pin<&Self::Data>) { let _ = (dev, this); } } diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index e136b676d372..03d8745b5e1d 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -65,7 +65,7 @@ pub(crate) unsafe fn new(device: &'a Device, resource: &'a Resource) -> S /// # type Data = Self; /// /// fn probe( - /// pdev: &platform::Device, + /// pdev: &platform::Device>, /// info: Option<&Self::IdInfo>, /// ) -> impl PinInit { /// let offset = 0; // Some offset. @@ -130,7 +130,7 @@ pub fn iomap_exclusive_sized( /// # type Data = Self; /// /// fn probe( - /// pdev: &platform::Device, + /// pdev: &platform::Device>, /// info: Option<&Self::IdInfo>, /// ) -> impl PinInit { /// let offset = 0; // Some offset. diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index d214a861375d..314ad9fefdb0 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -105,7 +105,7 @@ extern "C" fn probe_callback( // `struct pci_dev`. // // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. - let pdev = unsafe { &*pdev.cast::>() }; + let pdev = unsafe { &*pdev.cast::>>() }; // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct pci_device_id` and // does not add additional invariants, so it's safe to transmute. @@ -125,7 +125,7 @@ extern "C" fn remove_callback(pdev: *mut bindings::pci_dev) { // `struct pci_dev`. // // INVARIANT: `pdev` is valid for the duration of `remove_callback()`. - let pdev = unsafe { &*pdev.cast::>() }; + let pdev = unsafe { &*pdev.cast::>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called @@ -283,7 +283,7 @@ macro_rules! pci_device_table { /// const ID_TABLE: pci::IdTable = &PCI_TABLE; /// /// fn probe( -/// _pdev: &pci::Device, +/// _pdev: &pci::Device>, /// _id_info: &Self::IdInfo, /// ) -> impl PinInit { /// Err(ENODEV) @@ -311,8 +311,10 @@ pub trait Driver { /// /// Called when a new pci device is added or discovered. Implementers should /// attempt to initialize the device here. - fn probe(dev: &Device, id_info: &Self::IdInfo) - -> impl PinInit; + fn probe( + dev: &Device>, + id_info: &Self::IdInfo, + ) -> impl PinInit; /// PCI driver unbind. /// @@ -324,7 +326,7 @@ fn probe(dev: &Device, id_info: &Self::IdInfo) /// operations to gracefully tear down the device. /// /// Otherwise, release operations for driver resources should be performed in `Drop`. - fn unbind(dev: &Device, this: Pin<&Self::Data>) { + fn unbind(dev: &Device>, this: Pin<&Self::Data>) { let _ = (dev, this); } } @@ -359,7 +361,7 @@ impl Device { /// /// ``` /// # use kernel::{device::Core, pci::{self, Vendor}, prelude::*}; - /// fn log_device_info(pdev: &pci::Device) -> Result { + /// fn log_device_info(pdev: &pci::Device>) -> Result { /// // Get an instance of `Vendor`. /// let vendor = pdev.vendor_id(); /// dev_info!( @@ -450,7 +452,7 @@ pub fn pci_class(&self) -> Class { } } -impl Device { +impl<'a> Device> { /// Enable memory resources for this device. pub fn enable_device_mem(&self) -> Result { // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`. @@ -476,7 +478,7 @@ unsafe impl device::AsBusDevice for Device kernel::impl_device_context_deref!(unsafe { Device }); kernel::impl_device_context_into_aref!(Device); -impl crate::dma::Device for Device {} +impl<'a> crate::dma::Device<'a> for Device> {} // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::sync::aref::AlwaysRefCounted for Device { diff --git a/rust/kernel/pci/id.rs b/rust/kernel/pci/id.rs index 50005d176561..dbaf301666e7 100644 --- a/rust/kernel/pci/id.rs +++ b/rust/kernel/pci/id.rs @@ -19,7 +19,7 @@ /// /// ``` /// # use kernel::{device::Core, pci::{self, Class}, prelude::*}; -/// fn probe_device(pdev: &pci::Device) -> Result { +/// fn probe_device(pdev: &pci::Device>) -> Result { /// let pci_class = pdev.pci_class(); /// dev_info!( /// pdev, diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 106a5ed57ea6..257b7084338c 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -97,7 +97,7 @@ extern "C" fn probe_callback(pdev: *mut bindings::platform_device) -> kernel::ff // `struct platform_device`. // // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. - let pdev = unsafe { &*pdev.cast::>() }; + let pdev = unsafe { &*pdev.cast::>>() }; let info = ::id_info(pdev.as_ref()); from_result(|| { @@ -113,7 +113,7 @@ extern "C" fn remove_callback(pdev: *mut bindings::platform_device) { // `struct platform_device`. // // INVARIANT: `pdev` is valid for the duration of `remove_callback()`. - let pdev = unsafe { &*pdev.cast::>() }; + let pdev = unsafe { &*pdev.cast::>>() }; // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called @@ -197,7 +197,7 @@ macro_rules! module_platform_driver { /// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); /// /// fn probe( -/// _pdev: &platform::Device, +/// _pdev: &platform::Device>, /// _id_info: Option<&Self::IdInfo>, /// ) -> impl PinInit { /// Err(ENODEV) @@ -227,7 +227,7 @@ pub trait Driver { /// Called when a new platform device is added or discovered. /// Implementers should attempt to initialize the device here. fn probe( - dev: &Device, + dev: &Device>, id_info: Option<&Self::IdInfo>, ) -> impl PinInit; @@ -241,7 +241,7 @@ fn probe( /// operations to gracefully tear down the device. /// /// Otherwise, release operations for driver resources should be performed in `Drop`. - fn unbind(dev: &Device, this: Pin<&Self::Data>) { + fn unbind(dev: &Device>, this: Pin<&Self::Data>) { let _ = (dev, this); } } @@ -513,7 +513,7 @@ pub fn optional_irq_by_name(&self, name: &CStr) -> Result> { kernel::impl_device_context_deref!(unsafe { Device }); kernel::impl_device_context_into_aref!(Device); -impl crate::dma::Device for Device {} +impl<'a> crate::dma::Device<'a> for Device> {} // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::sync::aref::AlwaysRefCounted for Device { diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs index 6c917d8fa883..1dbb8387b463 100644 --- a/rust/kernel/usb.rs +++ b/rust/kernel/usb.rs @@ -82,7 +82,7 @@ extern "C" fn probe_callback( // `struct usb_interface` and `struct usb_device_id`. // // INVARIANT: `intf` is valid for the duration of `probe_callback()`. - let intf = unsafe { &*intf.cast::>() }; + let intf = unsafe { &*intf.cast::>>() }; from_result(|| { // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct usb_device_id` and @@ -92,7 +92,7 @@ extern "C" fn probe_callback( let info = T::ID_TABLE.info(id.index()); let data = T::probe(intf, id, info); - let dev: &device::Device = intf.as_ref(); + let dev: &device::Device> = intf.as_ref(); dev.set_drvdata(data)?; Ok(0) }) @@ -103,9 +103,9 @@ extern "C" fn disconnect_callback(intf: *mut bindings::usb_interface) { // `struct usb_interface`. // // INVARIANT: `intf` is valid for the duration of `disconnect_callback()`. - let intf = unsafe { &*intf.cast::>() }; + let intf = unsafe { &*intf.cast::>>() }; - let dev: &device::Device = intf.as_ref(); + let dev: &device::Device> = intf.as_ref(); // SAFETY: `disconnect_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called @@ -291,14 +291,14 @@ macro_rules! usb_device_table { /// const ID_TABLE: usb::IdTable = &USB_TABLE; /// /// fn probe( -/// _interface: &usb::Interface, +/// _interface: &usb::Interface>, /// _id: &usb::DeviceId, /// _info: &Self::IdInfo, /// ) -> impl PinInit { /// Err(ENODEV) /// } /// -/// fn disconnect(_interface: &usb::Interface, _data: Pin<&Self::Data>) {} +/// fn disconnect(_interface: &usb::Interface>, _data: Pin<&Self::Data>) {} /// } ///``` pub trait Driver { @@ -316,7 +316,7 @@ pub trait Driver { /// Called when a new USB interface is bound to this driver. /// Implementers should attempt to initialize the interface here. fn probe( - interface: &Interface, + interface: &Interface>, id: &DeviceId, id_info: &Self::IdInfo, ) -> impl PinInit; @@ -324,7 +324,7 @@ fn probe( /// USB driver disconnect. /// /// Called when the USB interface is about to be unbound from this driver. - fn disconnect(interface: &Interface, data: Pin<&Self::Data>); + fn disconnect(interface: &Interface>, data: Pin<&Self::Data>); } /// A USB interface. diff --git a/samples/rust/rust_debugfs.rs b/samples/rust/rust_debugfs.rs index 478c4f693deb..37640ed33642 100644 --- a/samples/rust/rust_debugfs.rs +++ b/samples/rust/rust_debugfs.rs @@ -122,7 +122,7 @@ impl platform::Driver for RustDebugFs { const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); fn probe( - pdev: &platform::Device, + pdev: &platform::Device>, _info: Option<&Self::IdInfo>, ) -> impl PinInit { RustDebugFs::new(pdev).pin_chain(|this| { @@ -147,7 +147,7 @@ fn build_inner(dir: &Dir) -> impl PinInit>> + '_ { dir.read_write_file(c"pair", new_mutex!(Inner { x: 3, y: 10 })) } - fn new(pdev: &platform::Device) -> impl PinInit + '_ { + fn new<'a>(pdev: &'a platform::Device>) -> impl PinInit + 'a { let debugfs = Dir::new(c"sample_debugfs"); let dev = pdev.as_ref(); diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index e583c6b8390a..9a243e7c7298 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -61,7 +61,7 @@ impl pci::Driver for DmaSampleDriver { type Data = Self; const ID_TABLE: pci::IdTable = &PCI_TABLE; - fn probe(pdev: &pci::Device, _info: &Self::IdInfo) -> impl PinInit { + fn probe(pdev: &pci::Device>, _info: &Self::IdInfo) -> impl PinInit { pin_init::pin_init_scope(move || { dev_info!(pdev, "Probe DMA test driver.\n"); diff --git a/samples/rust/rust_driver_auxiliary.rs b/samples/rust/rust_driver_auxiliary.rs index 61d5bf2e8c0d..f0d419823f9a 100644 --- a/samples/rust/rust_driver_auxiliary.rs +++ b/samples/rust/rust_driver_auxiliary.rs @@ -35,7 +35,10 @@ impl auxiliary::Driver for AuxiliaryDriver { const ID_TABLE: auxiliary::IdTable = &AUX_TABLE; - fn probe(adev: &auxiliary::Device, _info: &Self::IdInfo) -> impl PinInit { + fn probe( + adev: &auxiliary::Device>, + _info: &Self::IdInfo, + ) -> impl PinInit { dev_info!( adev, "Probing auxiliary driver for auxiliary device with id={}\n", @@ -70,7 +73,7 @@ impl pci::Driver for ParentDriver { const ID_TABLE: pci::IdTable = &PCI_TABLE; - fn probe(pdev: &pci::Device, _info: &Self::IdInfo) -> impl PinInit { + fn probe(pdev: &pci::Device>, _info: &Self::IdInfo) -> impl PinInit { Ok(Self { _reg0: auxiliary::Registration::new( pdev.as_ref(), diff --git a/samples/rust/rust_driver_i2c.rs b/samples/rust/rust_driver_i2c.rs index 8269f1798611..171550ea0b6f 100644 --- a/samples/rust/rust_driver_i2c.rs +++ b/samples/rust/rust_driver_i2c.rs @@ -42,7 +42,7 @@ impl i2c::Driver for SampleDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe( - idev: &i2c::I2cClient, + idev: &i2c::I2cClient>, info: Option<&Self::IdInfo>, ) -> impl PinInit { let dev = idev.as_ref(); @@ -56,11 +56,11 @@ fn probe( Ok(Self) } - fn shutdown(idev: &i2c::I2cClient, _this: Pin<&Self>) { + fn shutdown(idev: &i2c::I2cClient>, _this: Pin<&Self>) { dev_info!(idev.as_ref(), "Shutdown Rust I2C driver sample.\n"); } - fn unbind(idev: &i2c::I2cClient, _this: Pin<&Self>) { + fn unbind(idev: &i2c::I2cClient>, _this: Pin<&Self>) { dev_info!(idev.as_ref(), "Unbind Rust I2C driver sample.\n"); } } diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs index f43c6a660b39..3106f766fd93 100644 --- a/samples/rust/rust_driver_pci.rs +++ b/samples/rust/rust_driver_pci.rs @@ -144,7 +144,7 @@ impl pci::Driver for SampleDriver { const ID_TABLE: pci::IdTable = &PCI_TABLE; - fn probe(pdev: &pci::Device, info: &Self::IdInfo) -> impl PinInit { + fn probe(pdev: &pci::Device>, info: &Self::IdInfo) -> impl PinInit { pin_init::pin_init_scope(move || { let vendor = pdev.vendor_id(); dev_dbg!( @@ -175,7 +175,7 @@ fn probe(pdev: &pci::Device, info: &Self::IdInfo) -> impl PinInit, this: Pin<&Self>) { + fn unbind(pdev: &pci::Device>, this: Pin<&Self>) { if let Ok(bar) = this.bar.access(pdev.as_ref()) { // Reset pci-testdev by writing a new test index. bar.write_reg(regs::TEST::zeroed().with_index(this.index)); diff --git a/samples/rust/rust_driver_platform.rs b/samples/rust/rust_driver_platform.rs index 6505902f8200..04d40f836275 100644 --- a/samples/rust/rust_driver_platform.rs +++ b/samples/rust/rust_driver_platform.rs @@ -106,7 +106,7 @@ impl platform::Driver for SampleDriver { const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); fn probe( - pdev: &platform::Device, + pdev: &platform::Device>, info: Option<&Self::IdInfo>, ) -> impl PinInit { let dev = pdev.as_ref(); diff --git a/samples/rust/rust_driver_usb.rs b/samples/rust/rust_driver_usb.rs index 5942e4b01fd8..e900993335e9 100644 --- a/samples/rust/rust_driver_usb.rs +++ b/samples/rust/rust_driver_usb.rs @@ -30,18 +30,18 @@ impl usb::Driver for SampleDriver { const ID_TABLE: usb::IdTable = &USB_TABLE; fn probe( - intf: &usb::Interface, + intf: &usb::Interface>, _id: &usb::DeviceId, _info: &Self::IdInfo, ) -> impl PinInit { - let dev: &device::Device = intf.as_ref(); + let dev: &device::Device> = intf.as_ref(); dev_info!(dev, "Rust USB driver sample probed\n"); Ok(Self { _intf: intf.into() }) } - fn disconnect(intf: &usb::Interface, _data: Pin<&Self>) { - let dev: &device::Device = intf.as_ref(); + fn disconnect(intf: &usb::Interface>, _data: Pin<&Self>) { + let dev: &device::Device> = intf.as_ref(); dev_info!(dev, "Rust USB driver sample disconnected\n"); } } diff --git a/samples/rust/rust_i2c_client.rs b/samples/rust/rust_i2c_client.rs index 5956b647294d..3f273c754f86 100644 --- a/samples/rust/rust_i2c_client.rs +++ b/samples/rust/rust_i2c_client.rs @@ -111,7 +111,7 @@ impl platform::Driver for SampleDriver { const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); fn probe( - pdev: &platform::Device, + pdev: &platform::Device>, _info: Option<&Self::IdInfo>, ) -> impl PinInit { dev_info!( @@ -130,7 +130,7 @@ fn probe( }) } - fn unbind(pdev: &platform::Device, _this: Pin<&Self>) { + fn unbind(pdev: &platform::Device>, _this: Pin<&Self>) { dev_info!( pdev.as_ref(), "Unbind Rust I2C Client registration sample.\n" diff --git a/samples/rust/rust_soc.rs b/samples/rust/rust_soc.rs index a5e72582f4a2..c466653491d2 100644 --- a/samples/rust/rust_soc.rs +++ b/samples/rust/rust_soc.rs @@ -42,7 +42,7 @@ impl platform::Driver for SampleSocDriver { const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); fn probe( - pdev: &platform::Device, + pdev: &platform::Device>, _info: Option<&Self::IdInfo>, ) -> impl PinInit { dev_dbg!(pdev, "Probe Rust SoC driver sample.\n"); From 16c2b8fdab7c0808ff36430b2f49569029a8f484 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:20:59 +0200 Subject: [PATCH 075/131] rust: pci: make Driver trait lifetime-parameterized Add a 'bound lifetime to the associated Data, changing type Data to type Data<'bound>. This allows the driver's bus device private data to capture the device / driver bound lifetime; device resources can be stored directly by reference rather than requiring Devres. The probe() and unbind() callbacks thus gain a 'bound lifetime parameter on the methods themselves; avoiding a global lifetime on the trait impl. Existing drivers set type Data<'bound> = Self, preserving the current behavior. Reviewed-by: Alexandre Courbot Reviewed-by: Greg Kroah-Hartman Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-13-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 9 ++++++--- rust/kernel/pci.rs | 28 +++++++++++++-------------- samples/rust/rust_dma.rs | 7 +++++-- samples/rust/rust_driver_auxiliary.rs | 7 +++++-- samples/rust/rust_driver_pci.rs | 7 +++++-- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 13c5ff15e87f..6ad1a856694c 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -74,10 +74,13 @@ pub(crate) struct NovaCore { impl pci::Driver for NovaCore { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const ID_TABLE: pci::IdTable = &PCI_TABLE; - fn probe(pdev: &pci::Device>, _info: &Self::IdInfo) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound pci::Device>, + _info: &'bound Self::IdInfo, + ) -> impl PinInit + 'bound { pin_init::pin_init_scope(move || { dev_dbg!(pdev, "Probe Nova Core GPU driver.\n"); @@ -109,7 +112,7 @@ fn probe(pdev: &pci::Device>, _info: &Self::IdInfo) -> impl PinInit>, this: Pin<&Self>) { + fn unbind<'bound>(pdev: &'bound pci::Device>, this: Pin<&Self>) { this.gpu.unbind(pdev.as_ref()); } } diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 314ad9fefdb0..5071cae6543f 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -64,7 +64,7 @@ // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::pci_driver; - type DriverData<'bound> = T::Data; + type DriverData<'bound> = T::Data<'bound>; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } @@ -129,8 +129,8 @@ extern "C" fn remove_callback(pdev: *mut bindings::pci_dev) { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { pdev.as_ref().drvdata_borrow::() }; + // and stored a `Pin>>`. + let data = unsafe { pdev.as_ref().drvdata_borrow::>() }; T::unbind(pdev, data); } @@ -279,13 +279,13 @@ macro_rules! pci_device_table { /// /// impl pci::Driver for MyDriver { /// type IdInfo = (); -/// type Data = Self; +/// type Data<'bound> = Self; /// const ID_TABLE: pci::IdTable = &PCI_TABLE; /// -/// fn probe( -/// _pdev: &pci::Device>, -/// _id_info: &Self::IdInfo, -/// ) -> impl PinInit { +/// fn probe<'bound>( +/// _pdev: &'bound pci::Device>, +/// _id_info: &'bound Self::IdInfo, +/// ) -> impl PinInit, Error> + 'bound { /// Err(ENODEV) /// } /// } @@ -302,7 +302,7 @@ pub trait Driver { type IdInfo: 'static; /// The type of the driver's bus device private data. - type Data: Send; + type Data<'bound>: Send + 'bound; /// The table of device ids supported by the driver. const ID_TABLE: IdTable; @@ -311,10 +311,10 @@ pub trait Driver { /// /// Called when a new pci device is added or discovered. Implementers should /// attempt to initialize the device here. - fn probe( - dev: &Device>, - id_info: &Self::IdInfo, - ) -> impl PinInit; + fn probe<'bound>( + dev: &'bound Device>, + id_info: &'bound Self::IdInfo, + ) -> impl PinInit, Error> + 'bound; /// PCI driver unbind. /// @@ -326,7 +326,7 @@ fn probe( /// operations to gracefully tear down the device. /// /// Otherwise, release operations for driver resources should be performed in `Drop`. - fn unbind(dev: &Device>, this: Pin<&Self::Data>) { + fn unbind<'bound>(dev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { let _ = (dev, this); } } diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index 9a243e7c7298..c4d2d36602af 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -58,10 +58,13 @@ unsafe impl kernel::transmute::FromBytes for MyStruct {} impl pci::Driver for DmaSampleDriver { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const ID_TABLE: pci::IdTable = &PCI_TABLE; - fn probe(pdev: &pci::Device>, _info: &Self::IdInfo) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound pci::Device>, + _info: &'bound Self::IdInfo, + ) -> impl PinInit + 'bound { pin_init::pin_init_scope(move || { dev_info!(pdev, "Probe DMA test driver.\n"); diff --git a/samples/rust/rust_driver_auxiliary.rs b/samples/rust/rust_driver_auxiliary.rs index f0d419823f9a..0e979f45cd68 100644 --- a/samples/rust/rust_driver_auxiliary.rs +++ b/samples/rust/rust_driver_auxiliary.rs @@ -69,11 +69,14 @@ struct ParentDriver { impl pci::Driver for ParentDriver { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const ID_TABLE: pci::IdTable = &PCI_TABLE; - fn probe(pdev: &pci::Device>, _info: &Self::IdInfo) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound pci::Device>, + _info: &'bound Self::IdInfo, + ) -> impl PinInit + 'bound { Ok(Self { _reg0: auxiliary::Registration::new( pdev.as_ref(), diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs index 3106f766fd93..6791d98e1c79 100644 --- a/samples/rust/rust_driver_pci.rs +++ b/samples/rust/rust_driver_pci.rs @@ -140,11 +140,14 @@ fn config_space(pdev: &pci::Device) { impl pci::Driver for SampleDriver { type IdInfo = TestIndex; - type Data = Self; + type Data<'bound> = Self; const ID_TABLE: pci::IdTable = &PCI_TABLE; - fn probe(pdev: &pci::Device>, info: &Self::IdInfo) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound pci::Device>, + info: &'bound Self::IdInfo, + ) -> impl PinInit + 'bound { pin_init::pin_init_scope(move || { let vendor = pdev.vendor_id(); dev_dbg!( From 81fdc788144348f295cfaa4b1e1edf6c74441c15 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:00 +0200 Subject: [PATCH 076/131] rust: platform: make Driver trait lifetime-parameterized MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a 'bound lifetime to the associated Data, changing type Data to type Data<'bound>. This allows the driver's bus device private data to capture the device / driver bound lifetime; device resources can be stored directly by reference rather than requiring Devres. The probe() and unbind() callbacks thus gain a 'bound lifetime parameter on the methods themselves; avoiding a global lifetime on the trait impl. Existing drivers set type Data<'bound> = Self, preserving the current behavior. Acked-by: Uwe Kleine-König Reviewed-by: Alexandre Courbot Reviewed-by: Greg Kroah-Hartman Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-14-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/cpufreq/rcpufreq_dt.rs | 10 +++++----- drivers/gpu/drm/tyr/driver.rs | 10 +++++----- drivers/pwm/pwm_th1520.rs | 10 +++++----- rust/kernel/cpufreq.rs | 10 +++++----- rust/kernel/io/mem.rs | 20 ++++++++++---------- rust/kernel/platform.rs | 28 ++++++++++++++-------------- samples/rust/rust_debugfs.rs | 10 +++++----- samples/rust/rust_driver_platform.rs | 10 +++++----- samples/rust/rust_i2c_client.rs | 15 +++++++++------ samples/rust/rust_soc.rs | 10 +++++----- 10 files changed, 68 insertions(+), 65 deletions(-) diff --git a/drivers/cpufreq/rcpufreq_dt.rs b/drivers/cpufreq/rcpufreq_dt.rs index 5e0b224f6699..10106fa13095 100644 --- a/drivers/cpufreq/rcpufreq_dt.rs +++ b/drivers/cpufreq/rcpufreq_dt.rs @@ -201,13 +201,13 @@ fn register_em(policy: &mut cpufreq::Policy) { impl platform::Driver for CPUFreqDTDriver { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); - fn probe( - pdev: &platform::Device>, - _id_info: Option<&Self::IdInfo>, - ) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound platform::Device>, + _id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit + 'bound { cpufreq::Registration::::new_foreign_owned(pdev.as_ref())?; Ok(Self {}) } diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 001727f44fc8..797f09e23a4c 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -91,13 +91,13 @@ fn issue_soft_reset(dev: &Device, iomem: &Devres) -> Result { impl platform::Driver for TyrPlatformDriverData { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); - fn probe( - pdev: &platform::Device>, - _info: Option<&Self::IdInfo>, - ) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound platform::Device>, + _info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit + 'bound { let core_clk = Clk::get(pdev.as_ref(), Some(c"core"))?; let stacks_clk = OptionalClk::get(pdev.as_ref(), Some(c"stacks"))?; let coregroup_clk = OptionalClk::get(pdev.as_ref(), Some(c"coregroup"))?; diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs index df83a4a9a507..6c5b791f3153 100644 --- a/drivers/pwm/pwm_th1520.rs +++ b/drivers/pwm/pwm_th1520.rs @@ -316,13 +316,13 @@ fn drop(self: Pin<&mut Self>) { impl platform::Driver for Th1520PwmPlatformDriver { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); - fn probe( - pdev: &platform::Device>, - _id_info: Option<&Self::IdInfo>, - ) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound platform::Device>, + _id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit + 'bound { let dev = pdev.as_ref(); let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 0df518fa1d77..d94c6cdbc45a 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -888,13 +888,13 @@ fn register_em(_policy: &mut Policy) { /// /// impl platform::Driver for SampleDriver { /// type IdInfo = (); -/// type Data = Self; +/// type Data<'bound> = Self; /// const OF_ID_TABLE: Option> = None; /// -/// fn probe( -/// pdev: &platform::Device>, -/// _id_info: Option<&Self::IdInfo>, -/// ) -> impl PinInit { +/// fn probe<'bound>( +/// pdev: &'bound platform::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit + 'bound { /// cpufreq::Registration::::new_foreign_owned(pdev.as_ref())?; /// Ok(Self {}) /// } diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 03d8745b5e1d..51ba347220ee 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -62,12 +62,12 @@ pub(crate) unsafe fn new(device: &'a Device, resource: &'a Resource) -> S /// /// impl platform::Driver for SampleDriver { /// # type IdInfo = (); - /// # type Data = Self; + /// # type Data<'bound> = Self; /// - /// fn probe( - /// pdev: &platform::Device>, - /// info: Option<&Self::IdInfo>, - /// ) -> impl PinInit { + /// fn probe<'bound>( + /// pdev: &'bound platform::Device>, + /// info: Option<&'bound Self::IdInfo>, + /// ) -> impl PinInit + 'bound { /// let offset = 0; // Some offset. /// /// // If the size is known at compile time, use [`Self::iomap_sized`]. @@ -127,12 +127,12 @@ pub fn iomap_exclusive_sized( /// /// impl platform::Driver for SampleDriver { /// # type IdInfo = (); - /// # type Data = Self; + /// # type Data<'bound> = Self; /// - /// fn probe( - /// pdev: &platform::Device>, - /// info: Option<&Self::IdInfo>, - /// ) -> impl PinInit { + /// fn probe<'bound>( + /// pdev: &'bound platform::Device>, + /// info: Option<&'bound Self::IdInfo>, + /// ) -> impl PinInit + 'bound { /// let offset = 0; // Some offset. /// /// // Unlike [`Self::iomap_sized`], here the size of the memory region diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 257b7084338c..d8d48f60b0b9 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -50,7 +50,7 @@ // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::platform_driver; - type DriverData<'bound> = T::Data; + type DriverData<'bound> = T::Data<'bound>; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } @@ -117,8 +117,8 @@ extern "C" fn remove_callback(pdev: *mut bindings::platform_device) { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { pdev.as_ref().drvdata_borrow::() }; + // and stored a `Pin>>`. + let data = unsafe { pdev.as_ref().drvdata_borrow::>() }; T::unbind(pdev, data); } @@ -192,14 +192,14 @@ macro_rules! module_platform_driver { /// /// impl platform::Driver for MyDriver { /// type IdInfo = (); -/// type Data = Self; +/// type Data<'bound> = Self; /// const OF_ID_TABLE: Option> = Some(&OF_TABLE); /// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); /// -/// fn probe( -/// _pdev: &platform::Device>, -/// _id_info: Option<&Self::IdInfo>, -/// ) -> impl PinInit { +/// fn probe<'bound>( +/// _pdev: &'bound platform::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { /// Err(ENODEV) /// } /// } @@ -214,7 +214,7 @@ pub trait Driver { type IdInfo: 'static; /// The type of the driver's bus device private data. - type Data: Send; + type Data<'bound>: Send + 'bound; /// The table of OF device ids supported by the driver. const OF_ID_TABLE: Option> = None; @@ -226,10 +226,10 @@ pub trait Driver { /// /// Called when a new platform device is added or discovered. /// Implementers should attempt to initialize the device here. - fn probe( - dev: &Device>, - id_info: Option<&Self::IdInfo>, - ) -> impl PinInit; + fn probe<'bound>( + dev: &'bound Device>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; /// Platform driver unbind. /// @@ -241,7 +241,7 @@ fn probe( /// operations to gracefully tear down the device. /// /// Otherwise, release operations for driver resources should be performed in `Drop`. - fn unbind(dev: &Device>, this: Pin<&Self::Data>) { + fn unbind<'bound>(dev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { let _ = (dev, this); } } diff --git a/samples/rust/rust_debugfs.rs b/samples/rust/rust_debugfs.rs index 37640ed33642..1f59e08aaa4b 100644 --- a/samples/rust/rust_debugfs.rs +++ b/samples/rust/rust_debugfs.rs @@ -117,14 +117,14 @@ fn from_str(s: &str) -> Result { impl platform::Driver for RustDebugFs { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const OF_ID_TABLE: Option> = None; const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); - fn probe( - pdev: &platform::Device>, - _info: Option<&Self::IdInfo>, - ) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound platform::Device>, + _info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit + 'bound { RustDebugFs::new(pdev).pin_chain(|this| { this.counter.store(91, Relaxed); { diff --git a/samples/rust/rust_driver_platform.rs b/samples/rust/rust_driver_platform.rs index 04d40f836275..ec0d6cac4f57 100644 --- a/samples/rust/rust_driver_platform.rs +++ b/samples/rust/rust_driver_platform.rs @@ -101,14 +101,14 @@ struct SampleDriver { impl platform::Driver for SampleDriver { type IdInfo = Info; - type Data = Self; + type Data<'bound> = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); - fn probe( - pdev: &platform::Device>, - info: Option<&Self::IdInfo>, - ) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound platform::Device>, + info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit + 'bound { let dev = pdev.as_ref(); dev_dbg!(dev, "Probe Rust Platform driver sample.\n"); diff --git a/samples/rust/rust_i2c_client.rs b/samples/rust/rust_i2c_client.rs index 3f273c754f86..2d876f4e3ee0 100644 --- a/samples/rust/rust_i2c_client.rs +++ b/samples/rust/rust_i2c_client.rs @@ -106,14 +106,14 @@ struct SampleDriver { impl platform::Driver for SampleDriver { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); - fn probe( - pdev: &platform::Device>, - _info: Option<&Self::IdInfo>, - ) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound platform::Device>, + _info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit + 'bound { dev_info!( pdev.as_ref(), "Probe Rust I2C Client registration sample.\n" @@ -130,7 +130,10 @@ fn probe( }) } - fn unbind(pdev: &platform::Device>, _this: Pin<&Self>) { + fn unbind<'bound>( + pdev: &'bound platform::Device>, + _this: Pin<&Self::Data<'bound>>, + ) { dev_info!( pdev.as_ref(), "Unbind Rust I2C Client registration sample.\n" diff --git a/samples/rust/rust_soc.rs b/samples/rust/rust_soc.rs index c466653491d2..808d58200eb6 100644 --- a/samples/rust/rust_soc.rs +++ b/samples/rust/rust_soc.rs @@ -37,14 +37,14 @@ struct SampleSocDriver { impl platform::Driver for SampleSocDriver { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const OF_ID_TABLE: Option> = Some(&OF_TABLE); const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); - fn probe( - pdev: &platform::Device>, - _info: Option<&Self::IdInfo>, - ) -> impl PinInit { + fn probe<'bound>( + pdev: &'bound platform::Device>, + _info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit + 'bound { dev_dbg!(pdev, "Probe Rust SoC driver sample.\n"); let pdev = pdev.into(); From 46f651d88662ef931555cd135f09382af206295a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:01 +0200 Subject: [PATCH 077/131] rust: auxiliary: make Driver trait lifetime-parameterized Add a 'bound lifetime to the associated Data, changing type Data to type Data<'bound>. This allows the driver's bus device private data to capture the device / driver bound lifetime; device resources can be stored directly by reference rather than requiring Devres. The probe() and unbind() callbacks thus gain a 'bound lifetime parameter on the methods themselves; avoiding a global lifetime on the trait impl. Existing drivers set type Data<'bound> = Self, preserving the current behavior. Reviewed-by: Alexandre Courbot Reviewed-by: Greg Kroah-Hartman Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-15-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 10 +++++----- rust/kernel/auxiliary.rs | 18 +++++++++--------- samples/rust/rust_driver_auxiliary.rs | 10 +++++----- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 7605348af994..aa08644012f7 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -51,13 +51,13 @@ pub(crate) struct NovaData { impl auxiliary::Driver for NovaDriver { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const ID_TABLE: auxiliary::IdTable = &AUX_TABLE; - fn probe( - adev: &auxiliary::Device>, - _info: &Self::IdInfo, - ) -> impl PinInit { + fn probe<'bound>( + adev: &'bound auxiliary::Device>, + _info: &'bound Self::IdInfo, + ) -> impl PinInit + 'bound { let data = try_pin_init!(NovaData { adev: adev.into() }); let drm = drm::Device::::new(adev.as_ref(), data)?; diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index 6d504b0933d5..7a1b1a7b7ca6 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -46,7 +46,7 @@ // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::auxiliary_driver; - type DriverData<'bound> = T::Data; + type DriverData<'bound> = T::Data<'bound>; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } @@ -111,8 +111,8 @@ extern "C" fn remove_callback(adev: *mut bindings::auxiliary_device) { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { adev.as_ref().drvdata_borrow::() }; + // and stored a `Pin>>`. + let data = unsafe { adev.as_ref().drvdata_borrow::>() }; T::unbind(adev, data); } @@ -203,7 +203,7 @@ pub trait Driver { type IdInfo: 'static; /// The type of the driver's bus device private data. - type Data: Send; + type Data<'bound>: Send + 'bound; /// The table of device ids supported by the driver. const ID_TABLE: IdTable; @@ -211,10 +211,10 @@ pub trait Driver { /// Auxiliary driver probe. /// /// Called when an auxiliary device is matches a corresponding driver. - fn probe( - dev: &Device>, - id_info: &Self::IdInfo, - ) -> impl PinInit; + fn probe<'bound>( + dev: &'bound Device>, + id_info: &'bound Self::IdInfo, + ) -> impl PinInit, Error> + 'bound; /// Auxiliary driver unbind. /// @@ -226,7 +226,7 @@ fn probe( /// operations to gracefully tear down the device. /// /// Otherwise, release operations for driver resources should be performed in `Drop`. - fn unbind(dev: &Device>, this: Pin<&Self::Data>) { + fn unbind<'bound>(dev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { let _ = (dev, this); } } diff --git a/samples/rust/rust_driver_auxiliary.rs b/samples/rust/rust_driver_auxiliary.rs index 0e979f45cd68..b30a4d5cdf8a 100644 --- a/samples/rust/rust_driver_auxiliary.rs +++ b/samples/rust/rust_driver_auxiliary.rs @@ -31,14 +31,14 @@ impl auxiliary::Driver for AuxiliaryDriver { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const ID_TABLE: auxiliary::IdTable = &AUX_TABLE; - fn probe( - adev: &auxiliary::Device>, - _info: &Self::IdInfo, - ) -> impl PinInit { + fn probe<'bound>( + adev: &'bound auxiliary::Device>, + _info: &'bound Self::IdInfo, + ) -> impl PinInit + 'bound { dev_info!( adev, "Probing auxiliary driver for auxiliary device with id={}\n", From a3f09f8e47c4262510c979b384d6f85d376d91f5 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:02 +0200 Subject: [PATCH 078/131] rust: usb: make Driver trait lifetime-parameterized Add a 'bound lifetime to the associated Data, changing type Data to type Data<'bound>. This allows the driver's bus device private data to capture the device / driver bound lifetime; device resources can be stored directly by reference rather than requiring Devres. The probe() and disconnect() callbacks thus gain a 'bound lifetime parameter on the methods themselves; avoiding a global lifetime on the trait impl. Existing drivers set type Data<'bound> = Self, preserving the current behavior. Reviewed-by: Alexandre Courbot Reviewed-by: Greg Kroah-Hartman Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260525202921.124698-16-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/usb.rs | 37 ++++++++++++++++++++------------- samples/rust/rust_driver_usb.rs | 12 +++++------ 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs index 1dbb8387b463..7aff0c82d0af 100644 --- a/rust/kernel/usb.rs +++ b/rust/kernel/usb.rs @@ -41,7 +41,7 @@ // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::usb_driver; - type DriverData<'bound> = T::Data; + type DriverData<'bound> = T::Data<'bound>; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } @@ -109,8 +109,8 @@ extern "C" fn disconnect_callback(intf: *mut bindings::usb_interface) { // SAFETY: `disconnect_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { dev.drvdata_borrow::() }; + // and stored a `Pin>>`. + let data = unsafe { dev.drvdata_borrow::>() }; T::disconnect(intf, data); } @@ -287,18 +287,22 @@ macro_rules! usb_device_table { /// /// impl usb::Driver for MyDriver { /// type IdInfo = (); -/// type Data = Self; +/// type Data<'bound> = Self; /// const ID_TABLE: usb::IdTable = &USB_TABLE; /// -/// fn probe( -/// _interface: &usb::Interface>, +/// fn probe<'bound>( +/// _interface: &'bound usb::Interface>, /// _id: &usb::DeviceId, -/// _info: &Self::IdInfo, -/// ) -> impl PinInit { +/// _info: &'bound Self::IdInfo, +/// ) -> impl PinInit, Error> + 'bound { /// Err(ENODEV) /// } /// -/// fn disconnect(_interface: &usb::Interface>, _data: Pin<&Self::Data>) {} +/// fn disconnect<'bound>( +/// _interface: &'bound usb::Interface>, +/// _data: Pin<&Self::Data<'bound>>, +/// ) { +/// } /// } ///``` pub trait Driver { @@ -306,7 +310,7 @@ pub trait Driver { type IdInfo: 'static; /// The type of the driver's bus device private data. - type Data: Send; + type Data<'bound>: Send + 'bound; /// The table of device ids supported by the driver. const ID_TABLE: IdTable; @@ -315,16 +319,19 @@ pub trait Driver { /// /// Called when a new USB interface is bound to this driver. /// Implementers should attempt to initialize the interface here. - fn probe( - interface: &Interface>, + fn probe<'bound>( + interface: &'bound Interface>, id: &DeviceId, - id_info: &Self::IdInfo, - ) -> impl PinInit; + id_info: &'bound Self::IdInfo, + ) -> impl PinInit, Error> + 'bound; /// USB driver disconnect. /// /// Called when the USB interface is about to be unbound from this driver. - fn disconnect(interface: &Interface>, data: Pin<&Self::Data>); + fn disconnect<'bound>( + interface: &'bound Interface>, + data: Pin<&Self::Data<'bound>>, + ); } /// A USB interface. diff --git a/samples/rust/rust_driver_usb.rs b/samples/rust/rust_driver_usb.rs index e900993335e9..02bd5085f9bc 100644 --- a/samples/rust/rust_driver_usb.rs +++ b/samples/rust/rust_driver_usb.rs @@ -26,21 +26,21 @@ struct SampleDriver { impl usb::Driver for SampleDriver { type IdInfo = (); - type Data = Self; + type Data<'bound> = Self; const ID_TABLE: usb::IdTable = &USB_TABLE; - fn probe( - intf: &usb::Interface>, + fn probe<'bound>( + intf: &'bound usb::Interface>, _id: &usb::DeviceId, - _info: &Self::IdInfo, - ) -> impl PinInit { + _info: &'bound Self::IdInfo, + ) -> impl PinInit + 'bound { let dev: &device::Device> = intf.as_ref(); dev_info!(dev, "Rust USB driver sample probed\n"); Ok(Self { _intf: intf.into() }) } - fn disconnect(intf: &usb::Interface>, _data: Pin<&Self>) { + fn disconnect<'bound>(intf: &'bound usb::Interface>, _data: Pin<&Self>) { let dev: &device::Device> = intf.as_ref(); dev_info!(dev, "Rust USB driver sample disconnected\n"); } From 71e6b6a80b5158323be56e0a776e9fa3cc77d061 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:03 +0200 Subject: [PATCH 079/131] rust: i2c: make Driver trait lifetime-parameterized Add a 'bound lifetime to the associated Data, changing type Data to type Data<'bound>. This allows the driver's bus device private data to capture the device / driver bound lifetime; device resources can be stored directly by reference rather than requiring Devres. The probe() and unbind() callbacks thus gain a 'bound lifetime parameter on the methods themselves; avoiding a global lifetime on the trait impl. Existing drivers set type Data<'bound> = Self, preserving the current behavior. Acked-by: Igor Korotin Reviewed-by: Alexandre Courbot Reviewed-by: Greg Kroah-Hartman Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-17-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/i2c.rs | 39 ++++++++++++++++++--------------- samples/rust/rust_driver_i2c.rs | 14 ++++++------ 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs index 50feade0fb58..6094d32652e3 100644 --- a/rust/kernel/i2c.rs +++ b/rust/kernel/i2c.rs @@ -98,7 +98,7 @@ macro_rules! i2c_device_table { // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. unsafe impl driver::DriverLayout for Adapter { type DriverType = bindings::i2c_driver; - type DriverData<'bound> = T::Data; + type DriverData<'bound> = T::Data<'bound>; const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); } @@ -176,8 +176,8 @@ extern "C" fn remove_callback(idev: *mut bindings::i2c_client) { // SAFETY: `remove_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { idev.as_ref().drvdata_borrow::() }; + // and stored a `Pin>>`. + let data = unsafe { idev.as_ref().drvdata_borrow::>() }; T::unbind(idev, data); } @@ -188,8 +188,8 @@ extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) { // SAFETY: `shutdown_callback` is only ever called after a successful call to // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called - // and stored a `Pin>`. - let data = unsafe { idev.as_ref().drvdata_borrow::() }; + // and stored a `Pin>>`. + let data = unsafe { idev.as_ref().drvdata_borrow::>() }; T::shutdown(idev, data); } @@ -294,19 +294,22 @@ macro_rules! module_i2c_driver { /// /// impl i2c::Driver for MyDriver { /// type IdInfo = (); -/// type Data = Self; +/// type Data<'bound> = Self; /// const I2C_ID_TABLE: Option> = Some(&I2C_TABLE); /// const OF_ID_TABLE: Option> = Some(&OF_TABLE); /// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); /// -/// fn probe( -/// _idev: &i2c::I2cClient>, -/// _id_info: Option<&Self::IdInfo>, -/// ) -> impl PinInit { +/// fn probe<'bound>( +/// _idev: &'bound i2c::I2cClient>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { /// Err(ENODEV) /// } /// -/// fn shutdown(_idev: &i2c::I2cClient>, this: Pin<&Self::Data>) { +/// fn shutdown<'bound>( +/// _idev: &'bound i2c::I2cClient>, +/// this: Pin<&Self::Data<'bound>>, +/// ) { /// } /// } ///``` @@ -320,7 +323,7 @@ pub trait Driver { type IdInfo: 'static; /// The type of the driver's bus device private data. - type Data: Send; + type Data<'bound>: Send + 'bound; /// The table of device ids supported by the driver. const I2C_ID_TABLE: Option> = None; @@ -335,10 +338,10 @@ pub trait Driver { /// /// Called when a new i2c client is added or discovered. /// Implementers should attempt to initialize the client here. - fn probe( - dev: &I2cClient>, - id_info: Option<&Self::IdInfo>, - ) -> impl PinInit; + fn probe<'bound>( + dev: &'bound I2cClient>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; /// I2C driver shutdown. /// @@ -351,7 +354,7 @@ fn probe( /// This callback is distinct from final resource cleanup, as the driver instance remains valid /// after it returns. Any deallocation or teardown of driver-owned resources should instead be /// handled in `Drop`. - fn shutdown(dev: &I2cClient>, this: Pin<&Self::Data>) { + fn shutdown<'bound>(dev: &'bound I2cClient>, this: Pin<&Self::Data<'bound>>) { let _ = (dev, this); } @@ -365,7 +368,7 @@ fn shutdown(dev: &I2cClient>, this: Pin<&Self::Data>) { /// operations to gracefully tear down the device. /// /// Otherwise, release operations for driver resources should be performed in `Drop`. - fn unbind(dev: &I2cClient>, this: Pin<&Self::Data>) { + fn unbind<'bound>(dev: &'bound I2cClient>, this: Pin<&Self::Data<'bound>>) { let _ = (dev, this); } } diff --git a/samples/rust/rust_driver_i2c.rs b/samples/rust/rust_driver_i2c.rs index 171550ea0b6f..ead8263a7d48 100644 --- a/samples/rust/rust_driver_i2c.rs +++ b/samples/rust/rust_driver_i2c.rs @@ -35,16 +35,16 @@ impl i2c::Driver for SampleDriver { type IdInfo = u32; - type Data = Self; + type Data<'bound> = Self; const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); const I2C_ID_TABLE: Option> = Some(&I2C_TABLE); const OF_ID_TABLE: Option> = Some(&OF_TABLE); - fn probe( - idev: &i2c::I2cClient>, - info: Option<&Self::IdInfo>, - ) -> impl PinInit { + fn probe<'bound>( + idev: &'bound i2c::I2cClient>, + info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit + 'bound { let dev = idev.as_ref(); dev_info!(dev, "Probe Rust I2C driver sample.\n"); @@ -56,11 +56,11 @@ fn probe( Ok(Self) } - fn shutdown(idev: &i2c::I2cClient>, _this: Pin<&Self>) { + fn shutdown<'bound>(idev: &'bound i2c::I2cClient>, _this: Pin<&Self>) { dev_info!(idev.as_ref(), "Shutdown Rust I2C driver sample.\n"); } - fn unbind(idev: &i2c::I2cClient>, _this: Pin<&Self>) { + fn unbind<'bound>(idev: &'bound i2c::I2cClient>, _this: Pin<&Self>) { dev_info!(idev.as_ref(), "Unbind Rust I2C driver sample.\n"); } } From d31a349a7fd88c4cc7ba85bce6491c398408997a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:04 +0200 Subject: [PATCH 080/131] rust: driver: update module documentation for GAT-based Data type Now that all bus driver traits use type Data<'bound>: 'bound, update the illustrative driver trait in the module documentation to reflect the GAT pattern and lifetime-parameterized callbacks. Reviewed-by: Alexandre Courbot Reviewed-by: Greg Kroah-Hartman Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-18-dakr@kernel.org Signed-off-by: Danilo Krummrich --- rust/kernel/driver.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index 558fdef4a1c6..03c0dd713f4c 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -18,7 +18,7 @@ //! type IdInfo: 'static; //! //! /// The type of the driver's bus device private data. -//! type Data: Send; +//! type Data<'bound>: Send + 'bound; //! //! /// The table of OF device ids supported by the driver. //! const OF_ID_TABLE: Option> = None; @@ -27,11 +27,16 @@ //! const ACPI_ID_TABLE: Option> = None; //! //! /// Driver probe. -//! fn probe(dev: &Device>, id_info: &Self::IdInfo) -//! -> impl PinInit; +//! fn probe<'bound>( +//! dev: &'bound Device>, +//! id_info: &'bound Self::IdInfo, +//! ) -> impl PinInit, Error> + 'bound; //! //! /// Driver unbind (optional). -//! fn unbind(dev: &Device>, this: Pin<&Self::Data>) { +//! fn unbind<'bound>( +//! dev: &'bound Device>, +//! this: Pin<&Self::Data<'bound>>, +//! ) { //! let _ = (dev, this); //! } //! } @@ -46,9 +51,10 @@ )] #![cfg_attr(CONFIG_PCI, doc = "* [`pci::Driver`](kernel::pci::Driver)")] //! -//! The `probe()` callback should return a `impl PinInit`, i.e. the driver's -//! private data. The bus abstraction should store the pointer in the corresponding bus device. The -//! generic [`Device`] infrastructure provides common helpers for this purpose on its +//! The `probe()` callback should return a +//! `impl PinInit, Error>`, i.e. the driver's private data. The bus +//! abstraction should store the pointer in the corresponding bus device. The generic +//! [`Device`] infrastructure provides common helpers for this purpose on its //! [`Device`] implementation. //! //! All driver callbacks should provide a reference to the driver's private data. Once the driver From 8ea0b6d5bef5e4f4637964c3b2cf732d9bf4f408 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:05 +0200 Subject: [PATCH 081/131] rust: pci: make Bar lifetime-parameterized Convert pci::Bar to pci::Bar<'a, SIZE>, storing &'a Device to tie the BAR mapping lifetime to the device. iomap_region_sized() now returns Result> directly instead of impl PinInit>, Error>. Since the lifetime ties the mapping to the device's bound state, callers no longer need Devres for the common case where the Bar lives in the driver's private data. Add Bar::into_devres() to consume the bar and register it as a device-managed resource, returning Devres>. The lifetime is erased to 'static because Devres guarantees the bar does not actually outlive the device -- access is revoked on unbind. Reviewed-by: Eliot Courtney Reviewed-by: Greg Kroah-Hartman Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-19-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 7 +++-- rust/kernel/devres.rs | 2 +- rust/kernel/pci/io.rs | 52 +++++++++++++++++++-------------- samples/rust/rust_driver_pci.rs | 5 ++-- 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 6ad1a856694c..7dbec0470c26 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -45,7 +45,7 @@ pub(crate) struct NovaCore { // DMA addresses. These systems should be quite rare. const GPU_DMA_BITS: u32 = 47; -pub(crate) type Bar0 = pci::Bar; +pub(crate) type Bar0 = pci::Bar<'static, BAR0_SIZE>; kernel::pci_device_table!( PCI_TABLE, @@ -92,8 +92,9 @@ fn probe<'bound>( // other threads of execution. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::())? }; - let bar = Arc::pin_init( - pdev.iomap_region_sized::(0, c"nova-core/bar0"), + let bar = Arc::new( + pdev.iomap_region_sized::(0, c"nova-core/bar0")? + .into_devres()?, GFP_KERNEL, )?; diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index fd4633f977f6..82cbd8b969fb 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -304,7 +304,7 @@ pub fn device(&self) -> &Device { /// pci, // /// }; /// - /// fn from_core(dev: &pci::Device>, devres: Devres>) -> Result { + /// fn from_core(dev: &pci::Device>, devres: Devres>) -> Result { /// let bar = devres.access(dev.as_ref())?; /// /// let _ = bar.read32(0x0); diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index 3ce21482b079..0461e01aaa20 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -14,8 +14,7 @@ Mmio, MmioRaw, // }, - prelude::*, - sync::aref::ARef, // + prelude::*, // }; use core::{ marker::PhantomData, @@ -146,14 +145,18 @@ impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { /// /// `Bar` always holds an `IoRaw` instance that holds a valid pointer to the start of the I/O /// memory mapped PCI BAR and its size. -pub struct Bar { - pdev: ARef, +pub struct Bar<'a, const SIZE: usize = 0> { + pdev: &'a Device, io: MmioRaw, num: i32, } -impl Bar { - pub(super) fn new(pdev: &Device, num: u32, name: &'static CStr) -> Result { +impl<'a, const SIZE: usize> Bar<'a, SIZE> { + pub(super) fn new( + pdev: &'a Device, + num: u32, + name: &'static CStr, + ) -> Result { let len = pdev.resource_len(num)?; if len == 0 { return Err(ENOMEM); @@ -196,11 +199,7 @@ pub(super) fn new(pdev: &Device, num: u32, name: &'static CStr) -> Result } }; - Ok(Bar { - pdev: pdev.into(), - io, - num, - }) + Ok(Bar { pdev, io, num }) } /// # Safety @@ -219,11 +218,24 @@ unsafe fn do_release(pdev: &Device, ioptr: usize, num: i32) { fn release(&self) { // SAFETY: The safety requirements are guaranteed by the type invariant of `self.pdev`. - unsafe { Self::do_release(&self.pdev, self.io.addr(), self.num) }; + unsafe { Self::do_release(self.pdev, self.io.addr(), self.num) }; + } + + /// Consume the `Bar` and register it as a device-managed resource. + /// + /// The returned `Devres>` can outlive the original lifetime `'a`. Access + /// to the BAR is revoked when the device is unbound. + pub fn into_devres(self) -> Result>> { + // SAFETY: Casting to `'static` is sound because `Devres` guarantees the `Bar` does not + // actually outlive the device -- access is revoked and the resource is released when the + // device is unbound. + let bar: Bar<'static, SIZE> = unsafe { core::mem::transmute(self) }; + let pdev = bar.pdev; + Devres::new(pdev.as_ref(), bar) } } -impl Bar { +impl Bar<'_> { #[inline] pub(super) fn index_is_valid(index: u32) -> bool { // A `struct pci_dev` owns an array of resources with at most `PCI_NUM_RESOURCES` entries. @@ -231,13 +243,13 @@ pub(super) fn index_is_valid(index: u32) -> bool { } } -impl Drop for Bar { +impl Drop for Bar<'_, SIZE> { fn drop(&mut self) { self.release(); } } -impl Deref for Bar { +impl Deref for Bar<'_, SIZE> { type Target = Mmio; fn deref(&self) -> &Self::Target { @@ -253,16 +265,12 @@ pub fn iomap_region_sized<'a, const SIZE: usize>( &'a self, bar: u32, name: &'static CStr, - ) -> impl PinInit>, Error> + 'a { - Devres::new(self.as_ref(), Bar::::new(self, bar, name)) + ) -> Result> { + Bar::new(self, bar, name) } /// Maps an entire PCI BAR after performing a region-request on it. - pub fn iomap_region<'a>( - &'a self, - bar: u32, - name: &'static CStr, - ) -> impl PinInit, Error> + 'a { + pub fn iomap_region<'a>(&'a self, bar: u32, name: &'static CStr) -> Result> { self.iomap_region_sized::<0>(bar, name) } diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs index 6791d98e1c79..0353481b0690 100644 --- a/samples/rust/rust_driver_pci.rs +++ b/samples/rust/rust_driver_pci.rs @@ -45,7 +45,7 @@ mod regs { pub(super) const END: usize = 0x10; } -type Bar0 = pci::Bar<{ regs::END }>; +type Bar0 = pci::Bar<'static, { regs::END }>; #[derive(Copy, Clone, Debug)] struct TestIndex(u8); @@ -161,7 +161,8 @@ fn probe<'bound>( pdev.set_master(); Ok(try_pin_init!(Self { - bar <- pdev.iomap_region_sized::<{ regs::END }>(0, c"rust_driver_pci"), + bar: pdev.iomap_region_sized::<{ regs::END }>(0, c"rust_driver_pci")? + .into_devres()?, index: *info, _: { let bar = bar.access(pdev.as_ref())?; From 89f55d04c6028fa15800a4887faf51bdeebfa431 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:06 +0200 Subject: [PATCH 082/131] rust: io: make IoMem and ExclusiveIoMem lifetime-parameterized MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a lifetime parameter to IoMem<'a, SIZE> and ExclusiveIoMem<'a, SIZE>, storing a &'a Device reference to tie the mapping to the device's lifetime. This mirrors the pci::Bar<'a, SIZE> design and enables drivers to hold I/O memory mappings directly in their HRT private data, tied to the device lifetime. IoRequest::iomap_* methods now return the mapping directly instead of wrapping it in Devres. Callers that need device-managed revocation can call the new into_devres() method. Acked-by: Uwe Kleine-König Reviewed-by: Eliot Courtney Reviewed-by: Greg Kroah-Hartman Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-20-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/tyr/driver.rs | 4 +- drivers/pwm/pwm_th1520.rs | 4 +- rust/kernel/io/mem.rs | 103 +++++++++++++++++----------------- 3 files changed, 56 insertions(+), 55 deletions(-) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 797f09e23a4c..04f83fcf0937 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -37,7 +37,7 @@ regs, // }; -pub(crate) type IoMem = kernel::io::mem::IoMem; +pub(crate) type IoMem = kernel::io::mem::IoMem<'static, SZ_2M>; pub(crate) struct TyrDrmDriver; @@ -110,7 +110,7 @@ fn probe<'bound>( let sram_regulator = Regulator::::get(pdev.as_ref(), c"sram")?; let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; - let iomem = Arc::pin_init(request.iomap_sized::(), GFP_KERNEL)?; + let iomem = Arc::new(request.iomap_sized::()?.into_devres()?, GFP_KERNEL)?; issue_soft_reset(pdev.as_ref(), &iomem)?; gpu::l2_power_on(pdev.as_ref(), &iomem)?; diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs index 6c5b791f3153..48808cd80737 100644 --- a/drivers/pwm/pwm_th1520.rs +++ b/drivers/pwm/pwm_th1520.rs @@ -92,7 +92,7 @@ struct Th1520WfHw { #[pin_data(PinnedDrop)] struct Th1520PwmDriverData { #[pin] - iomem: devres::Devres>, + iomem: devres::Devres>, clk: Clk, } @@ -352,7 +352,7 @@ fn probe<'bound>( dev, TH1520_MAX_PWM_NUM, try_pin_init!(Th1520PwmDriverData { - iomem <- request.iomap_sized::(), + iomem <- request.iomap_sized::()?.into_devres(), clk <- clk, }), )?; diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 51ba347220ee..fc2a3e24f8d5 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -74,22 +74,19 @@ pub(crate) unsafe fn new(device: &'a Device, resource: &'a Resource) -> S /// // /// // No runtime checks will apply when reading and writing. /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; - /// let iomem = request.iomap_sized::<42>(); - /// let iomem = KBox::pin_init(iomem, GFP_KERNEL)?; - /// - /// let io = iomem.access(pdev.as_ref())?; + /// let iomem = request.iomap_sized::<42>()?; /// /// // Read and write a 32-bit value at `offset`. - /// let data = io.read32(offset); + /// let data = iomem.read32(offset); /// - /// io.write32(data, offset); + /// iomem.write32(data, offset); /// /// # Ok(SampleDriver) /// } /// } /// ``` - pub fn iomap_sized(self) -> impl PinInit>, Error> + 'a { - IoMem::new(self) + pub fn iomap_sized(self) -> Result> { + IoMem::ioremap(self.device, self.resource) } /// Same as [`Self::iomap_sized`] but with exclusive access to the @@ -98,10 +95,8 @@ pub fn iomap_sized(self) -> impl PinInit>, /// This uses the [`ioremap()`] C API. /// /// [`ioremap()`]: https://docs.kernel.org/driver-api/device-io.html#getting-access-to-the-device - pub fn iomap_exclusive_sized( - self, - ) -> impl PinInit>, Error> + 'a { - ExclusiveIoMem::new(self) + pub fn iomap_exclusive_sized(self) -> Result> { + ExclusiveIoMem::ioremap(self.device, self.resource) } /// Maps an [`IoRequest`] where the size is not known at compile time, @@ -140,27 +135,24 @@ pub fn iomap_exclusive_sized( /// // family of functions should be used, leading to runtime checks on every /// // access. /// let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; - /// let iomem = request.iomap(); - /// let iomem = KBox::pin_init(iomem, GFP_KERNEL)?; + /// let iomem = request.iomap()?; /// - /// let io = iomem.access(pdev.as_ref())?; + /// let data = iomem.try_read32(offset)?; /// - /// let data = io.try_read32(offset)?; - /// - /// io.try_write32(data, offset)?; + /// iomem.try_write32(data, offset)?; /// /// # Ok(SampleDriver) /// } /// } /// ``` - pub fn iomap(self) -> impl PinInit>, Error> + 'a { - Self::iomap_sized::<0>(self) + pub fn iomap(self) -> Result> { + self.iomap_sized::<0>() } /// Same as [`Self::iomap`] but with exclusive access to the underlying /// region. - pub fn iomap_exclusive(self) -> impl PinInit>, Error> + 'a { - Self::iomap_exclusive_sized::<0>(self) + pub fn iomap_exclusive(self) -> Result> { + self.iomap_exclusive_sized::<0>() } } @@ -169,9 +161,9 @@ pub fn iomap_exclusive(self) -> impl PinInit>, Error> + /// # Invariants /// /// - [`ExclusiveIoMem`] has exclusive access to the underlying [`IoMem`]. -pub struct ExclusiveIoMem { +pub struct ExclusiveIoMem<'a, const SIZE: usize> { /// The underlying `IoMem` instance. - iomem: IoMem, + iomem: IoMem<'a, SIZE>, /// The region abstraction. This represents exclusive access to the /// range represented by the underlying `iomem`. @@ -180,9 +172,9 @@ pub struct ExclusiveIoMem { _region: Region, } -impl ExclusiveIoMem { +impl<'a, const SIZE: usize> ExclusiveIoMem<'a, SIZE> { /// Creates a new `ExclusiveIoMem` instance. - fn ioremap(resource: &Resource) -> Result { + fn ioremap(dev: &'a Device, resource: &Resource) -> Result { let start = resource.start(); let size = resource.size(); let name = resource.name().unwrap_or_default(); @@ -196,26 +188,29 @@ fn ioremap(resource: &Resource) -> Result { ) .ok_or(EBUSY)?; - let iomem = IoMem::ioremap(resource)?; + let iomem = IoMem::ioremap(dev, resource)?; - let iomem = ExclusiveIoMem { + Ok(ExclusiveIoMem { iomem, _region: region, - }; - - Ok(iomem) + }) } - /// Creates a new `ExclusiveIoMem` instance from a previously acquired [`IoRequest`]. - pub fn new<'a>(io_request: IoRequest<'a>) -> impl PinInit, Error> + 'a { - let dev = io_request.device; - let res = io_request.resource; - - Devres::new(dev, Self::ioremap(res)) + /// Consume the `ExclusiveIoMem` and register it as a device-managed resource. + /// + /// The returned `Devres>` can outlive the original lifetime + /// `'a`. Access to the I/O memory is revoked when the device is unbound. + pub fn into_devres(self) -> Result>> { + // SAFETY: Casting to `'static` is sound because `Devres` guarantees the + // `ExclusiveIoMem` does not actually outlive the device -- access is revoked and the + // resource is released when the device is unbound. + let iomem: ExclusiveIoMem<'static, SIZE> = unsafe { core::mem::transmute(self) }; + let dev = iomem.iomem.dev; + Devres::new(dev, iomem) } } -impl Deref for ExclusiveIoMem { +impl Deref for ExclusiveIoMem<'_, SIZE> { type Target = Mmio; fn deref(&self) -> &Self::Target { @@ -232,12 +227,13 @@ fn deref(&self) -> &Self::Target { /// /// [`IoMem`] always holds an [`MmioRaw`] instance that holds a valid pointer to the /// start of the I/O memory mapped region. -pub struct IoMem { +pub struct IoMem<'a, const SIZE: usize = 0> { + dev: &'a Device, io: MmioRaw, } -impl IoMem { - fn ioremap(resource: &Resource) -> Result { +impl<'a, const SIZE: usize> IoMem<'a, SIZE> { + fn ioremap(dev: &'a Device, resource: &Resource) -> Result { // Note: Some ioremap() implementations use types that depend on the CPU // word width rather than the bus address width. // @@ -269,28 +265,33 @@ fn ioremap(resource: &Resource) -> Result { } let io = MmioRaw::new(addr as usize, size)?; - let io = IoMem { io }; - Ok(io) + Ok(IoMem { dev, io }) } - /// Creates a new `IoMem` instance from a previously acquired [`IoRequest`]. - pub fn new<'a>(io_request: IoRequest<'a>) -> impl PinInit, Error> + 'a { - let dev = io_request.device; - let res = io_request.resource; - - Devres::new(dev, Self::ioremap(res)) + /// Consume the `IoMem` and register it as a device-managed resource. + /// + /// The returned `Devres>` can outlive the original + /// lifetime `'a`. Access to the I/O memory is revoked when the device + /// is unbound. + pub fn into_devres(self) -> Result>> { + // SAFETY: Casting to `'static` is sound because `Devres` guarantees the `IoMem` does not + // actually outlive the device -- access is revoked and the resource is released when the + // device is unbound. + let iomem: IoMem<'static, SIZE> = unsafe { core::mem::transmute(self) }; + let dev = iomem.dev; + Devres::new(dev, iomem) } } -impl Drop for IoMem { +impl Drop for IoMem<'_, SIZE> { fn drop(&mut self) { // SAFETY: Safe as by the invariant of `Io`. unsafe { bindings::iounmap(self.io.addr() as *mut c_void) } } } -impl Deref for IoMem { +impl Deref for IoMem<'_, SIZE> { type Target = Mmio; fn deref(&self) -> &Self::Target { From e397d405c4c6117b4eeeeecb8170c17c604ee6cc Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:07 +0200 Subject: [PATCH 083/131] samples: rust: rust_driver_pci: use HRT lifetime for Bar Convert the sample driver to SampleDriver<'bound>, taking advantage of the lifetime-parameterized Driver trait. The driver struct holds &'bound pci::Device directly instead of ARef, and pci::Bar<'bound> directly instead of Devres. This removes PinnedDrop, pin_init_scope, and runtime revocation checks on BAR access. Reviewed-by: Eliot Courtney Reviewed-by: Greg Kroah-Hartman Reviewed-by: Alexandre Courbot Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-21-dakr@kernel.org Signed-off-by: Danilo Krummrich --- samples/rust/rust_driver_pci.rs | 83 +++++++++++++++------------------ 1 file changed, 37 insertions(+), 46 deletions(-) diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs index 0353481b0690..1aa8197d8698 100644 --- a/samples/rust/rust_driver_pci.rs +++ b/samples/rust/rust_driver_pci.rs @@ -9,7 +9,6 @@ Bound, Core, // }, - devres::Devres, io::{ register, register::Array, @@ -17,8 +16,7 @@ }, num::Bounded, pci, - prelude::*, - sync::aref::ARef, // + prelude::*, // }; mod regs { @@ -45,7 +43,7 @@ mod regs { pub(super) const END: usize = 0x10; } -type Bar0 = pci::Bar<'static, { regs::END }>; +type Bar0<'bound> = pci::Bar<'bound, { regs::END }>; #[derive(Copy, Clone, Debug)] struct TestIndex(u8); @@ -66,14 +64,14 @@ impl TestIndex { const NO_EVENTFD: Self = Self(0); } -#[pin_data(PinnedDrop)] -struct SampleDriver { - pdev: ARef, - #[pin] - bar: Devres, +struct SampleDriverData<'bound> { + pdev: &'bound pci::Device, + bar: Bar0<'bound>, index: TestIndex, } +struct SampleDriver; + kernel::pci_device_table!( PCI_TABLE, MODULE_PCI_TABLE, @@ -84,8 +82,8 @@ struct SampleDriver { )] ); -impl SampleDriver { - fn testdev(index: &TestIndex, bar: &Bar0) -> Result { +impl SampleDriverData<'_> { + fn testdev(index: &TestIndex, bar: &Bar0<'_>) -> Result { // Select the test. bar.write_reg(regs::TEST::zeroed().with_index(*index)); @@ -140,56 +138,49 @@ fn config_space(pdev: &pci::Device) { impl pci::Driver for SampleDriver { type IdInfo = TestIndex; - type Data<'bound> = Self; + type Data<'bound> = SampleDriverData<'bound>; const ID_TABLE: pci::IdTable = &PCI_TABLE; fn probe<'bound>( pdev: &'bound pci::Device>, info: &'bound Self::IdInfo, - ) -> impl PinInit + 'bound { - pin_init::pin_init_scope(move || { - let vendor = pdev.vendor_id(); - dev_dbg!( - pdev, - "Probe Rust PCI driver sample (PCI ID: {}, 0x{:x}).\n", - vendor, - pdev.device_id() - ); + ) -> impl PinInit, Error> + 'bound { + let vendor = pdev.vendor_id(); + dev_dbg!( + pdev, + "Probe Rust PCI driver sample (PCI ID: {}, 0x{:x}).\n", + vendor, + pdev.device_id() + ); - pdev.enable_device_mem()?; - pdev.set_master(); + pdev.enable_device_mem()?; + pdev.set_master(); - Ok(try_pin_init!(Self { - bar: pdev.iomap_region_sized::<{ regs::END }>(0, c"rust_driver_pci")? - .into_devres()?, - index: *info, - _: { - let bar = bar.access(pdev.as_ref())?; + let bar = pdev.iomap_region_sized::<{ regs::END }>(0, c"rust_driver_pci")?; - dev_info!( - pdev, - "pci-testdev data-match count: {}\n", - Self::testdev(info, bar)? - ); - Self::config_space(pdev); - }, - pdev: pdev.into(), - })) + dev_info!( + pdev, + "pci-testdev data-match count: {}\n", + SampleDriverData::testdev(info, &bar)? + ); + SampleDriverData::config_space(pdev); + + Ok(SampleDriverData { + pdev, + bar, + index: *info, }) } - fn unbind(pdev: &pci::Device>, this: Pin<&Self>) { - if let Ok(bar) = this.bar.access(pdev.as_ref()) { - // Reset pci-testdev by writing a new test index. - bar.write_reg(regs::TEST::zeroed().with_index(this.index)); - } + fn unbind<'bound>(_pdev: &'bound pci::Device>, this: Pin<&Self::Data<'bound>>) { + this.bar + .write_reg(regs::TEST::zeroed().with_index(this.index)); } } -#[pinned_drop] -impl PinnedDrop for SampleDriver { - fn drop(self: Pin<&mut Self>) { +impl Drop for SampleDriverData<'_> { + fn drop(&mut self) { dev_dbg!(self.pdev, "Remove Rust PCI driver sample.\n"); } } From bb1cf43f2fa85beba82a0d9bbea21013cf94d7a0 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:08 +0200 Subject: [PATCH 084/131] gpu: nova-core: separate driver type from driver data Introduce NovaCoreDriver as the driver type implementing pci::Driver, keeping NovaCore as the per-device data type. This prepares for making NovaCore lifetime-parameterized once auxiliary::Registration requires a lifetime for the binding scope. Reviewed-by: Greg Kroah-Hartman Reviewed-by: Alexandre Courbot Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-22-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 14 ++++++++------ drivers/gpu/nova-core/nova_core.rs | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 7dbec0470c26..fa898fe5c893 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -35,6 +35,8 @@ pub(crate) struct NovaCore { _reg: Devres>, } +pub(crate) struct NovaCoreDriver; + const BAR0_SIZE: usize = SZ_16M; // For now we only support Ampere which can use up to 47-bit DMA addresses. @@ -50,7 +52,7 @@ pub(crate) struct NovaCore { kernel::pci_device_table!( PCI_TABLE, MODULE_PCI_TABLE, - ::IdInfo, + ::IdInfo, [ // Modern NVIDIA GPUs will show up as either VGA or 3D controllers. ( @@ -72,15 +74,15 @@ pub(crate) struct NovaCore { ] ); -impl pci::Driver for NovaCore { +impl pci::Driver for NovaCoreDriver { type IdInfo = (); - type Data<'bound> = Self; + type Data<'bound> = NovaCore; const ID_TABLE: pci::IdTable = &PCI_TABLE; fn probe<'bound>( pdev: &'bound pci::Device>, _info: &'bound Self::IdInfo, - ) -> impl PinInit + 'bound { + ) -> impl PinInit + 'bound { pin_init::pin_init_scope(move || { dev_dbg!(pdev, "Probe Nova Core GPU driver.\n"); @@ -98,7 +100,7 @@ fn probe<'bound>( GFP_KERNEL, )?; - Ok(try_pin_init!(Self { + Ok(try_pin_init!(NovaCore { gpu <- Gpu::new(pdev, bar.clone(), bar.access(pdev.as_ref())?), _reg: auxiliary::Registration::new( pdev.as_ref(), @@ -113,7 +115,7 @@ fn probe<'bound>( }) } - fn unbind<'bound>(pdev: &'bound pci::Device>, this: Pin<&Self>) { + fn unbind<'bound>(pdev: &'bound pci::Device>, this: Pin<&NovaCore>) { this.gpu.unbind(pdev.as_ref()); } } diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 04a1fa6b25f8..073d87714d3a 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -47,7 +47,7 @@ struct NovaCoreModule { // Fields are dropped in declaration order, so `_driver` is dropped first, // then `_debugfs_guard` clears `DEBUGFS_ROOT`. #[pin] - _driver: Registration>, + _driver: Registration>, _debugfs_guard: DebugfsRootGuard, } From e189bdb687a56bcf389798f1d3a2f261fff2ef54 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Mon, 25 May 2026 22:21:09 +0200 Subject: [PATCH 085/131] rust: types: add `ForLt` trait for higher-ranked lifetime support There are a few cases, e.g. when dealing with data referencing each other, one might want to write code that is generic over lifetimes. For example, if you want to take a function that takes `&'a Foo` and gives `Bar<'a>`, you can write: f: impl for<'a> FnOnce(&'a Foo) -> Bar<'a>, However, it becomes tricky when you want that function to not have a fixed `Bar`, but have it be generic again. In this case, one needs something that is generic over types that are themselves generic over lifetimes. `ForLt` provides such support. It provides a trait `ForLt` which describes a type generic over a lifetime. One may use `ForLt::Of<'a>` to get an instance of a type for a specific lifetime. For the case of cross referencing, one would almost always want the lifetime to be covariant. Therefore this is also made a requirement for the `ForLt` trait, so functions with `ForLt` trait bound can assume covariance. A macro `ForLt!()` is provided to be able to obtain a type that implements `ForLt`. For example, `ForLt!(for<'a> Bar<'a>)` would yield a type that `::Of<'a>` is `Bar<'a>`. This also works with lifetime elision, e.g. `ForLt!(Bar<'_>)` or for types without lifetime at all, e.g. `ForLt!(u32)`. The API design draws inspiration from the higher-kinded-types [1] crate, however a different design decision has been taken (e.g. covariance requirement) and the implementation is independent. License headers use "Apache-2.0 OR MIT" because I anticipate this to be used in pin-init crate too which is licensed as such. Link: https://docs.rs/higher-kinded-types/ [1] Reviewed-by: Greg Kroah-Hartman Reviewed-by: Eliot Courtney Signed-off-by: Gary Guo Acked-by: Miguel Ojeda Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260525202921.124698-23-dakr@kernel.org [ Handle macro_rules! invocations in the ForLt! proc macro's covariance and WF checks. Since proc macros cannot expand macro_rules!, add a visit_macro() implementation to conservatively assume macro invocations may contain lifetimes, forcing them through the compiler-assisted covariance proof. Fix a few typos in the documentation and in the commit message, add empty lines before samples, add missing periods and consistently use markdown. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/Makefile | 1 + rust/kernel/types.rs | 4 + rust/kernel/types/for_lt.rs | 122 ++++++++++++++++++ rust/macros/for_lt.rs | 248 ++++++++++++++++++++++++++++++++++++ rust/macros/lib.rs | 13 ++ 5 files changed, 388 insertions(+) create mode 100644 rust/kernel/types/for_lt.rs create mode 100644 rust/macros/for_lt.rs diff --git a/rust/Makefile b/rust/Makefile index b361bfedfdf0..c5a9a3339416 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -110,6 +110,7 @@ syn-cfgs := \ feature="parsing" \ feature="printing" \ feature="proc-macro" \ + feature="visit" \ feature="visit-mut" syn-flags := \ diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 9cf9f869d195..ac316fd7b538 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -11,6 +11,10 @@ }; use pin_init::{PinInit, Wrapper, Zeroable}; +#[doc(hidden)] +pub mod for_lt; +pub use for_lt::ForLt; + /// Used to transfer ownership to and from foreign (non-Rust) languages. /// /// Ownership is transferred from Rust to a foreign language by calling [`Self::into_foreign`] and diff --git a/rust/kernel/types/for_lt.rs b/rust/kernel/types/for_lt.rs new file mode 100644 index 000000000000..d44323c28e8d --- /dev/null +++ b/rust/kernel/types/for_lt.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Provide implementation and test of the `ForLt` trait and macro. +//! +//! This module is hidden and user should just use `ForLt!` directly. + +use core::marker::PhantomData; + +/// Representation of types generic over a lifetime. +/// +/// The type must be covariant over the generic lifetime, i.e. the lifetime parameter +/// can be soundly shortened. +/// +/// The lifetime involved must be covariant. +/// +/// # Macro +/// +/// It is not recommended to implement this trait directly. `ForLt!` macro is provided to obtain a +/// type that implements this trait. +/// +/// The full syntax is +/// +/// ``` +/// # use kernel::types::ForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// ForLt!(for<'a> TypeThatUse<'a>) +/// # >(); +/// ``` +/// +/// which gives a type so that ` TypeThatUse<'a>) as ForLt>::Of<'b>` +/// is `TypeThatUse<'b>`. +/// +/// You may also use a short-hand syntax which works similar to lifetime elision. +/// The macro also accepts types that do not involve a lifetime at all. +/// +/// ``` +/// # use kernel::types::ForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// ForLt!(TypeThatUse<'_>) // Equivalent to `ForLt!(for<'a> TypeThatUse<'a>)`. +/// # >(); +/// # expect_lt::< +/// ForLt!(&u32) // Equivalent to `ForLt!(for<'a> &'a u32)`. +/// # >(); +/// # expect_lt::< +/// ForLt!(u32) // Equivalent to `ForLt!(for<'a> u32)`. +/// # >(); +/// ``` +/// +/// The macro will attempt to prove that the type is indeed covariant over the lifetime supplied. +/// When it cannot be syntactically proven, it will emit checks to ask the Rust compiler to prove +/// it. +/// +/// ```ignore,compile_fail +/// # use kernel::types::ForLt; +/// # fn expect_lt() {} +/// # expect_lt::< +/// ForLt!(fn(&u32)) // Contravariant, will fail compilation. +/// # >(); +/// ``` +/// +/// There is a limitation if the type refers to generic parameters; if the macro cannot prove the +/// covariance syntactically, the emitted checks will fail the compilation as it needs to refer to +/// the generic parameter but is in a separate item. +/// +/// ``` +/// # use kernel::types::ForLt; +/// fn expect_lt() {} +/// # #[allow(clippy::unnecessary_safety_comment, reason = "false positive")] +/// fn generic_fn() { +/// // Syntactically proven by the macro +/// expect_lt::(); +/// // Syntactically proven by the macro +/// expect_lt::)>(); +/// // Cannot be syntactically proven, need to check covariance of `KBox` +/// // expect_lt::)>(); +/// } +/// ``` +/// +/// # Safety +/// +/// `Self::Of<'a>` must be covariant over the lifetime `'a`. +pub unsafe trait ForLt { + /// The type parameterized by the lifetime. + type Of<'a>: 'a; + + /// Cast a reference to a shorter lifetime. + #[inline(always)] + fn cast_ref<'r, 'short: 'r, 'long: 'short>(long: &'r Self::Of<'long>) -> &'r Self::Of<'short> { + // SAFETY: This is sound as this trait guarantees covariance. + unsafe { core::mem::transmute(long) } + } +} +pub use macros::ForLt; + +/// This is intended to be an "unsafe-to-refer-to" type. +/// +/// Must only be used by the `ForLt!` macro. +/// +/// `T` is the magic `dyn for<'a> WithLt<'a, TypeThatUse<'a>>` generated by macro. +/// +/// `WF` is a type that the macro can use to assert some specific type is well-formed. +/// +/// `N` is to provide the macro a place to emit arbitrary items, in case it needs to prove +/// additional properties. +#[doc(hidden)] +pub struct UnsafeForLtImpl(PhantomData<(WF, T)>); + +// This is a helper trait for implementation `ForLt` to be able to use HRTB. +#[doc(hidden)] +pub trait WithLt<'a> { + type Of: 'a; +} + +// SAFETY: In `ForLt!` macro, a covariance proof is generated when naming `UnsafeForLtImpl` +// and it will fail to evaluate if the type is not covariant. +unsafe impl WithLt<'a>, WF> ForLt for UnsafeForLtImpl { + type Of<'a> = >::Of; +} diff --git a/rust/macros/for_lt.rs b/rust/macros/for_lt.rs new file mode 100644 index 000000000000..364d4113cd10 --- /dev/null +++ b/rust/macros/for_lt.rs @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro2::{ + Span, + TokenStream, // +}; +use quote::{ + format_ident, + quote, // +}; +use syn::{ + parse::{ + Parse, + ParseStream, // + }, + visit::Visit, + visit_mut::VisitMut, + Lifetime, + Result, + Token, + Type, // +}; + +pub(crate) enum HigherRankedType { + Explicit { + _for_token: Token![for], + _lt_token: Token![<], + lifetime: Lifetime, + _gt_token: Token![>], + ty: Type, + }, + Implicit { + ty: Type, + }, +} + +impl Parse for HigherRankedType { + fn parse(input: ParseStream<'_>) -> Result { + if input.peek(Token![for]) { + Ok(Self::Explicit { + _for_token: input.parse()?, + _lt_token: input.parse()?, + lifetime: input.parse()?, + _gt_token: input.parse()?, + ty: input.parse()?, + }) + } else { + Ok(Self::Implicit { ty: input.parse()? }) + } + } +} + +trait TypeExt { + fn expand_elided_lifetime(&self, explicit_lt: &Lifetime) -> Type; + fn replace_lifetime(&self, src: &Lifetime, dst: &Lifetime) -> Type; + fn has_lifetime(&self, lt: &Lifetime) -> bool; +} + +impl TypeExt for Type { + fn expand_elided_lifetime(&self, explicit_lt: &Lifetime) -> Type { + struct ElidedLifetimeExpander<'a>(&'a Lifetime); + + impl VisitMut for ElidedLifetimeExpander<'_> { + fn visit_lifetime_mut(&mut self, lifetime: &mut Lifetime) { + // Expand explicit `'_` + if lifetime.ident == "_" { + *lifetime = self.0.clone(); + } + } + + fn visit_type_reference_mut(&mut self, reference: &mut syn::TypeReference) { + syn::visit_mut::visit_type_reference_mut(self, reference); + + if reference.lifetime.is_none() { + reference.lifetime = Some(self.0.clone()); + } + } + } + + let mut ret = self.clone(); + ElidedLifetimeExpander(explicit_lt).visit_type_mut(&mut ret); + ret + } + + fn replace_lifetime(&self, src: &Lifetime, dst: &Lifetime) -> Type { + struct LifetimeReplacer<'a>(&'a Lifetime, &'a Lifetime); + + impl VisitMut for LifetimeReplacer<'_> { + fn visit_lifetime_mut(&mut self, lifetime: &mut Lifetime) { + if lifetime.ident == self.0.ident { + *lifetime = self.1.clone(); + } + } + } + + let mut ret = self.clone(); + LifetimeReplacer(src, dst).visit_type_mut(&mut ret); + ret + } + + fn has_lifetime(&self, lt: &Lifetime) -> bool { + struct HasLifetime<'a>(&'a Lifetime, bool); + + impl Visit<'_> for HasLifetime<'_> { + fn visit_lifetime(&mut self, lifetime: &Lifetime) { + if lifetime.ident == self.0.ident { + self.1 = true; + } + } + + // Macro invocations are opaque; conservatively assume they may + // reference the lifetime. + fn visit_macro(&mut self, _: &syn::Macro) { + self.1 = true; + } + } + + let mut visitor = HasLifetime(lt, false); + visitor.visit_type(self); + visitor.1 + } +} + +struct Prover<'a>(&'a Lifetime, Vec<&'a Type>); + +impl<'a> Prover<'a> { + /// Prove that `ty` is covariant over `'lt`. + /// + /// This also needs to prove that it'll be wellformed for any instance of `'lt`. + /// It can be assumed that `ty` will be wellformed if `'lt` is substituted to `'static`. + fn prove(&mut self, ty: &'a Type) { + match ty { + Type::Paren(ty) => self.prove(&ty.elem), + Type::Group(ty) => self.prove(&ty.elem), + + // No lifetime involved + Type::Never(_) => {} + + // `[T; N]` and `[T]` is covariant over `T`. + Type::Array(ty) => self.prove(&ty.elem), + Type::Slice(ty) => self.prove(&ty.elem), + + Type::Tuple(ty) => { + for elem in &ty.elems { + self.prove(elem); + } + } + + // `*const T` is covariant over `T` + Type::Ptr(ty) if ty.const_token.is_some() => self.prove(&ty.elem), + + // `&T` is covariant over `T` and lifetime. + // + // Note that if we encounter `&'other_lt T`, then we still need to make sure the type + // is wellformed if `T` involves `&'lt`, so we defer to the compiler. + // + // This is to block cases like `ForLt!(for<'a> &'static &'a u32)`, as the presence of + // the type implies `'a: 'static` but this is unsound. + Type::Reference(ty) + if ty.mutability.is_none() && ty.lifetime.as_ref() == Some(self.0) => + { + self.prove(&ty.elem) + } + + // `&[mut] T` is covariant over lifetime. + // In case we have `&[mut] NoLifetime`, we don't need to do additional checks. + Type::Reference(ty) if !ty.elem.has_lifetime(self.0) => (), + + // No mention of lifetime at all, no need to perform compiler check. + ty if !ty.has_lifetime(self.0) => (), + + // Otherwise, we need to emit checks so that compiler can determine if the types are + // actually covariant. + ty => self.1.push(ty), + } + } +} + +pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { + let (ty, lifetime) = match input { + HigherRankedType::Explicit { lifetime, ty, .. } => (ty, lifetime), + HigherRankedType::Implicit { ty } => { + // If there's no explicit `for<'a>` binder, inject a synthetic `'__elided` lifetime + // and expand elided sites. + let lifetime = Lifetime { + apostrophe: Span::mixed_site(), + ident: format_ident!("__elided", span = Span::mixed_site()), + }; + (ty.expand_elided_lifetime(&lifetime), lifetime) + } + }; + + let mut prover = Prover(&lifetime, Vec::new()); + prover.prove(&ty); + + let mut proof = Vec::new(); + + // Emit proofs for every type that requires additional compiler help in proving covariance. + for (idx, required_proof) in prover.1.into_iter().enumerate() { + // Insert a proof that the type is well-formed. + // + // This is intended to workaround a Rust compiler soundness bug related to HRTB. + // https://github.com/rust-lang/rust/issues/152489 + // + // This needs to be a struct instead of fn to avoid the implied WF bounds. + let wf_proof_name = format_ident!("ProveWf{idx}"); + proof.push(quote!( + struct #wf_proof_name<#lifetime>( + ::core::marker::PhantomData<&#lifetime ()>, #required_proof + ); + )); + + // Insert a proof that the type is covariant. + let cov_proof_name = format_ident!("prove_covariant_{idx}"); + proof.push(quote!( + fn #cov_proof_name<'__short, '__long: '__short>( + long: #wf_proof_name<'__long> + ) -> #wf_proof_name<'__short> { + long + } + )); + } + + // Make sure that the type is wellformed when substituting lifetime with `'static`. + // + // Currently the Rust compiler doesn't check this, see the above `ProveWf` documentation. + // + // We prefer to use this way of proving WF-ness as it can work when generics are involved. + let ty_static = ty.replace_lifetime( + &lifetime, + &Lifetime { + apostrophe: Span::mixed_site(), + ident: format_ident!("static"), + }, + ); + + quote!( + ::kernel::types::for_lt::UnsafeForLtImpl::< + dyn for<#lifetime> ::kernel::types::for_lt::WithLt<#lifetime, Of = #ty>, + #ty_static, + { + #(#proof)* + + 0 + } + > + ) +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 2cfd59e0f9e7..4a48fabbc268 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -17,6 +17,7 @@ mod concat_idents; mod export; mod fmt; +mod for_lt; mod helpers; mod kunit; mod module; @@ -489,3 +490,15 @@ pub fn kunit_tests(attr: TokenStream, input: TokenStream) -> TokenStream { .unwrap_or_else(|e| e.into_compile_error()) .into() } + +/// Obtain a type that implements [`ForLt`] for the given higher-ranked type. +/// +/// Please refer to the documentation of the [`ForLt`] trait. +/// +/// [`ForLt`]: trait.ForLt.html +#[proc_macro] +// The macro shares the name with the trait. +#[allow(non_snake_case)] +pub fn ForLt(input: TokenStream) -> TokenStream { + for_lt::for_lt(parse_macro_input!(input)).into() +} From 4555291ddae9abe2c40a7eae192b1976b07a1fad Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:10 +0200 Subject: [PATCH 086/131] rust: auxiliary: generalize Registration over ForLt Generalize Registration to Registration and Device::registration_data() to return Pin<&F::Of<'_>>. The stored 'static lifetime is shortened to the borrow lifetime of &self via ForLt::cast_ref; ForLt's covariance guarantee makes this sound. Reviewed-by: Greg Kroah-Hartman Reviewed-by: Alexandre Courbot Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260525202921.124698-24-dakr@kernel.org [ Use PhantomData> instead of PhantomData<(fn(&'a ()) -> &'a (), F)>], which also gets us rid of #[allow(clippy::type_complexity)]. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 13 ++-- rust/kernel/auxiliary.rs | 107 ++++++++++++++++++-------- samples/rust/rust_driver_auxiliary.rs | 19 +++-- 3 files changed, 95 insertions(+), 44 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index fa898fe5c893..d3f2245ba2e0 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -3,7 +3,6 @@ use kernel::{ auxiliary, device::Core, - devres::Devres, dma::Device, dma::DmaMask, pci, @@ -21,6 +20,7 @@ }, Arc, }, + types::ForLt, }; use crate::gpu::Gpu; @@ -29,10 +29,11 @@ static AUXILIARY_ID_COUNTER: Atomic = Atomic::new(0); #[pin_data] -pub(crate) struct NovaCore { +pub(crate) struct NovaCore<'bound> { #[pin] pub(crate) gpu: Gpu, - _reg: Devres>, + #[allow(clippy::type_complexity)] + _reg: auxiliary::Registration<'bound, ForLt!(())>, } pub(crate) struct NovaCoreDriver; @@ -76,13 +77,13 @@ pub(crate) struct NovaCore { impl pci::Driver for NovaCoreDriver { type IdInfo = (); - type Data<'bound> = NovaCore; + type Data<'bound> = NovaCore<'bound>; const ID_TABLE: pci::IdTable = &PCI_TABLE; fn probe<'bound>( pdev: &'bound pci::Device>, _info: &'bound Self::IdInfo, - ) -> impl PinInit + 'bound { + ) -> impl PinInit, Error> + 'bound { pin_init::pin_init_scope(move || { dev_dbg!(pdev, "Probe Nova Core GPU driver.\n"); @@ -115,7 +116,7 @@ fn probe<'bound>( }) } - fn unbind<'bound>(pdev: &'bound pci::Device>, this: Pin<&NovaCore>) { + fn unbind<'bound>(pdev: &'bound pci::Device>, this: Pin<&Self::Data<'bound>>) { this.gpu.unbind(pdev.as_ref()); } } diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index 7a1b1a7b7ca6..c42928d5a239 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -12,7 +12,7 @@ RawDeviceId, RawDeviceIdIndex, // }, - devres::Devres, + driver, error::{ from_result, @@ -20,6 +20,7 @@ }, prelude::*, types::{ + ForLt, ForeignOwnable, Opaque, // }, @@ -271,12 +272,16 @@ pub fn parent(&self) -> &device::Device { /// Returns a pinned reference to the registration data set by the registering (parent) driver. /// - /// Returns [`EINVAL`] if `T` does not match the type used by the parent driver when calling + /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The returned + /// reference has its lifetime shortened from `'static` to `&self`'s borrow lifetime via + /// [`ForLt::cast_ref`]. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling /// [`Registration::new()`]. /// /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was /// registered by a C driver. - pub fn registration_data(&self) -> Result> { + pub fn registration_data(&self) -> Result>> { // SAFETY: By the type invariant, `self.as_raw()` is a valid `struct auxiliary_device`. let ptr = unsafe { (*self.as_raw()).registration_data_rust }; if ptr.is_null() { @@ -289,18 +294,23 @@ pub fn registration_data(&self) -> Result> { // SAFETY: `ptr` is non-null and was set via `into_foreign()` in `Registration::new()`; // `RegistrationData` is `#[repr(C)]` with `type_id` at offset 0, so reading a `TypeId` - // at the start of the allocation is valid regardless of `T`. + // at the start of the allocation is valid regardless of `F`. let type_id = unsafe { ptr.cast::().read() }; - if type_id != TypeId::of::() { + if type_id != TypeId::of::() { return Err(EINVAL); } - // SAFETY: The `TypeId` check above confirms that the stored type is `T`; `ptr` remains - // valid until `Registration::drop()` calls `from_foreign()`. - let wrapper = unsafe { Pin::>>::borrow(ptr) }; + // SAFETY: The `TypeId` check above confirms that the stored type matches + // `F::Of<'static>`; `ptr` remains valid until `Registration::drop()` calls + // `from_foreign()`. + let wrapper = unsafe { Pin::>>>::borrow(ptr) }; // SAFETY: `data` is a structurally pinned field of `RegistrationData`. - Ok(unsafe { wrapper.map_unchecked(|w| &w.data) }) + let pinned: Pin<&F::Of<'_>> = unsafe { wrapper.map_unchecked(|w| &w.data) }; + + // SAFETY: The data was pinned when stored; `cast_ref` only shortens + // the lifetime, so the pinning guarantee is preserved. + Ok(unsafe { Pin::new_unchecked(F::cast_ref(pinned.get_ref())) }) } } @@ -389,43 +399,60 @@ struct RegistrationData { /// This type represents the registration of a [`struct auxiliary_device`]. When its parent device /// is unbound, the corresponding auxiliary device will be unregistered from the system. /// -/// The type parameter `T` is the type of the registration data owned by the registering (parent) -/// driver. It can be accessed by the auxiliary driver through -/// [`Device::registration_data()`]. +/// The type parameter `F` is a [`ForLt`](trait@ForLt) encoding of the registration +/// data type. For non-lifetime-parameterized types, use [`ForLt!(T)`](macro@ForLt). +/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`]. /// /// # Invariants /// /// `self.adev` always holds a valid pointer to an initialized and registered /// [`struct auxiliary_device`] whose `registration_data_rust` field points to a -/// valid `Pin>>`. -pub struct Registration { +/// valid `Pin>>>`. +pub struct Registration<'a, F: ForLt + 'static> { adev: NonNull, - _data: PhantomData, + _phantom: PhantomData>, } -impl Registration { +impl<'a, F: ForLt> Registration<'a, F> +where + for<'b> F::Of<'b>: Send + Sync, +{ /// Create and register a new auxiliary device with the given registration data. /// /// The `data` is owned by the registration and can be accessed through the auxiliary device /// via [`Device::registration_data()`]. - pub fn new( - parent: &device::Device, + /// + /// # Safety + /// + /// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since the registration data may contain borrowed + /// references that become invalid after `'a` ends. + /// + /// If the registration data is `'static`, use the safe [`Registration::new()`] instead. + pub unsafe fn new_with_lt( + parent: &'a device::Device, name: &CStr, id: u32, modname: &CStr, - data: impl PinInit, - ) -> Result> + data: impl PinInit, E>, + ) -> Result where Error: From, { let data = KBox::pin_init::( try_pin_init!(RegistrationData { - type_id: TypeId::of::(), + type_id: TypeId::of::(), data <- data, }), GFP_KERNEL, )?; + // SAFETY: `'a` is invariant (via `Registration`'s `PhantomData`). Lifetimes do not + // affect layout, so RegistrationData> and RegistrationData> + // have identical representation. + let data: Pin>>> = + unsafe { core::mem::transmute(data) }; + let boxed: KBox> = KBox::zeroed(GFP_KERNEL)?; let adev = boxed.get(); @@ -455,7 +482,9 @@ pub fn new( if ret != 0 { // SAFETY: `registration_data` was set above via `into_foreign()`. drop(unsafe { - Pin::>>::from_foreign((*adev).registration_data_rust) + Pin::>>>::from_foreign( + (*adev).registration_data_rust, + ) }); // SAFETY: `adev` is guaranteed to be a valid pointer to a @@ -467,18 +496,36 @@ pub fn new( // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is // called, which happens in `Self::drop()`. - let reg = Self { + Ok(Self { // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated // successfully. adev: unsafe { NonNull::new_unchecked(adev) }, - _data: PhantomData, - }; + _phantom: PhantomData, + }) + } - Devres::new::(parent, reg) + /// Create and register a new auxiliary device with `'static` registration data. + /// + /// Safe variant of [`Registration::new_with_lt()`] for registration data that does not contain + /// borrowed references. + pub fn new( + parent: &'a device::Device, + name: &CStr, + id: u32, + modname: &CStr, + data: impl PinInit, E>, + ) -> Result + where + F::Of<'a>: 'static, + Error: From, + { + // SAFETY: `F::Of<'a>: 'static` guarantees the data contains no borrowed references, + // so forgetting the `Registration` cannot cause use-after-free. + unsafe { Self::new_with_lt(parent, name, id, modname, data) } } } -impl Drop for Registration { +impl Drop for Registration<'_, F> { fn drop(&mut self) { // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered // `struct auxiliary_device`. @@ -486,7 +533,7 @@ fn drop(&mut self) { // SAFETY: `registration_data` was set in `new()` via `into_foreign()`. drop(unsafe { - Pin::>>::from_foreign( + Pin::>>>::from_foreign( (*self.adev.as_ptr()).registration_data_rust, ) }); @@ -500,7 +547,7 @@ fn drop(&mut self) { } // SAFETY: A `Registration` of a `struct auxiliary_device` can be released from any thread. -unsafe impl Send for Registration {} +unsafe impl Send for Registration<'_, F> where for<'a> F::Of<'a>: Send {} // SAFETY: `Registration` does not expose any methods or fields that need synchronization. -unsafe impl Sync for Registration {} +unsafe impl Sync for Registration<'_, F> where for<'a> F::Of<'a>: Send {} diff --git a/samples/rust/rust_driver_auxiliary.rs b/samples/rust/rust_driver_auxiliary.rs index b30a4d5cdf8a..e3e811a14110 100644 --- a/samples/rust/rust_driver_auxiliary.rs +++ b/samples/rust/rust_driver_auxiliary.rs @@ -10,10 +10,10 @@ Bound, Core, // }, - devres::Devres, driver, pci, prelude::*, + types::ForLt, InPlaceModule, // }; @@ -55,9 +55,12 @@ struct Data { index: u32, } -struct ParentDriver { - _reg0: Devres>, - _reg1: Devres>, +struct ParentDriver; + +#[allow(clippy::type_complexity)] +struct ParentData<'bound> { + _reg0: auxiliary::Registration<'bound, ForLt!(Data)>, + _reg1: auxiliary::Registration<'bound, ForLt!(Data)>, } kernel::pci_device_table!( @@ -69,15 +72,15 @@ struct ParentDriver { impl pci::Driver for ParentDriver { type IdInfo = (); - type Data<'bound> = Self; + type Data<'bound> = ParentData<'bound>; const ID_TABLE: pci::IdTable = &PCI_TABLE; fn probe<'bound>( pdev: &'bound pci::Device>, _info: &'bound Self::IdInfo, - ) -> impl PinInit + 'bound { - Ok(Self { + ) -> impl PinInit, Error> + 'bound { + Ok(ParentData { _reg0: auxiliary::Registration::new( pdev.as_ref(), AUXILIARY_NAME, @@ -101,7 +104,7 @@ fn connect(adev: &auxiliary::Device) -> Result { let dev = adev.parent(); let pdev: &pci::Device = dev.try_into()?; - let data = adev.registration_data::()?; + let data = adev.registration_data::()?; dev_info!( dev, From d18f3646184fc805d213fc049fc3b5d9fb9a6a27 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Mon, 25 May 2026 22:21:11 +0200 Subject: [PATCH 087/131] samples: rust: rust_driver_auxiliary: showcase lifetime-bound registration data Make the Data struct lifetime-parameterized, storing a reference to the parent pci::Device. This demonstrates that registration data can hold device resources tied to the parent driver's lifetime. In connect(), retrieve the parent PCI device from the registration data rather than casting through adev.parent(). Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Reviewed-by: Greg Kroah-Hartman Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260525202921.124698-25-dakr@kernel.org Signed-off-by: Danilo Krummrich --- samples/rust/rust_driver_auxiliary.rs | 58 ++++++++++++++++----------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/samples/rust/rust_driver_auxiliary.rs b/samples/rust/rust_driver_auxiliary.rs index e3e811a14110..2c1351040e45 100644 --- a/samples/rust/rust_driver_auxiliary.rs +++ b/samples/rust/rust_driver_auxiliary.rs @@ -51,16 +51,17 @@ fn probe<'bound>( } } -struct Data { +struct Data<'bound> { index: u32, + parent: &'bound pci::Device, } struct ParentDriver; #[allow(clippy::type_complexity)] struct ParentData<'bound> { - _reg0: auxiliary::Registration<'bound, ForLt!(Data)>, - _reg1: auxiliary::Registration<'bound, ForLt!(Data)>, + _reg0: auxiliary::Registration<'bound, ForLt!(Data<'_>)>, + _reg1: auxiliary::Registration<'bound, ForLt!(Data<'_>)>, } kernel::pci_device_table!( @@ -81,33 +82,44 @@ fn probe<'bound>( _info: &'bound Self::IdInfo, ) -> impl PinInit, Error> + 'bound { Ok(ParentData { - _reg0: auxiliary::Registration::new( - pdev.as_ref(), - AUXILIARY_NAME, - 0, - MODULE_NAME, - Data { index: 0 }, - )?, - _reg1: auxiliary::Registration::new( - pdev.as_ref(), - AUXILIARY_NAME, - 1, - MODULE_NAME, - Data { index: 1 }, - )?, + // SAFETY: `ParentData` is the driver's private data, which is dropped when the + // device is unbound; i.e. `mem::forget()` is never called on it. + _reg0: unsafe { + auxiliary::Registration::new_with_lt( + pdev.as_ref(), + AUXILIARY_NAME, + 0, + MODULE_NAME, + Data { + index: 0, + parent: pdev, + }, + )? + }, + // SAFETY: See `_reg0` above. + _reg1: unsafe { + auxiliary::Registration::new_with_lt( + pdev.as_ref(), + AUXILIARY_NAME, + 1, + MODULE_NAME, + Data { + index: 1, + parent: pdev, + }, + )? + }, }) } } impl ParentDriver { fn connect(adev: &auxiliary::Device) -> Result { - let dev = adev.parent(); - let pdev: &pci::Device = dev.try_into()?; - - let data = adev.registration_data::()?; + let data = adev.registration_data::)>()?; + let pdev = data.parent; dev_info!( - dev, + pdev, "Connect auxiliary {} with parent: VendorID={}, DeviceID={:#x}\n", adev.id(), pdev.vendor_id(), @@ -115,7 +127,7 @@ fn connect(adev: &auxiliary::Device) -> Result { ); dev_info!( - dev, + pdev, "Connected to auxiliary device with index {}.\n", data.index ); From 9c81596851a342e68ec200f69fc1d5c1dbede289 Mon Sep 17 00:00:00 2001 From: Laura Nao Date: Thu, 7 May 2026 10:09:14 +0200 Subject: [PATCH 088/131] rust: drm: add FEAT_RENDER flag for render node support Add FEAT_RENDER bool constant to the Driver trait to control render node support. When enabled, the driver exposes /dev/dri/renderDXX render nodes to userspace. The flag defaults to false, drivers can opt in by setting it to true in their Driver implementation. This is then enabled in the Tyr driver, while it's left disabled for Nova for the time being. Co-developed-by: Daniel Almeida Signed-off-by: Daniel Almeida Reviewed-by: Alice Ryhl Signed-off-by: Laura Nao Link: https://patch.msgid.link/20260507080914.95478-2-laura.nao@collabora.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/tyr/driver.rs | 1 + rust/kernel/drm/device.rs | 12 +++++++++++- rust/kernel/drm/driver.rs | 12 ++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index e20a5978eed6..f0c2d6e4db3a 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -186,6 +186,7 @@ impl drm::Driver for TyrDrmDriver { type Object = drm::gem::shmem::Object; const INFO: drm::DriverInfo = INFO; + const FEAT_RENDER: bool = true; kernel::declare_drm_ioctls! { (PANTHOR_DEV_QUERY, drm_panthor_dev_query, ioctl::RENDER_ALLOW, TyrDrmFileData::dev_query), diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 403fc35353c7..dc16738eeb38 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -80,6 +80,16 @@ pub struct Device { } impl Device { + const fn compute_features() -> u32 { + let mut features = drm::driver::FEAT_GEM; + + if T::FEAT_RENDER { + features |= drm::driver::FEAT_RENDER; + } + + features + } + const VTABLE: bindings::drm_driver = drm_legacy_fields! { load: None, open: Some(drm::File::::open_callback), @@ -105,7 +115,7 @@ impl Device { name: crate::str::as_char_ptr_in_const_context(T::INFO.name).cast_mut(), desc: crate::str::as_char_ptr_in_const_context(T::INFO.desc).cast_mut(), - driver_features: drm::driver::FEAT_GEM, + driver_features: Self::compute_features(), ioctls: T::IOCTLS.as_ptr(), num_ioctls: T::IOCTLS.len() as i32, fops: &Self::GEM_FOPS, diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 5233bdebc9fc..6886396c8fa7 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -16,6 +16,8 @@ /// Driver use the GEM memory manager. This should be set for all modern drivers. pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +/// Driver supports render nodes, i.e.: /dev/dri/renderDXX devices. +pub(crate) const FEAT_RENDER: u32 = bindings::drm_driver_feature_DRIVER_RENDER; /// Information data for a DRM Driver. pub struct DriverInfo { @@ -115,6 +117,16 @@ pub trait Driver { /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; + + /// Sets the `DRIVER_RENDER` feature for this driver. + /// + /// When enabled, the driver exposes `/dev/dri/renderDXX` render nodes to + /// userspace. The render node is an alternate low-priviledge way to access + /// the driver, which is enforced on a per-ioctl level. Userspace processes + /// that open the render node can only invoke ioctls explicitly listed as + /// usable from the render node (i.e. marked DRM_RENDER_ALLOW), whereas + /// userspace processes using the master node can invoke any ioctl. + const FEAT_RENDER: bool = false; } /// The registration type of a `drm::Device`. From 88b70f5a05afabe6e176469919986282225036ee Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 26 May 2026 00:58:29 +0200 Subject: [PATCH 089/131] gpu: nova-core: use lifetime for Bar Take advantage of the lifetime-parameterized pci::Bar<'bound> to hold the BAR mapping directly in NovaCore<'bound>, and pass a borrowed reference to Gpu<'bound>. This eliminates the Arc> indirection, removes runtime revocation checks for BAR access, and simplifies Gpu::unbind(). Reviewed-by: Eliot Courtney Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260525225838.276108-2-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 32 +++++++++++++++----------------- drivers/gpu/nova-core/gpu.rs | 33 +++++++++++++-------------------- 2 files changed, 28 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index d3f2245ba2e0..d4cf4379ee87 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -13,12 +13,9 @@ }, prelude::*, sizes::SZ_16M, - sync::{ - atomic::{ - Atomic, - Relaxed, // - }, - Arc, + sync::atomic::{ + Atomic, + Relaxed, // }, types::ForLt, }; @@ -31,7 +28,8 @@ #[pin_data] pub(crate) struct NovaCore<'bound> { #[pin] - pub(crate) gpu: Gpu, + pub(crate) gpu: Gpu<'bound>, + bar: pci::Bar<'bound, BAR0_SIZE>, #[allow(clippy::type_complexity)] _reg: auxiliary::Registration<'bound, ForLt!(())>, } @@ -48,7 +46,7 @@ pub(crate) struct NovaCore<'bound> { // DMA addresses. These systems should be quite rare. const GPU_DMA_BITS: u32 = 47; -pub(crate) type Bar0 = pci::Bar<'static, BAR0_SIZE>; +pub(crate) type Bar0 = kernel::io::Mmio; kernel::pci_device_table!( PCI_TABLE, @@ -95,14 +93,14 @@ fn probe<'bound>( // other threads of execution. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::())? }; - let bar = Arc::new( - pdev.iomap_region_sized::(0, c"nova-core/bar0")? - .into_devres()?, - GFP_KERNEL, - )?; - Ok(try_pin_init!(NovaCore { - gpu <- Gpu::new(pdev, bar.clone(), bar.access(pdev.as_ref())?), + bar: pdev.iomap_region_sized::(0, c"nova-core/bar0")?, + // TODO: Use `&bar` self-referential pin-init syntax once available. + // + // SAFETY: `bar` is initialized before this expression is evaluated + // (`try_pin_init!()` initializes fields in declaration order), lives at a pinned + // stable address, and is dropped after `gpu` (struct field drop order). + gpu <- Gpu::new(pdev, unsafe { &*core::ptr::from_ref(bar) }), _reg: auxiliary::Registration::new( pdev.as_ref(), c"nova-drm", @@ -116,7 +114,7 @@ fn probe<'bound>( }) } - fn unbind<'bound>(pdev: &'bound pci::Device>, this: Pin<&Self::Data<'bound>>) { - this.gpu.unbind(pdev.as_ref()); + fn unbind<'bound>(_pdev: &'bound pci::Device>, this: Pin<&Self::Data<'bound>>) { + this.gpu.unbind(); } } diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 78c4195e6132..108dd094e354 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -2,13 +2,11 @@ use kernel::{ device, - devres::Devres, fmt, io::Io, num::Bounded, pci, - prelude::*, - sync::Arc, // + prelude::*, // }; use crate::{ @@ -246,10 +244,10 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { /// Structure holding the resources required to operate the GPU. #[pin_data] -pub(crate) struct Gpu { +pub(crate) struct Gpu<'gpu> { spec: Spec, - /// MMIO mapping of PCI BAR 0 - bar: Arc>, + /// MMIO mapping of PCI BAR 0. + bar: &'gpu Bar0, /// System memory page required for flushing all pending GPU-side memory writes done through /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). sysmem_flush: SysmemFlush, @@ -262,12 +260,11 @@ pub(crate) struct Gpu { gsp: Gsp, } -impl Gpu { - pub(crate) fn new<'a>( - pdev: &'a pci::Device, - devres_bar: Arc>, - bar: &'a Bar0, - ) -> impl PinInit + 'a { +impl<'gpu> Gpu<'gpu> { + pub(crate) fn new( + pdev: &'gpu pci::Device, + bar: &'gpu Bar0, + ) -> impl PinInit + 'gpu { try_pin_init!(Self { spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { dev_info!(pdev,"NVIDIA ({})\n", spec); @@ -279,6 +276,8 @@ pub(crate) fn new<'a>( .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?; }, + bar, + sysmem_flush: SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?, gsp_falcon: Falcon::new( @@ -292,19 +291,13 @@ pub(crate) fn new<'a>( gsp <- Gsp::new(pdev), _: { gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)? }, - - bar: devres_bar, }) } /// Called when the corresponding [`Device`](device::Device) is unbound. /// /// Note: This method must only be called from `Driver::unbind`. - pub(crate) fn unbind(&self, dev: &device::Device>) { - kernel::warn_on!(self - .bar - .access(dev) - .inspect(|bar| self.sysmem_flush.unregister(bar)) - .is_err()); + pub(crate) fn unbind(&self) { + self.sysmem_flush.unregister(self.bar); } } From d8143c9e988be01c9471f1cf0551208992d49918 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 26 May 2026 00:58:30 +0200 Subject: [PATCH 090/131] gpu: nova-core: unregister sysmem flush page from Drop Now that SysmemFlush can borrow the Bar via HRT lifetime, store a &'bound Bar0 reference and implement Drop to automatically unregister the sysmem flush page. This removes the need for manual unregister() calls and the Gpu::unbind() method. Reported-by: Eliot Courtney Closes: https://lore.kernel.org/all/20260409-fix-systemflush-v1-1-a1d6c968f17c@nvidia.com/ Fixes: 6554ad65b589 ("gpu: nova-core: register sysmem flush page") Reviewed-by: Eliot Courtney Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260525225838.276108-3-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 4 ---- drivers/gpu/nova-core/fb.rs | 22 ++++++++++------------ drivers/gpu/nova-core/gpu.rs | 9 +-------- 3 files changed, 11 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index d4cf4379ee87..cff5034c2dcd 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -113,8 +113,4 @@ fn probe<'bound>( })) }) } - - fn unbind<'bound>(_pdev: &'bound pci::Device>, this: Pin<&Self::Data<'bound>>) { - this.gpu.unbind(); - } } diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 6ee87050ce69..3b3271790cc9 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -43,21 +43,20 @@ /// Because of this, the sysmem flush memory page must be registered as early as possible during /// driver initialization, and before any falcon is reset. /// -/// Users are responsible for manually calling [`Self::unregister`] before dropping this object, -/// otherwise the GPU might still use it even after it has been freed. -pub(crate) struct SysmemFlush { +pub(crate) struct SysmemFlush<'sys> { /// Chipset we are operating on. chipset: Chipset, device: ARef, + bar: &'sys Bar0, /// Keep the page alive as long as we need it. page: CoherentHandle, } -impl SysmemFlush { +impl<'sys> SysmemFlush<'sys> { /// Allocate a memory page and register it as the sysmem flush page. pub(crate) fn register( dev: &device::Device, - bar: &Bar0, + bar: &'sys Bar0, chipset: Chipset, ) -> Result { let page = CoherentHandle::alloc(dev, kernel::page::PAGE_SIZE, GFP_KERNEL)?; @@ -67,19 +66,18 @@ pub(crate) fn register( Ok(Self { chipset, device: dev.into(), + bar, page, }) } +} - /// Unregister the managed sysmem flush page. - /// - /// In order to gracefully tear down the GPU, users must make sure to call this method before - /// dropping the object. - pub(crate) fn unregister(&self, bar: &Bar0) { +impl Drop for SysmemFlush<'_> { + fn drop(&mut self) { let hal = hal::fb_hal(self.chipset); - if hal.read_sysmem_flush_page(bar) == self.page.dma_handle() { - let _ = hal.write_sysmem_flush_page(bar, 0).inspect_err(|e| { + if hal.read_sysmem_flush_page(self.bar) == self.page.dma_handle() { + let _ = hal.write_sysmem_flush_page(self.bar, 0).inspect_err(|e| { dev_warn!( &self.device, "failed to unregister sysmem flush page: {:?}\n", diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 108dd094e354..cf134cab49cd 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -250,7 +250,7 @@ pub(crate) struct Gpu<'gpu> { bar: &'gpu Bar0, /// System memory page required for flushing all pending GPU-side memory writes done through /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). - sysmem_flush: SysmemFlush, + sysmem_flush: SysmemFlush<'gpu>, /// GSP falcon instance, used for GSP boot up and cleanup. gsp_falcon: Falcon, /// SEC2 falcon instance, used for GSP boot up and cleanup. @@ -293,11 +293,4 @@ pub(crate) fn new( _: { gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)? }, }) } - - /// Called when the corresponding [`Device`](device::Device) is unbound. - /// - /// Note: This method must only be called from `Driver::unbind`. - pub(crate) fn unbind(&self) { - self.sysmem_flush.unregister(self.bar); - } } From 17b6544ec14198b53bcd546f891b3b22d0a015b2 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 26 May 2026 00:58:31 +0200 Subject: [PATCH 091/131] gpu: nova-core: replace ARef with &'bound Device in SysmemFlush Now that SysmemFlush is lifetime-parameterized, the ARef is unnecessary -- a plain &'bound Device reference suffices. Reviewed-by: Eliot Courtney Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260525225838.276108-4-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/fb.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 3b3271790cc9..1fb65d4eb290 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -15,8 +15,7 @@ Alignable, Alignment, // }, - sizes::*, - sync::aref::ARef, // + sizes::*, // }; use crate::{ @@ -46,7 +45,7 @@ pub(crate) struct SysmemFlush<'sys> { /// Chipset we are operating on. chipset: Chipset, - device: ARef, + device: &'sys device::Device, bar: &'sys Bar0, /// Keep the page alive as long as we need it. page: CoherentHandle, @@ -55,7 +54,7 @@ pub(crate) struct SysmemFlush<'sys> { impl<'sys> SysmemFlush<'sys> { /// Allocate a memory page and register it as the sysmem flush page. pub(crate) fn register( - dev: &device::Device, + dev: &'sys device::Device, bar: &'sys Bar0, chipset: Chipset, ) -> Result { @@ -65,7 +64,7 @@ pub(crate) fn register( Ok(Self { chipset, - device: dev.into(), + device: dev, bar, page, }) From 859805f070bb4e8a7f42f389db6e745850e7e07e Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 26 May 2026 00:58:32 +0200 Subject: [PATCH 092/131] gpu: nova-core: gsp: replace ARef with &'a Device in sequencer GspSequencer, GspSeqIter, and GspSequencerParams are already lifetime-parameterized; the ARef is unnecessary -- a plain &'a Device reference suffices. Reviewed-by: Eliot Courtney Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260525225838.276108-5-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp/boot.rs | 2 +- drivers/gpu/nova-core/gsp/sequencer.rs | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index df105ef4b371..3e8f9611d2b3 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -234,7 +234,7 @@ pub(crate) fn boot( libos_dma_handle: libos_handle, gsp_falcon, sec2_falcon, - dev: pdev.as_ref().into(), + dev, bar, }; GspSequencer::run(&self.cmdq, seq_params)?; diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index 474e4c8021db..b3015483ed17 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -11,7 +11,6 @@ Io, // }, prelude::*, - sync::aref::ARef, time::{ delay::fsleep, Delta, // @@ -142,7 +141,7 @@ pub(crate) struct GspSequencer<'a> { /// Bootloader application version. bootloader_app_version: u32, /// Device for logging. - dev: ARef, + dev: &'a device::Device, } impl fw::RegWritePayload { @@ -281,7 +280,7 @@ pub(crate) struct GspSeqIter<'a> { /// Number of commands processed so far. cmds_processed: u32, /// Device for logging. - dev: ARef, + dev: &'a device::Device, } impl<'a> Iterator for GspSeqIter<'a> { @@ -309,7 +308,7 @@ fn next(&mut self) -> Option { self.cmd_data.len() - offset }; buffer[..copy_len].copy_from_slice(&self.cmd_data[offset..offset + copy_len]); - let cmd_result = GspSeqCmd::new(&buffer, &self.dev); + let cmd_result = GspSeqCmd::new(&buffer, self.dev); cmd_result.map_or_else( |_err| { @@ -334,7 +333,7 @@ fn iter(&self) -> GspSeqIter<'_> { current_offset: 0, total_cmds: self.seq_info.cmd_index, cmds_processed: 0, - dev: self.dev.clone(), + dev: self.dev, } } } @@ -350,7 +349,7 @@ pub(crate) struct GspSequencerParams<'a> { /// SEC2 falcon for core operations. pub(crate) sec2_falcon: &'a Falcon, /// Device for logging. - pub(crate) dev: ARef, + pub(crate) dev: &'a device::Device, /// BAR0 for register access. pub(crate) bar: &'a Bar0, } From dc395c2831b59a90b4605ef38e6c6ef83cf8cc4f Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Tue, 26 May 2026 00:58:33 +0200 Subject: [PATCH 093/131] gpu: nova: separate driver type from driver data Split NovaDriver into a unit struct for trait implementations and a separate Nova struct for the private driver data. Reviewed-by: Eliot Courtney Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260525225838.276108-6-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index e4765bc5b3ec..4289df7de01c 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -15,9 +15,11 @@ use crate::file::File; use crate::gem::NovaObject; -pub(crate) struct NovaDriver { +pub(crate) struct NovaDriver; + +pub(crate) struct Nova { #[expect(unused)] - drm: ARef>, + drm: ARef>, } /// Convienence type alias for the DRM device type for this driver @@ -51,19 +53,19 @@ pub(crate) struct NovaData { impl auxiliary::Driver for NovaDriver { type IdInfo = (); - type Data<'bound> = Self; + type Data<'bound> = Nova; const ID_TABLE: auxiliary::IdTable = &AUX_TABLE; fn probe<'bound>( adev: &'bound auxiliary::Device>, _info: &'bound Self::IdInfo, - ) -> impl PinInit + 'bound { + ) -> impl PinInit, Error> + 'bound { let data = try_pin_init!(NovaData { adev: adev.into() }); let drm = drm::Device::::new(adev.as_ref(), data)?; drm::Registration::new_foreign_owned(&drm, adev.as_ref(), 0)?; - Ok(Self { drm }) + Ok(Nova { drm }) } } From 7740837d84b48cecc78a4672ad7f3279f4aac245 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sun, 19 Apr 2026 11:48:14 +0900 Subject: [PATCH 094/131] gpu: nova-core: remove unneeded get_gsp_info proxy function This function was useful before the generic command-queue send methods got merged, but it is just boilerplate now. Replace it with the correct sequence to queue the `GetGspStaticInfo` command directly. Reviewed-by: Eliot Courtney Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260521-nova-unload-v6-1-65f581c812c9@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/boot.rs | 2 +- drivers/gpu/nova-core/gsp/commands.rs | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 3e8f9611d2b3..0e6eea072e5b 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -243,7 +243,7 @@ pub(crate) fn boot( commands::wait_gsp_init_done(&self.cmdq)?; // Obtain and display basic GPU information. - let info = commands::get_gsp_info(&self.cmdq, bar)?; + let info = self.cmdq.send_command(bar, commands::GetGspStaticInfo)?; match info.gpu_name() { Ok(name) => dev_info!(pdev, "GPU name: {}\n", name), Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e), diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index c89c7b57a751..e81a865050e0 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -18,7 +18,6 @@ }; use crate::{ - driver::Bar0, gsp::{ cmdq::{ Cmdq, @@ -176,7 +175,7 @@ pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq) -> Result { } /// The `GetGspStaticInfo` command. -struct GetGspStaticInfo; +pub(crate) struct GetGspStaticInfo; impl CommandToGsp for GetGspStaticInfo { const FUNCTION: MsgFunction = MsgFunction::GetGspStaticInfo; @@ -232,8 +231,3 @@ pub(crate) fn gpu_name(&self) -> core::result::Result<&str, GpuNameError> { .map_err(GpuNameError::InvalidUtf8) } } - -/// Send the [`GetGspInfo`] command and awaits for its reply. -pub(crate) fn get_gsp_info(cmdq: &Cmdq, bar: &Bar0) -> Result { - cmdq.send_command(bar, GetGspStaticInfo) -} From 60b8bfabf5fccb7a08906f4c35cffed4683190ed Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Mon, 20 Apr 2026 18:42:27 +0900 Subject: [PATCH 095/131] gpu: nova-core: do not import firmware commands into GSP command module Importing all the firmware commands like we did is a bit confusing, as the layer of a command type (fw or GSP) cannot be inferred from looking at its name alone. Furthermore it makes it impossible to create commands that have the same name as their firmware command. Thus, stop importing all commands and refer to them from the `fw` module instead. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260521-nova-unload-v6-2-65f581c812c9@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/commands.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index e81a865050e0..ac9cef312b10 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use core::{ array, @@ -26,7 +27,7 @@ NoReply, // }, fw::{ - commands::*, + self, MsgFunction, // }, }, @@ -47,12 +48,12 @@ pub(crate) fn new(pdev: &'a pci::Device) -> Self { impl<'a> CommandToGsp for SetSystemInfo<'a> { const FUNCTION: MsgFunction = MsgFunction::GspSetSystemInfo; - type Command = GspSetSystemInfo; + type Command = fw::commands::GspSetSystemInfo; type Reply = NoReply; type InitError = Error; fn init(&self) -> impl Init { - GspSetSystemInfo::init(self.pdev) + Self::Command::init(self.pdev) } } @@ -99,12 +100,12 @@ pub(crate) fn new() -> Self { impl CommandToGsp for SetRegistry { const FUNCTION: MsgFunction = MsgFunction::SetRegistry; - type Command = PackedRegistryTable; + type Command = fw::commands::PackedRegistryTable; type Reply = NoReply; type InitError = Infallible; fn init(&self) -> impl Init { - PackedRegistryTable::init(Self::NUM_ENTRIES as u32, self.variable_payload_len() as u32) + Self::Command::init(Self::NUM_ENTRIES as u32, self.variable_payload_len() as u32) } fn variable_payload_len(&self) -> usize { @@ -112,22 +113,22 @@ fn variable_payload_len(&self) -> usize { for i in 0..Self::NUM_ENTRIES { key_size += self.entries[i].key.len() + 1; // +1 for NULL terminator } - Self::NUM_ENTRIES * size_of::() + key_size + Self::NUM_ENTRIES * size_of::() + key_size } fn init_variable_payload( &self, dst: &mut SBufferIter>, ) -> Result { - let string_data_start_offset = - size_of::() + Self::NUM_ENTRIES * size_of::(); + let string_data_start_offset = size_of::() + + Self::NUM_ENTRIES * size_of::(); // Array for string data. let mut string_data = KVec::new(); for entry in self.entries.iter().take(Self::NUM_ENTRIES) { dst.write_all( - PackedRegistryEntry::new( + fw::commands::PackedRegistryEntry::new( (string_data_start_offset + string_data.len()) as u32, entry.value, ) @@ -179,12 +180,12 @@ pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq) -> Result { impl CommandToGsp for GetGspStaticInfo { const FUNCTION: MsgFunction = MsgFunction::GetGspStaticInfo; - type Command = GspStaticConfigInfo; + type Command = fw::commands::GspStaticConfigInfo; type Reply = GetGspStaticInfoReply; type InitError = Infallible; fn init(&self) -> impl Init { - GspStaticConfigInfo::init_zeroed() + Self::Command::init_zeroed() } } @@ -195,7 +196,7 @@ pub(crate) struct GetGspStaticInfoReply { impl MessageFromGsp for GetGspStaticInfoReply { const FUNCTION: MsgFunction = MsgFunction::GetGspStaticInfo; - type Message = GspStaticConfigInfo; + type Message = fw::commands::GspStaticConfigInfo; type InitError = Infallible; fn read( From c852faa9817981eb3fccd69e9a650d729e17d24a Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 10 Apr 2026 19:49:35 -0700 Subject: [PATCH 096/131] gpu: nova-core: refactor SEC2 booter loading into BooterFirmware::run() Move the SEC2 reset/load/boot sequence into a BooterFirmware::run() method. This is mostly refactoring, with no significant behavior change, done in preparation for adding an alternative FSP boot path. Suggested-by: Danilo Krummrich Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260521-nova-unload-v6-4-65f581c812c9@nvidia.com [acourbot: fix typo in commit message.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware/booter.rs | 31 ++++++++++++++++++++++++ drivers/gpu/nova-core/gsp/boot.rs | 30 ++++++----------------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index de2a4536b532..e45e5dc8d5d2 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. //! Support for loading and patching the `Booter` firmware. `Booter` is a Heavy Secured firmware //! running on [`Sec2`], that is used on Turing/Ampere to load the GSP firmware into the GSP falcon @@ -8,6 +9,7 @@ use kernel::{ device, + dma::Coherent, prelude::*, transmute::FromBytes, // }; @@ -396,6 +398,35 @@ pub(crate) fn new( ucode: ucode_signed, }) } + + /// Load and run the booter firmware on SEC2. + /// + /// Resets SEC2, loads this firmware image, then boots with the WPR metadata + /// address passed via the SEC2 mailboxes. + pub(crate) fn run( + &self, + dev: &device::Device, + bar: &Bar0, + sec2_falcon: &Falcon, + wpr_meta: &Coherent, + ) -> Result { + sec2_falcon.reset(bar)?; + sec2_falcon.load(dev, bar, self)?; + let wpr_handle = wpr_meta.dma_handle(); + let (mbox0, mbox1) = sec2_falcon.boot( + bar, + Some(wpr_handle as u32), + Some((wpr_handle >> 32) as u32), + )?; + dev_dbg!(dev, "SEC2 MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); + + if mbox0 != 0 { + dev_err!(dev, "Booter-load failed with error {:#x}\n", mbox0); + return Err(ENODEV); + } + + Ok(()) + } } impl FalconDmaLoadable for BooterFirmware { diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 0e6eea072e5b..14927b374f3c 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -171,15 +171,6 @@ pub(crate) fn boot( Self::run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?; } - let booter_loader = BooterFirmware::new( - dev, - BooterKind::Loader, - chipset, - FIRMWARE_VERSION, - sec2_falcon, - bar, - )?; - let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; self.cmdq @@ -201,20 +192,15 @@ pub(crate) fn boot( "Using SEC2 to load and run the booter_load firmware...\n" ); - sec2_falcon.reset(bar)?; - sec2_falcon.load(dev, bar, &booter_loader)?; - let wpr_handle = wpr_meta.dma_handle(); - let (mbox0, mbox1) = sec2_falcon.boot( + BooterFirmware::new( + dev, + BooterKind::Loader, + chipset, + FIRMWARE_VERSION, + sec2_falcon, bar, - Some(wpr_handle as u32), - Some((wpr_handle >> 32) as u32), - )?; - dev_dbg!(pdev, "SEC2 MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); - - if mbox0 != 0 { - dev_err!(pdev, "Booter-load failed with error {:#x}\n", mbox0); - return Err(ENODEV); - } + )? + .run(dev, bar, sec2_falcon, &wpr_meta)?; gsp_falcon.write_os_version(bar, gsp_fw.bootloader.app_version); From 0e42ec83d46ab8877d38d37493328ed7d1a24de8 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Sat, 25 Apr 2026 22:11:12 +0900 Subject: [PATCH 097/131] gpu: nova-core: gsp: shuffle boot code a bit to keep chipset-specific parts close Some parts of the GSP boot process are chip-specific actions, whereas others (like sending the initial post-boot messages) deal directly with the working GSP. Reorganize the boot code a bit so the chipset-specific parts are clumped together, which will make their extraction into a HAL easier. This has no effect on the GSP boot process. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260521-nova-unload-v6-5-65f581c812c9@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/boot.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 14927b374f3c..259f7c4d94f5 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -166,18 +166,13 @@ pub(crate) fn boot( let fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?; dev_dbg!(dev, "{:#x?}\n", fb_layout); + let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; + // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). if !fb_layout.frts.is_empty() { Self::run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?; } - let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; - - self.cmdq - .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev))?; - self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new())?; - gsp_falcon.reset(bar)?; let libos_handle = self.libos.dma_handle(); let (mbox0, mbox1) = gsp_falcon.boot( @@ -214,6 +209,11 @@ pub(crate) fn boot( dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(bar),); + self.cmdq + .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev))?; + self.cmdq + .send_command_no_wait(bar, commands::SetRegistry::new())?; + // Create and run the GSP sequencer. let seq_params = GspSequencerParams { bootloader_app_version: gsp_fw.bootloader.app_version, From a355d2dd381d129135d1199d66a7f96490551bac Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 29 May 2026 16:33:41 +0900 Subject: [PATCH 098/131] gpu: nova-core: gsp: move chipset-specific parts of the boot process into a HAL Booting the GSP is done differently depending on the architecture. Move the parts that are chipset-specific under a HAL. This does not change much at the moment, since the differences between Turing and Ampere are rather benign, but will become critical to properly support the FSP boot process used by Hopper and Blackwell. The Hopper/Blackwell support is not merged yet, so their HAL is a stub for now. This patch is intended to be a mechanical code extraction with no behavioral changes. Reviewed-by: Eliot Courtney Reviewed-by: Danilo Krummrich Link: https://patch.msgid.link/20260529-nova-unload-v7-1-678f39209e00@nvidia.com [acourbot: `Result<()>` -> `Result`] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp.rs | 1 + drivers/gpu/nova-core/gsp/boot.rs | 166 ++------------------ drivers/gpu/nova-core/gsp/hal.rs | 74 +++++++++ drivers/gpu/nova-core/gsp/hal/gh100.rs | 50 ++++++ drivers/gpu/nova-core/gsp/hal/tu102.rs | 206 +++++++++++++++++++++++++ 5 files changed, 344 insertions(+), 153 deletions(-) create mode 100644 drivers/gpu/nova-core/gsp/hal.rs create mode 100644 drivers/gpu/nova-core/gsp/hal/gh100.rs create mode 100644 drivers/gpu/nova-core/gsp/hal/tu102.rs diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index ba5b7f990031..38378f104068 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 mod boot; +mod hal; use kernel::{ debugfs, diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 259f7c4d94f5..1bd9f21fc443 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -4,7 +4,6 @@ device, dma::Coherent, io::poll::read_poll_timeout, - io::Io, pci, prelude::*, time::Delta, // @@ -19,121 +18,17 @@ }, fb::FbLayout, firmware::{ - booter::{ - BooterFirmware, - BooterKind, // - }, - fwsec::{ - bootloader::FwsecFirmwareWithBl, - FwsecCommand, - FwsecFirmware, // - }, gsp::GspFirmware, FIRMWARE_VERSION, // }, - gpu::{ - Architecture, - Chipset, // - }, + gpu::Chipset, gsp::{ commands, - sequencer::{ - GspSequencer, - GspSequencerParams, // - }, GspFwWprMeta, // }, - regs, - vbios::Vbios, }; impl super::Gsp { - /// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly - /// created the WPR2 region. - fn run_fwsec_frts( - dev: &device::Device, - chipset: Chipset, - falcon: &Falcon, - bar: &Bar0, - bios: &Vbios, - fb_layout: &FbLayout, - ) -> Result<()> { - // Check that the WPR2 region does not already exists - if it does, we cannot run - // FWSEC-FRTS until the GPU is reset. - if bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound() != 0 { - dev_err!( - dev, - "WPR2 region already exists - GPU needs to be reset to proceed\n" - ); - return Err(EBUSY); - } - - // FWSEC-FRTS will create the WPR2 region. - let fwsec_frts = FwsecFirmware::new( - dev, - falcon, - bar, - bios, - FwsecCommand::Frts { - frts_addr: fb_layout.frts.start, - frts_size: fb_layout.frts.len(), - }, - )?; - - if chipset.needs_fwsec_bootloader() { - let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; - // Load and run the bootloader, which will load FWSEC-FRTS and run it. - fwsec_frts_bl.run(dev, falcon, bar)?; - } else { - // Load and run FWSEC-FRTS directly. - fwsec_frts.run(dev, falcon, bar)?; - } - - // SCRATCH_E contains the error code for FWSEC-FRTS. - let frts_status = bar - .read(regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR) - .frts_err_code(); - if frts_status != 0 { - dev_err!( - dev, - "FWSEC-FRTS returned with error code {:#x}\n", - frts_status - ); - - return Err(EIO); - } - - // Check that the WPR2 region has been created as we requested. - let (wpr2_lo, wpr2_hi) = ( - bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO).lower_bound(), - bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound(), - ); - - match (wpr2_lo, wpr2_hi) { - (_, 0) => { - dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); - - Err(EIO) - } - (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { - dev_err!( - dev, - "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", - wpr2_lo, - fb_layout.frts.start, - ); - - Err(EIO) - } - (wpr2_lo, wpr2_hi) => { - dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); - dev_dbg!(dev, "GPU instance built\n"); - - Ok(()) - } - } - } - /// Attempt to boot the GSP. /// /// This is a GPU-dependent and complex procedure that involves loading firmware files from @@ -149,17 +44,8 @@ pub(crate) fn boot( gsp_falcon: &Falcon, sec2_falcon: &Falcon, ) -> Result { - // The FSP boot process of Hopper+ is not supported for now. - if matches!( - chipset.arch(), - Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x - ) { - return Err(ENOTSUPP); - } - let dev = pdev.as_ref(); - - let bios = Vbios::new(dev, bar)?; + let hal = super::hal::gsp_hal(chipset); let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset, FIRMWARE_VERSION), GFP_KERNEL)?; @@ -168,38 +54,21 @@ pub(crate) fn boot( let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; - // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). - if !fb_layout.frts.is_empty() { - Self::run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?; - } - - gsp_falcon.reset(bar)?; - let libos_handle = self.libos.dma_handle(); - let (mbox0, mbox1) = gsp_falcon.boot( - bar, - Some(libos_handle as u32), - Some((libos_handle >> 32) as u32), - )?; - dev_dbg!(pdev, "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); - - dev_dbg!( - pdev, - "Using SEC2 to load and run the booter_load firmware...\n" - ); - - BooterFirmware::new( + // Perform the chipset-specific boot sequence. + hal.boot( + &self, dev, - BooterKind::Loader, - chipset, - FIRMWARE_VERSION, - sec2_falcon, bar, - )? - .run(dev, bar, sec2_falcon, &wpr_meta)?; + chipset, + &fb_layout, + &wpr_meta, + gsp_falcon, + sec2_falcon, + )?; gsp_falcon.write_os_version(bar, gsp_fw.bootloader.app_version); - // Poll for RISC-V to become active before running sequencer + // Poll for RISC-V to become active before continuing. read_poll_timeout( || Ok(gsp_falcon.is_riscv_active(bar)), |val: &bool| *val, @@ -214,16 +83,7 @@ pub(crate) fn boot( self.cmdq .send_command_no_wait(bar, commands::SetRegistry::new())?; - // Create and run the GSP sequencer. - let seq_params = GspSequencerParams { - bootloader_app_version: gsp_fw.bootloader.app_version, - libos_dma_handle: libos_handle, - gsp_falcon, - sec2_falcon, - dev, - bar, - }; - GspSequencer::run(&self.cmdq, seq_params)?; + hal.post_boot(&self, dev, bar, &gsp_fw, gsp_falcon, sec2_falcon)?; // Wait until GSP is fully initialized. commands::wait_gsp_init_done(&self.cmdq)?; diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs new file mode 100644 index 000000000000..fb3edaeb3160 --- /dev/null +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +mod gh100; +mod tu102; + +use kernel::prelude::*; + +use kernel::{ + device, + dma::Coherent, // +}; + +use crate::{ + driver::Bar0, + falcon::{ + gsp::Gsp as GspEngine, + sec2::Sec2, + Falcon, // + }, + fb::FbLayout, + firmware::gsp::GspFirmware, + gpu::{ + Architecture, + Chipset, // + }, + gsp::{ + Gsp, + GspFwWprMeta, // + }, +}; + +/// Trait implemented by GSP HALs. +pub(super) trait GspHal: Send { + /// Performs the GSP boot process, loading and running the required firmwares as needed. + #[allow(clippy::too_many_arguments)] + fn boot( + &self, + gsp: &Gsp, + dev: &device::Device, + bar: &Bar0, + chipset: Chipset, + fb_layout: &FbLayout, + wpr_meta: &Coherent, + gsp_falcon: &Falcon, + sec2_falcon: &Falcon, + ) -> Result; + + /// Performs HAL-specific post-GSP boot tasks. + /// + /// This method is called by the GSP boot code after the GSP is confirmed to be running, and + /// after the initialization commands have been pushed onto its queue. + fn post_boot( + &self, + _gsp: &Gsp, + _dev: &device::Device, + _bar: &Bar0, + _gsp_fw: &GspFirmware, + _gsp_falcon: &Falcon, + _sec2_falcon: &Falcon, + ) -> Result { + Ok(()) + } +} + +/// Returns the GSP HAL to be used for `chipset`. +pub(super) fn gsp_hal(chipset: Chipset) -> &'static dyn GspHal { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL, + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + gh100::GH100_HAL + } + } +} diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs new file mode 100644 index 000000000000..3f3675f9c16a --- /dev/null +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::prelude::*; + +use kernel::{ + device, + dma::Coherent, // +}; + +use crate::{ + driver::Bar0, + falcon::{ + gsp::Gsp as GspEngine, + sec2::Sec2, + Falcon, // + }, + fb::FbLayout, + gpu::Chipset, + gsp::{ + hal::GspHal, + Gsp, + GspFwWprMeta, // + }, +}; + +struct Gh100; + +impl GspHal for Gh100 { + /// Boot GSP via FSP Chain of Trust (Hopper/Blackwell+ path). + /// + /// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles + /// the GSP boot internally - no manual GSP reset/boot is needed. + fn boot( + &self, + _gsp: &Gsp, + _dev: &device::Device, + _bar: &Bar0, + _chipset: Chipset, + _fb_layout: &FbLayout, + _wpr_meta: &Coherent, + _gsp_falcon: &Falcon, + _sec2_falcon: &Falcon, + ) -> Result { + Err(ENOTSUPP) + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn GspHal = &GH100; diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs new file mode 100644 index 000000000000..aacf9c77f070 --- /dev/null +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::prelude::*; + +use kernel::{ + device, + dma::Coherent, + io::Io, // +}; + +use crate::{ + driver::Bar0, + falcon::{ + gsp::Gsp as GspEngine, + sec2::Sec2, + Falcon, // + }, + fb::FbLayout, + firmware::{ + booter::{ + BooterFirmware, + BooterKind, // + }, + fwsec::{ + bootloader::FwsecFirmwareWithBl, + FwsecCommand, + FwsecFirmware, // + }, + gsp::GspFirmware, + FIRMWARE_VERSION, // + }, + gpu::Chipset, + gsp::{ + hal::GspHal, + sequencer::{ + GspSequencer, + GspSequencerParams, // + }, + Gsp, + GspFwWprMeta, // + }, + regs, + vbios::Vbios, // +}; + +/// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly +/// created the WPR2 region. +fn run_fwsec_frts( + dev: &device::Device, + chipset: Chipset, + falcon: &Falcon, + bar: &Bar0, + bios: &Vbios, + fb_layout: &FbLayout, +) -> Result { + // Check that the WPR2 region does not already exist - if it does, we cannot run + // FWSEC-FRTS until the GPU is reset. + if bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound() != 0 { + dev_err!( + dev, + "WPR2 region already exists - GPU needs to be reset to proceed\n" + ); + return Err(EBUSY); + } + + // FWSEC-FRTS will create the WPR2 region. + let fwsec_frts = FwsecFirmware::new( + dev, + falcon, + bar, + bios, + FwsecCommand::Frts { + frts_addr: fb_layout.frts.start, + frts_size: fb_layout.frts.len(), + }, + )?; + + if chipset.needs_fwsec_bootloader() { + let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; + // Load and run the bootloader, which will load FWSEC-FRTS and run it. + fwsec_frts_bl.run(dev, falcon, bar)?; + } else { + // Load and run FWSEC-FRTS directly. + fwsec_frts.run(dev, falcon, bar)?; + } + + // SCRATCH_E contains the error code for FWSEC-FRTS. + let frts_status = bar + .read(regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR) + .frts_err_code(); + if frts_status != 0 { + dev_err!( + dev, + "FWSEC-FRTS returned with error code {:#x}\n", + frts_status + ); + + return Err(EIO); + } + + // Check that the WPR2 region has been created as we requested. + let (wpr2_lo, wpr2_hi) = ( + bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO).lower_bound(), + bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound(), + ); + + match (wpr2_lo, wpr2_hi) { + (_, 0) => { + dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); + + Err(EIO) + } + (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { + dev_err!( + dev, + "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", + wpr2_lo, + fb_layout.frts.start, + ); + + Err(EIO) + } + (wpr2_lo, wpr2_hi) => { + dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); + dev_dbg!(dev, "GPU instance built\n"); + + Ok(()) + } + } +} + +struct Tu102; + +impl GspHal for Tu102 { + fn boot( + &self, + gsp: &Gsp, + dev: &device::Device, + bar: &Bar0, + chipset: Chipset, + fb_layout: &FbLayout, + wpr_meta: &Coherent, + gsp_falcon: &Falcon, + sec2_falcon: &Falcon, + ) -> Result { + let bios = Vbios::new(dev, bar)?; + + // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). + if !fb_layout.frts.is_empty() { + run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, fb_layout)?; + } + + gsp_falcon.reset(bar)?; + let libos_handle = gsp.libos.dma_handle(); + let (mbox0, mbox1) = gsp_falcon.boot( + bar, + Some(libos_handle as u32), + Some((libos_handle >> 32) as u32), + )?; + dev_dbg!(dev, "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); + + dev_dbg!( + dev, + "Using SEC2 to load and run the booter_load firmware...\n" + ); + + BooterFirmware::new( + dev, + BooterKind::Loader, + chipset, + FIRMWARE_VERSION, + sec2_falcon, + bar, + )? + .run(dev, bar, sec2_falcon, wpr_meta)?; + + Ok(()) + } + + fn post_boot( + &self, + gsp: &Gsp, + dev: &device::Device, + bar: &Bar0, + gsp_fw: &GspFirmware, + gsp_falcon: &Falcon, + sec2_falcon: &Falcon, + ) -> Result { + // Create and run the GSP sequencer. + let seq_params = GspSequencerParams { + bootloader_app_version: gsp_fw.bootloader.app_version, + libos_dma_handle: gsp.libos.dma_handle(), + gsp_falcon, + sec2_falcon, + dev, + bar, + }; + GspSequencer::run(&gsp.cmdq, seq_params)?; + + Ok(()) + } +} + +const TU102: Tu102 = Tu102; +pub(super) const TU102_HAL: &dyn GspHal = &TU102; From 96c377e999737cddbfbf1de975b5285fa047c4bc Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 29 May 2026 16:33:42 +0900 Subject: [PATCH 099/131] gpu: nova-core: send UNLOADING_GUEST_DRIVER GSP command upon unloading Currently, the GSP is left running after the driver is unbound. This is not great for several reasons, notably that it can still access shared memory areas that the kernel will now reclaim (especially problematic on setups without an IOMMU). Fix this by sending the `UNLOADING_GUEST_DRIVER` GSP command when the `Gpu` is dropped. This stops the GSP and lets us proceed with the rest of the unbind sequence in a later patch. Reviewed-by: Eliot Courtney Co-developed-by: Eliot Courtney Signed-off-by: Eliot Courtney Reviewed-by: Danilo Krummrich Link: https://patch.msgid.link/20260529-nova-unload-v7-2-678f39209e00@nvidia.com [acourbot: `Result<()>` -> `Result`] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 21 ++++++++- drivers/gpu/nova-core/gsp/boot.rs | 45 +++++++++++++++++++ drivers/gpu/nova-core/gsp/commands.rs | 43 ++++++++++++++++++ drivers/gpu/nova-core/gsp/fw.rs | 4 ++ drivers/gpu/nova-core/gsp/fw/commands.rs | 45 +++++++++++++++++++ .../gpu/nova-core/gsp/fw/r570_144/bindings.rs | 11 +++++ 6 files changed, 168 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index cf134cab49cd..011d504830e4 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -243,8 +243,10 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { } /// Structure holding the resources required to operate the GPU. -#[pin_data] +#[pin_data(PinnedDrop)] pub(crate) struct Gpu<'gpu> { + /// Device owning the GPU. + device: &'gpu device::Device, spec: Spec, /// MMIO mapping of PCI BAR 0. bar: &'gpu Bar0, @@ -266,6 +268,7 @@ pub(crate) fn new( bar: &'gpu Bar0, ) -> impl PinInit + 'gpu { try_pin_init!(Self { + device: pdev.as_ref(), spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { dev_info!(pdev,"NVIDIA ({})\n", spec); })?, @@ -294,3 +297,19 @@ pub(crate) fn new( }) } } + +#[pinned_drop] +impl PinnedDrop for Gpu<'_> { + fn drop(self: Pin<&mut Self>) { + let this = self.project(); + let device = *this.device; + let bar = *this.bar; + + let _ = this + .gsp + .as_ref() + .get_ref() + .unload(device, bar, &*this.gsp_falcon) + .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e)); + } +} diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 1bd9f21fc443..199a13e1843f 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ + bits, device, dma::Coherent, io::poll::read_poll_timeout, @@ -23,6 +25,7 @@ }, gpu::Chipset, gsp::{ + cmdq::Cmdq, commands, GspFwWprMeta, // }, @@ -97,4 +100,46 @@ pub(crate) fn boot( Ok(()) } + + /// Shut down the GSP and wait until it is offline. + fn shutdown_gsp( + cmdq: &Cmdq, + bar: &Bar0, + gsp_falcon: &Falcon, + mode: commands::PowerStateLevel, + ) -> Result { + // Command to shut the GSP down. + cmdq.send_command(bar, commands::UnloadingGuestDriver::new(mode))?; + + // Wait until GSP signals it is suspended. + const LIBOS_INTERRUPT_PROCESSOR_SUSPENDED: u32 = bits::bit_u32(31); + read_poll_timeout( + || Ok(gsp_falcon.read_mailbox0(bar)), + |&mb0| mb0 & LIBOS_INTERRUPT_PROCESSOR_SUSPENDED != 0, + Delta::from_millis(10), + Delta::from_secs(5), + ) + .map(|_| ()) + } + + /// Attempts to unload the GSP firmware. + /// + /// This stops all activity on the GSP. + pub(crate) fn unload( + &self, + dev: &device::Device, + bar: &Bar0, + gsp_falcon: &Falcon, + ) -> Result { + // Shut down the GSP. + Self::shutdown_gsp( + &self.cmdq, + bar, + gsp_falcon, + commands::PowerStateLevel::Level0, + ) + .inspect_err(|e| dev_err!(dev, "Unload guest driver failed: {:?}\n", e))?; + + Ok(()) + } } diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index ac9cef312b10..3a365455d10c 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -232,3 +232,46 @@ pub(crate) fn gpu_name(&self) -> core::result::Result<&str, GpuNameError> { .map_err(GpuNameError::InvalidUtf8) } } + +pub(crate) use fw::commands::PowerStateLevel; + +/// The `UnloadingGuestDriver` command, used to shut down the GSP. +/// +/// Only used within the `gsp` module. +pub(super) struct UnloadingGuestDriver { + level: PowerStateLevel, +} + +impl UnloadingGuestDriver { + /// Creates a new `UnloadingGuestDriver` command for the given [`PowerStateLevel`]. + pub(super) fn new(level: PowerStateLevel) -> Self { + Self { level } + } +} + +impl CommandToGsp for UnloadingGuestDriver { + const FUNCTION: MsgFunction = MsgFunction::UnloadingGuestDriver; + type Command = fw::commands::UnloadingGuestDriver; + type Reply = UnloadingGuestDriverReply; + type InitError = Infallible; + + fn init(&self) -> impl Init { + fw::commands::UnloadingGuestDriver::new(self.level) + } +} + +/// The reply from the GSP to the [`UnloadingGuestDriver`] command. +pub(super) struct UnloadingGuestDriverReply; + +impl MessageFromGsp for UnloadingGuestDriverReply { + const FUNCTION: MsgFunction = MsgFunction::UnloadingGuestDriver; + type InitError = Infallible; + type Message = (); + + fn read( + _msg: &Self::Message, + _sbuffer: &mut SBufferIter>, + ) -> Result { + Ok(UnloadingGuestDriverReply) + } +} diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 3245793bbe42..33c9f5860771 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -279,6 +279,7 @@ pub(crate) enum MsgFunction { Nop = bindings::NV_VGPU_MSG_FUNCTION_NOP, SetGuestSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO, SetRegistry = bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY, + UnloadingGuestDriver = bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, // Event codes GspInitDone = bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE, @@ -323,6 +324,9 @@ fn try_from(value: u32) -> Result { Ok(MsgFunction::SetGuestSystemInfo) } bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY => Ok(MsgFunction::SetRegistry), + bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER => { + Ok(MsgFunction::UnloadingGuestDriver) + } // Event codes bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE => Ok(MsgFunction::GspInitDone), diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index db46276430be..42985d446bae 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ device, @@ -129,3 +130,47 @@ unsafe impl AsBytes for GspStaticConfigInfo {} // SAFETY: This struct only contains integer types for which all bit patterns // are valid. unsafe impl FromBytes for GspStaticConfigInfo {} + +/// Power level requested to the [`UnloadingGuestDriver`] command. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u32)] +#[expect(unused)] +pub(crate) enum PowerStateLevel { + /// Full unload. + Level0 = bindings::NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0, + /// S3 (suspend to RAM). + Level3 = bindings::NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3, + /// Hibernate (suspend to disk). + Level7 = bindings::NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_7, +} + +impl PowerStateLevel { + /// Returns `true` if this state represents a power management transition, i.e. some GPU state + /// must survive it (as opposed to a full unload). + pub(crate) fn is_power_transition(self) -> bool { + self != PowerStateLevel::Level0 + } +} + +/// Payload of the `UnloadingGuestDriver` command and message. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Zeroable)] +pub(crate) struct UnloadingGuestDriver(bindings::rpc_unloading_guest_driver_v1F_07); + +impl UnloadingGuestDriver { + pub(crate) fn new(level: PowerStateLevel) -> Self { + Self(bindings::rpc_unloading_guest_driver_v1F_07 { + bInPMTransition: u8::from(level.is_power_transition()), + bGc6Entering: 0, + newLevel: level as u32, + ..Zeroable::zeroed() + }) + } +} + +// SAFETY: Padding is explicit and will not contain uninitialized data. +unsafe impl AsBytes for UnloadingGuestDriver {} + +// SAFETY: This struct only contains integer types for which all bit patterns +// are valid. +unsafe impl FromBytes for UnloadingGuestDriver {} diff --git a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs index 334e8be5fde8..f82ed097b283 100644 --- a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs +++ b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs @@ -30,6 +30,9 @@ fn fmt(&self, fmt: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { fmt.write_str("__IncompleteArrayField") } } +pub const NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0: u32 = 0; +pub const NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3: u32 = 3; +pub const NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_7: u32 = 7; pub const NV_VGPU_MSG_SIGNATURE_VALID: u32 = 1129337430; pub const GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2: u32 = 0; pub const GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL: u32 = 23068672; @@ -880,6 +883,14 @@ fn default() -> Self { } } #[repr(C)] +#[derive(Debug, Default, Copy, Clone, MaybeZeroable)] +pub struct rpc_unloading_guest_driver_v1F_07 { + pub bInPMTransition: u8_, + pub bGc6Entering: u8_, + pub __bindgen_padding_0: [u8; 2usize], + pub newLevel: u32_, +} +#[repr(C)] #[derive(Debug, Default, MaybeZeroable)] pub struct rpc_run_cpu_sequencer_v17_00 { pub bufferSizeDWord: u32_, From adb99ce3cc78d277a719f15a8131eafc60162f22 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 29 May 2026 16:33:43 +0900 Subject: [PATCH 100/131] gpu: nova-core: run Booter Unloader and FWSEC-SB upon unbinding When probing the driver, the FWSEC-FRTS firmware creates a WPR2 secure memory region to store the GSP firmware, and the Booter Loader loads and starts that firmware into the GSP, making it run in RISC-V mode. These operations need to be reverted upon unloading, particularly the WPR2 secure region creation, as its presence prevents the driver from subsequently probing. Thus, prepare the Booter Unloader and FWSEC-SB firmware images when booting the GSP, so they can be executed at unbind time to put the GPU into a state where it can be probed again. Reviewed-by: Eliot Courtney Co-developed-by: Eliot Courtney Signed-off-by: Eliot Courtney Reviewed-by: Danilo Krummrich Link: https://patch.msgid.link/20260529-nova-unload-v7-3-678f39209e00@nvidia.com [acourbot: `Result<()>` -> `Result`] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware/booter.rs | 1 - drivers/gpu/nova-core/firmware/fwsec.rs | 1 - drivers/gpu/nova-core/gpu.rs | 15 ++- drivers/gpu/nova-core/gsp.rs | 3 + drivers/gpu/nova-core/gsp/boot.rs | 38 ++++-- drivers/gpu/nova-core/gsp/hal.rs | 21 +++- drivers/gpu/nova-core/gsp/hal/gh100.rs | 2 +- drivers/gpu/nova-core/gsp/hal/tu102.rs | 142 ++++++++++++++++++++++- drivers/gpu/nova-core/regs.rs | 5 + 9 files changed, 209 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index e45e5dc8d5d2..c5e17605e1a3 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -282,7 +282,6 @@ fn new_booter(data: &[u8]) -> Result { #[derive(Copy, Clone, Debug, PartialEq)] pub(crate) enum BooterKind { Loader, - #[expect(unused)] Unloader, } diff --git a/drivers/gpu/nova-core/firmware/fwsec.rs b/drivers/gpu/nova-core/firmware/fwsec.rs index 8810cb49db67..4108f28cd338 100644 --- a/drivers/gpu/nova-core/firmware/fwsec.rs +++ b/drivers/gpu/nova-core/firmware/fwsec.rs @@ -144,7 +144,6 @@ pub(crate) enum FwsecCommand { /// image into it. Frts { frts_addr: u64, frts_size: u64 }, /// Asks [`FwsecFirmware`] to load pre-OS apps on the PMU. - #[expect(dead_code)] Sb, } diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 011d504830e4..aed992488db3 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -18,7 +18,10 @@ Falcon, // }, fb::SysmemFlush, - gsp::Gsp, + gsp::{ + self, + Gsp, // + }, regs, }; @@ -260,6 +263,8 @@ pub(crate) struct Gpu<'gpu> { /// GSP runtime data. Temporarily an empty placeholder. #[pin] gsp: Gsp, + /// GSP unload firmware bundle, if any. + unload_bundle: Option, } impl<'gpu> Gpu<'gpu> { @@ -293,7 +298,10 @@ pub(crate) fn new( gsp <- Gsp::new(pdev), - _: { gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)? }, + // This member must be initialized last, so the `UnloadBundle` can never be dropped from + // outside of the constructed `Gpu`, ensuring that the unload sequence is properly run + // in case of failure. + unload_bundle: gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)?, }) } } @@ -304,12 +312,13 @@ fn drop(self: Pin<&mut Self>) { let this = self.project(); let device = *this.device; let bar = *this.bar; + let bundle = this.unload_bundle.take(); let _ = this .gsp .as_ref() .get_ref() - .unload(device, bar, &*this.gsp_falcon) + .unload(device, bar, &*this.gsp_falcon, &*this.sec2_falcon, bundle) .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e)); } } diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 38378f104068..1885cfa5cb38 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -185,3 +185,6 @@ pub(crate) fn new(pdev: &pci::Device) -> impl PinInit); diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 199a13e1843f..f92ff2034d2d 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -38,7 +38,8 @@ impl super::Gsp { /// user-space, patching them with signatures, and building firmware-specific intricate data /// structures that the GSP will use at runtime. /// - /// Upon return, the GSP is up and running, and its runtime object given as return value. + /// Upon return, the GSP is up and running, and its unload bundle (to be given as argument to + /// [`Self::unload`]) returned. pub(crate) fn boot( self: Pin<&mut Self>, pdev: &pci::Device, @@ -46,7 +47,7 @@ pub(crate) fn boot( chipset: Chipset, gsp_falcon: &Falcon, sec2_falcon: &Falcon, - ) -> Result { + ) -> Result> { let dev = pdev.as_ref(); let hal = super::hal::gsp_hal(chipset); @@ -57,8 +58,8 @@ pub(crate) fn boot( let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; - // Perform the chipset-specific boot sequence. - hal.boot( + // Perform the chipset-specific boot sequence, and retrieve the unload bundle. + let unload_bundle = hal.boot( &self, dev, bar, @@ -98,7 +99,7 @@ pub(crate) fn boot( Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e), } - Ok(()) + Ok(unload_bundle) } /// Shut down the GSP and wait until it is offline. @@ -130,16 +131,35 @@ pub(crate) fn unload( dev: &device::Device, bar: &Bar0, gsp_falcon: &Falcon, + sec2_falcon: &Falcon, + unload_bundle: Option, ) -> Result { - // Shut down the GSP. - Self::shutdown_gsp( + // Shut down the GSP. Keep going even in case of error. + let mut res = Self::shutdown_gsp( &self.cmdq, bar, gsp_falcon, commands::PowerStateLevel::Level0, ) - .inspect_err(|e| dev_err!(dev, "Unload guest driver failed: {:?}\n", e))?; + .inspect_err(|e| dev_err!(dev, "GSP shutdown failed: {:?}\n", e)); - Ok(()) + // Run the unload bundle to reset the GSP so it can be booted again. + if let Some(unload_bundle) = unload_bundle { + res = res.and( + unload_bundle + .0 + .run(dev, bar, gsp_falcon, sec2_falcon) + .inspect_err(|e| dev_err!(dev, "Unload bundle failed: {:?}\n", e)), + ); + } else { + dev_warn!( + dev, + "Unload bundle is missing, GSP won't be properly reset.\n" + ); + + res = Err(EAGAIN); + } + + res.inspect(|()| dev_info!(dev, "GSP successfully unloaded\n")) } } diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index fb3edaeb3160..501b852dcb29 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -30,9 +30,28 @@ }, }; +/// Trait for types containing the resources and code required to fully reset the GSP. +/// +/// The GSP unload code might run in a situation where we cannot load firmware dynamically (e.g. +/// because we are in shutdown and the file system is not accessible anymore). Thus, the firmware +/// required for unloading is prepared at load time, and stored here until it needs to be run. +pub(super) trait UnloadBundle: Send { + /// Performs the steps required to properly reset the GSP after it has been stopped. + fn run( + &self, + dev: &device::Device, + bar: &Bar0, + gsp_falcon: &Falcon, + sec2_falcon: &Falcon, + ) -> Result; +} + /// Trait implemented by GSP HALs. pub(super) trait GspHal: Send { /// Performs the GSP boot process, loading and running the required firmwares as needed. + /// + /// Upon success, returns the [`UnloadBundle`] to be run (if any) in order to properly reset the + /// GSP after it has been stopped. #[allow(clippy::too_many_arguments)] fn boot( &self, @@ -44,7 +63,7 @@ fn boot( wpr_meta: &Coherent, gsp_falcon: &Falcon, sec2_falcon: &Falcon, - ) -> Result; + ) -> Result>; /// Performs HAL-specific post-GSP boot tasks. /// diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 3f3675f9c16a..0a8b7f763883 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -41,7 +41,7 @@ fn boot( _wpr_meta: &Coherent, _gsp_falcon: &Falcon, _sec2_falcon: &Falcon, - ) -> Result { + ) -> Result> { Err(ENOTSUPP) } } diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index aacf9c77f070..53d117ccdddb 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -32,7 +32,10 @@ }, gpu::Chipset, gsp::{ - hal::GspHal, + hal::{ + GspHal, + UnloadBundle, // + }, sequencer::{ GspSequencer, GspSequencerParams, // @@ -44,6 +47,124 @@ vbios::Vbios, // }; +// A ready-to-run FWSEC unload firmware. +// +// Since there are two variants of the prepared firmware (with and without a bootloader), this type +// abstracts the difference. +enum FwsecUnloadFirmware { + WithoutBl(FwsecFirmware), + WithBl(FwsecFirmwareWithBl), +} + +impl FwsecUnloadFirmware { + /// Loads the FWSEC SB firmware, as well as its bootloader if `chipset` requires it. + fn new( + dev: &device::Device, + bar: &Bar0, + chipset: Chipset, + bios: &Vbios, + gsp_falcon: &Falcon, + ) -> Result { + let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bar, bios, FwsecCommand::Sb)?; + + Ok(if chipset.needs_fwsec_bootloader() { + Self::WithBl(FwsecFirmwareWithBl::new(fwsec_sb, dev, chipset)?) + } else { + Self::WithoutBl(fwsec_sb) + }) + } + + /// Runs the FWSEC SB firmware. + fn run( + &self, + dev: &device::Device, + bar: &Bar0, + gsp_falcon: &Falcon, + ) -> Result { + match self { + Self::WithoutBl(fw) => fw.run(dev, gsp_falcon, bar), + Self::WithBl(fw) => fw.run(dev, gsp_falcon, bar), + } + } +} + +// Contains the firmware required to fully reset GSP on chipsets where the GSP is started using +// FWSEC/Booter. +struct Sec2UnloadBundle { + fwsec_sb: FwsecUnloadFirmware, + booter_unloader: BooterFirmware, +} + +impl Sec2UnloadBundle { + /// Load and prepare the resources required to properly reset the GSP after it has been stopped. + fn build( + dev: &device::Device, + bar: &Bar0, + chipset: Chipset, + bios: &Vbios, + gsp_falcon: &Falcon, + sec2_falcon: &Falcon, + ) -> Result> { + KBox::new( + Self { + fwsec_sb: FwsecUnloadFirmware::new(dev, bar, chipset, bios, gsp_falcon)?, + booter_unloader: BooterFirmware::new( + dev, + BooterKind::Unloader, + chipset, + FIRMWARE_VERSION, + sec2_falcon, + bar, + )?, + }, + GFP_KERNEL, + ) + .map(|b| b as KBox) + .map_err(Into::into) + } +} + +impl UnloadBundle for Sec2UnloadBundle { + fn run( + &self, + dev: &device::Device, + bar: &Bar0, + gsp_falcon: &Falcon, + sec2_falcon: &Falcon, + ) -> Result { + // Run FWSEC-SB to reset the GSP falcon to its pre-libos state. + self.fwsec_sb.run(dev, bar, gsp_falcon)?; + + // Remove WPR2 region if set. + let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); + if wpr2_hi.is_wpr2_set() { + sec2_falcon.reset(bar)?; + sec2_falcon.load(dev, bar, &self.booter_unloader)?; + + // Sentinel value to confirm that Booter Unloader has run. + const MAILBOX_SENTINEL: u32 = 0xff; + let (mbox0, _) = + sec2_falcon.boot(bar, Some(MAILBOX_SENTINEL), Some(MAILBOX_SENTINEL))?; + if mbox0 != 0 { + dev_err!(dev, "Booter Unloader returned error 0x{:x}\n", mbox0); + return Err(EINVAL); + } + + // Confirm that the WPR2 region has been removed. + let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); + if wpr2_hi.is_wpr2_set() { + dev_err!( + dev, + "WPR2 region still set after Booter Unloader returned\n" + ); + return Err(EBUSY); + } + } + + Ok(()) + } +} + /// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly /// created the WPR2 region. fn run_fwsec_frts( @@ -143,9 +264,24 @@ fn boot( wpr_meta: &Coherent, gsp_falcon: &Falcon, sec2_falcon: &Falcon, - ) -> Result { + ) -> Result> { let bios = Vbios::new(dev, bar)?; + // Try and prepare the unload bundle. If this fails, the GPU will need to be reset + // before the driver can be probed again. + let unload_bundle = + Sec2UnloadBundle::build(dev, bar, chipset, &bios, gsp_falcon, sec2_falcon) + .inspect_err(|e| { + dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e); + dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); + dev_warn!( + dev, + "The GPU will need to be reset before the driver can bind again.\n" + ); + }) + .map(crate::gsp::UnloadBundle) + .ok(); + // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). if !fb_layout.frts.is_empty() { run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, fb_layout)?; @@ -175,7 +311,7 @@ fn boot( )? .run(dev, bar, sec2_falcon, wpr_meta)?; - Ok(()) + Ok(unload_bundle) } fn post_boot( diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 6faeed73901d..356fbf364ea5 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -175,6 +175,11 @@ impl NV_PFB_PRI_MMU_WPR2_ADDR_HI { pub(crate) fn higher_bound(self) -> u64 { u64::from(self.hi_val()) << 12 } + + /// Returns whether the WPR2 region is currently set. + pub(crate) fn is_wpr2_set(self) -> bool { + self.hi_val() != 0 + } } // PGSP From 75d59327367dc6e2141cf4e11cdf57c55851b5c2 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Fri, 29 May 2026 16:33:44 +0900 Subject: [PATCH 101/131] gpu: nova-core: gsp: run the unload bundle if Gsp::boot() fails If `Gsp::boot` fails, the GSP can be left in a state where boot cannot be attempted again unless it is reset first. To avoid this, we want to run the unload bundle whenever `boot` fails to try and clear the partially-initialized state. Do this by wrapping the unload bundle into a drop guard up until `boot` returns. After that, running the unload bundle becomes the responsibility of the caller. Reviewed-by: Danilo Krummrich Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260529-nova-unload-v7-4-678f39209e00@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/boot.rs | 67 ++++++++++++++++++++++++-- drivers/gpu/nova-core/gsp/hal.rs | 19 ++++---- drivers/gpu/nova-core/gsp/hal/gh100.rs | 15 +++--- drivers/gpu/nova-core/gsp/hal/tu102.rs | 31 +++++++----- 4 files changed, 101 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index f92ff2034d2d..087ee59da6d9 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -8,7 +8,8 @@ io::poll::read_poll_timeout, pci, prelude::*, - time::Delta, // + time::Delta, + types::ScopeGuard, // }; use crate::{ @@ -31,6 +32,66 @@ }, }; +/// Arguments required to call [`Gsp::unload`](super::Gsp::unload). +/// +/// Stored as their own type to avoid repeating a long and tedious list in [`BootUnloadGuard`]. +pub(super) struct BootUnloadArgs<'a> { + gsp: &'a super::Gsp, + dev: &'a device::Device, + bar: &'a Bar0, + gsp_falcon: &'a Falcon, + sec2_falcon: &'a Falcon, + unload_bundle: Option, +} + +/// Guard that calls [`Gsp::unload`](super::Gsp::unload) with a +/// [`UnloadBundle`](super::UnloadBundle) when dropped. +/// +/// Used to ensure the `UnloadBundle` is run during failure paths. +pub(super) struct BootUnloadGuard<'a> { + guard: ScopeGuard, fn(BootUnloadArgs<'a>)>, +} + +impl<'a> BootUnloadGuard<'a> { + /// Wraps `unload_bundle` into a guard that executes it when dropped. + pub(super) fn new( + gsp: &'a super::Gsp, + dev: &'a device::Device, + bar: &'a Bar0, + gsp_falcon: &'a Falcon, + sec2_falcon: &'a Falcon, + unload_bundle: Option, + ) -> Self { + Self { + guard: ScopeGuard::new_with_data( + BootUnloadArgs { + gsp, + dev, + bar, + gsp_falcon, + sec2_falcon, + unload_bundle, + }, + |args| { + let _ = super::Gsp::unload( + args.gsp, + args.dev, + args.bar, + args.gsp_falcon, + args.sec2_falcon, + args.unload_bundle, + ); + }, + ), + } + } + + /// Disarms the guard and returns the [`UnloadBundle`](super::UnloadBundle) it contains. + pub(super) fn dismiss(self) -> Option { + self.guard.dismiss().unload_bundle + } +} + impl super::Gsp { /// Attempt to boot the GSP. /// @@ -59,7 +120,7 @@ pub(crate) fn boot( let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; // Perform the chipset-specific boot sequence, and retrieve the unload bundle. - let unload_bundle = hal.boot( + let unload_guard = hal.boot( &self, dev, bar, @@ -99,7 +160,7 @@ pub(crate) fn boot( Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e), } - Ok(unload_bundle) + Ok(unload_guard.dismiss()) } /// Shut down the GSP and wait until it is offline. diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 501b852dcb29..88fc3e791114 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -25,6 +25,7 @@ Chipset, // }, gsp::{ + boot::BootUnloadGuard, Gsp, GspFwWprMeta, // }, @@ -50,20 +51,20 @@ fn run( pub(super) trait GspHal: Send { /// Performs the GSP boot process, loading and running the required firmwares as needed. /// - /// Upon success, returns the [`UnloadBundle`] to be run (if any) in order to properly reset the - /// GSP after it has been stopped. + /// Upon success, returns a guard that runs the GSP unload sequence if GSP boot does not + /// complete. #[allow(clippy::too_many_arguments)] - fn boot( + fn boot<'a>( &self, - gsp: &Gsp, - dev: &device::Device, - bar: &Bar0, + gsp: &'a Gsp, + dev: &'a device::Device, + bar: &'a Bar0, chipset: Chipset, fb_layout: &FbLayout, wpr_meta: &Coherent, - gsp_falcon: &Falcon, - sec2_falcon: &Falcon, - ) -> Result>; + gsp_falcon: &'a Falcon, + sec2_falcon: &'a Falcon, + ) -> Result>; /// Performs HAL-specific post-GSP boot tasks. /// diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 0a8b7f763883..9a4bb22578b3 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -18,6 +18,7 @@ fb::FbLayout, gpu::Chipset, gsp::{ + boot::BootUnloadGuard, hal::GspHal, Gsp, GspFwWprMeta, // @@ -31,17 +32,17 @@ impl GspHal for Gh100 { /// /// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles /// the GSP boot internally - no manual GSP reset/boot is needed. - fn boot( + fn boot<'a>( &self, - _gsp: &Gsp, - _dev: &device::Device, - _bar: &Bar0, + _gsp: &'a Gsp, + _dev: &'a device::Device, + _bar: &'a Bar0, _chipset: Chipset, _fb_layout: &FbLayout, _wpr_meta: &Coherent, - _gsp_falcon: &Falcon, - _sec2_falcon: &Falcon, - ) -> Result> { + _gsp_falcon: &'a Falcon, + _sec2_falcon: &'a Falcon, + ) -> Result> { Err(ENOTSUPP) } } diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 53d117ccdddb..a033bc892066 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -32,6 +32,7 @@ }, gpu::Chipset, gsp::{ + boot::BootUnloadGuard, hal::{ GspHal, UnloadBundle, // @@ -254,21 +255,23 @@ fn run_fwsec_frts( struct Tu102; impl GspHal for Tu102 { - fn boot( + fn boot<'a>( &self, - gsp: &Gsp, - dev: &device::Device, - bar: &Bar0, + gsp: &'a Gsp, + dev: &'a device::Device, + bar: &'a Bar0, chipset: Chipset, fb_layout: &FbLayout, wpr_meta: &Coherent, - gsp_falcon: &Falcon, - sec2_falcon: &Falcon, - ) -> Result> { + gsp_falcon: &'a Falcon, + sec2_falcon: &'a Falcon, + ) -> Result> { let bios = Vbios::new(dev, bar)?; - // Try and prepare the unload bundle. If this fails, the GPU will need to be reset - // before the driver can be probed again. + // Try and prepare the unload bundle. + // + // If the unload bundle creation fails, the GPU will need to be reset before the driver can + // be probed again. let unload_bundle = Sec2UnloadBundle::build(dev, bar, chipset, &bios, gsp_falcon, sec2_falcon) .inspect_err(|e| { @@ -279,8 +282,12 @@ fn boot( "The GPU will need to be reset before the driver can bind again.\n" ); }) - .map(crate::gsp::UnloadBundle) - .ok(); + .ok() + .map(crate::gsp::UnloadBundle); + + // Wrap the unload bundle into a drop guard so it is automatically run upon failure. + let unload_guard = + BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, unload_bundle); // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). if !fb_layout.frts.is_empty() { @@ -311,7 +318,7 @@ fn boot( )? .run(dev, bar, sec2_falcon, wpr_meta)?; - Ok(unload_bundle) + Ok(unload_guard) } fn post_boot( From ba47c4604c277b3a3833b90bccdc036ceee83270 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 29 May 2026 02:00:53 +0200 Subject: [PATCH 102/131] drm/tyr: separate driver type from driver data Introduce TyrPlatformDriver as a unit struct for the platform::Driver trait implementation and keep TyrPlatformDriverData for the private driver data. Reviewed-by: Gary Guo Tested-by: Deborah Brouwer Reviewed-by: Boris Brezillon Reviewed-by: Eliot Courtney Signed-off-by: Danilo Krummrich Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260529000106.2257996-2-dakr@kernel.org Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/driver.rs | 10 ++++++---- drivers/gpu/drm/tyr/tyr.rs | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 98732afc096f..6276e9743c32 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -51,6 +51,8 @@ /// Convenience type alias for the DRM device type for this driver. pub(crate) type TyrDrmDevice = drm::Device; +pub(crate) struct TyrPlatformDriver; + #[pin_data(PinnedDrop)] pub(crate) struct TyrPlatformDriverData { _device: ARef, @@ -93,22 +95,22 @@ fn issue_soft_reset(dev: &Device, iomem: &Devres) -> Result { kernel::of_device_table!( OF_TABLE, MODULE_OF_TABLE, - ::IdInfo, + ::IdInfo, [ (of::DeviceId::new(c"rockchip,rk3588-mali"), ()), (of::DeviceId::new(c"arm,mali-valhall-csf"), ()) ] ); -impl platform::Driver for TyrPlatformDriverData { +impl platform::Driver for TyrPlatformDriver { type IdInfo = (); - type Data<'bound> = Self; + type Data<'bound> = TyrPlatformDriverData; const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe<'bound>( pdev: &'bound platform::Device>, _info: Option<&'bound Self::IdInfo>, - ) -> impl PinInit + 'bound { + ) -> impl PinInit, Error> + 'bound { let core_clk = Clk::get(pdev.as_ref(), Some(c"core"))?; let stacks_clk = OptionalClk::get(pdev.as_ref(), Some(c"stacks"))?; let coregroup_clk = OptionalClk::get(pdev.as_ref(), Some(c"coregroup"))?; diff --git a/drivers/gpu/drm/tyr/tyr.rs b/drivers/gpu/drm/tyr/tyr.rs index 9432ddd6b5b8..95cda7b0962f 100644 --- a/drivers/gpu/drm/tyr/tyr.rs +++ b/drivers/gpu/drm/tyr/tyr.rs @@ -5,7 +5,7 @@ //! The name "Tyr" is inspired by Norse mythology, reflecting Arm's tradition of //! naming their GPUs after Nordic mythological figures and places. -use crate::driver::TyrPlatformDriverData; +use crate::driver::TyrPlatformDriver; mod driver; mod file; @@ -14,7 +14,7 @@ mod regs; kernel::module_platform_driver! { - type: TyrPlatformDriverData, + type: TyrPlatformDriver, name: "tyr", authors: ["The Tyr driver authors"], description: "Arm Mali Tyr DRM driver", From f97525b5264441ece3616bbf2a41395d3355590a Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Fri, 29 May 2026 02:00:54 +0200 Subject: [PATCH 103/131] drm/tyr: use IoMem directly instead of Devres Now that IoMem is lifetime-parameterized, use it directly in probe rather than wrapping it in Devres and Arc. The I/O memory mapping is only used during probe and not stored in driver data, so device-managed revocation is unnecessary. This removes the Devres access(dev) pattern from issue_soft_reset(), GpuInfo::new(), and l2_power_on(), simplifying register access. Reviewed-by: Eliot Courtney Reviewed-by: Alexandre Courbot Signed-off-by: Danilo Krummrich Reviewed-by: Boris Brezillon Tested-by: Deborah Brouwer Link: https://patch.msgid.link/20260529000106.2257996-3-dakr@kernel.org Signed-off-by: Alice Ryhl --- drivers/gpu/drm/tyr/driver.rs | 19 ++++++------------- drivers/gpu/drm/tyr/gpu.rs | 17 +++++------------ 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 6276e9743c32..227ea2adccea 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -6,11 +6,9 @@ OptionalClk, // }, device::{ - Bound, Core, Device, // }, - devres::Devres, dma::{ Device as DmaDevice, DmaMask, // @@ -30,7 +28,6 @@ sizes::SZ_2M, sync::{ aref::ARef, - Arc, Mutex, // }, time, // @@ -44,7 +41,7 @@ regs::gpu_control::*, // }; -pub(crate) type IoMem = kernel::io::mem::IoMem<'static, SZ_2M>; +pub(crate) type IoMem<'a> = kernel::io::mem::IoMem<'a, SZ_2M>; pub(crate) struct TyrDrmDriver; @@ -74,15 +71,11 @@ pub(crate) struct TyrDrmDeviceData { pub(crate) gpu_info: GpuInfo, } -fn issue_soft_reset(dev: &Device, iomem: &Devres) -> Result { - let io = (*iomem).access(dev)?; - io.write_reg(GPU_COMMAND::reset(ResetMode::SoftReset)); +fn issue_soft_reset(dev: &Device, iomem: &IoMem<'_>) -> Result { + iomem.write_reg(GPU_COMMAND::reset(ResetMode::SoftReset)); poll::read_poll_timeout( - || { - let io = (*iomem).access(dev)?; - Ok(io.read(GPU_IRQ_RAWSTAT)) - }, + || Ok(iomem.read(GPU_IRQ_RAWSTAT)), |status| status.reset_completed(), time::Delta::from_millis(1), time::Delta::from_millis(100), @@ -123,12 +116,12 @@ fn probe<'bound>( let sram_regulator = Regulator::::get(pdev.as_ref(), c"sram")?; let request = pdev.io_request_by_index(0).ok_or(ENODEV)?; - let iomem = Arc::new(request.iomap_sized::()?.into_devres()?, GFP_KERNEL)?; + let iomem = request.iomap_sized::()?; issue_soft_reset(pdev.as_ref(), &iomem)?; gpu::l2_power_on(pdev.as_ref(), &iomem)?; - let gpu_info = GpuInfo::new(pdev.as_ref(), &iomem)?; + let gpu_info = GpuInfo::new(&iomem); gpu_info.log(pdev.as_ref()); let pa_bits = MMU_FEATURES::from_raw(gpu_info.mmu_features) diff --git a/drivers/gpu/drm/tyr/gpu.rs b/drivers/gpu/drm/tyr/gpu.rs index 652556026f50..592b8bb16eba 100644 --- a/drivers/gpu/drm/tyr/gpu.rs +++ b/drivers/gpu/drm/tyr/gpu.rs @@ -9,7 +9,6 @@ Bound, Device, // }, - devres::Devres, io::{ poll, register::Array, @@ -40,10 +39,8 @@ pub(crate) struct GpuInfo(pub(crate) uapi::drm_panthor_gpu_info); impl GpuInfo { - pub(crate) fn new(dev: &Device, iomem: &Devres) -> Result { - let io = (*iomem).access(dev)?; - - Ok(Self(uapi::drm_panthor_gpu_info { + pub(crate) fn new(io: &IoMem<'_>) -> Self { + Self(uapi::drm_panthor_gpu_info { gpu_id: io.read(GPU_ID).into_raw(), gpu_rev: io.read(REVIDR).into_raw(), csf_id: io.read(CSF_ID).into_raw(), @@ -81,7 +78,7 @@ pub(crate) fn new(dev: &Device, iomem: &Devres) -> Result { pad: 0, //GPU_FEATURES register is not available; it was introduced in arch 11.x. gpu_features: 0, - })) + }) } pub(crate) fn log(&self, dev: &Device) { @@ -163,15 +160,11 @@ struct GpuModels { }]; /// Powers on the l2 block. -pub(crate) fn l2_power_on(dev: &Device, iomem: &Devres) -> Result { - let io = (*iomem).access(dev)?; +pub(crate) fn l2_power_on(dev: &Device, io: &IoMem<'_>) -> Result { io.write_reg(L2_PWRON_LO::zeroed().with_const_request::<1>()); poll::read_poll_timeout( - || { - let io = (*iomem).access(dev)?; - Ok(io.read(L2_READY_LO)) - }, + || Ok(io.read(L2_READY_LO)), |status| status.ready() == 1, Delta::from_millis(1), Delta::from_millis(100), From 3411e9aac6a63e000b10a6a65afb624194e92be3 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:20:49 -0700 Subject: [PATCH 104/131] gpu: nova-core: set DMA mask width based on GPU architecture Replace the hardcoded 47-bit DMA mask with a GPU HAL method that provides the correct value for the architecture. Set the DMA mask in Gpu::new(). Gpu owns all DMA allocations for the device, so no concurrent allocations can exist while the constructor is still running. Signed-off-by: John Hubbard Reviewed-by: Gary Guo Reviewed-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260602032111.224790-2-jhubbard@nvidia.com Co-developed-by: Alexandre Courbot Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/driver.rs | 15 --------------- drivers/gpu/nova-core/gpu.rs | 12 ++++++++++-- drivers/gpu/nova-core/gpu/hal.rs | 8 +++++++- drivers/gpu/nova-core/gpu/hal/gh100.rs | 9 ++++++++- drivers/gpu/nova-core/gpu/hal/tu102.rs | 5 +++++ 5 files changed, 30 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index cff5034c2dcd..ade73da68be5 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -3,8 +3,6 @@ use kernel::{ auxiliary, device::Core, - dma::Device, - dma::DmaMask, pci, pci::{ Class, @@ -38,14 +36,6 @@ pub(crate) struct NovaCore<'bound> { const BAR0_SIZE: usize = SZ_16M; -// For now we only support Ampere which can use up to 47-bit DMA addresses. -// -// TODO: Add an abstraction for this to support newer GPUs which may support -// larger DMA addresses. Limiting these GPUs to smaller address widths won't -// have any adverse affects, unless installed on systems which require larger -// DMA addresses. These systems should be quite rare. -const GPU_DMA_BITS: u32 = 47; - pub(crate) type Bar0 = kernel::io::Mmio; kernel::pci_device_table!( @@ -88,11 +78,6 @@ fn probe<'bound>( pdev.enable_device_mem()?; pdev.set_master(); - // SAFETY: No concurrent DMA allocations or mappings can be made because - // the device is still being probed and therefore isn't being used by - // other threads of execution. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::())? }; - Ok(try_pin_init!(NovaCore { bar: pdev.iomap_region_sized::(0, c"nova-core/bar0")?, // TODO: Use `&bar` self-referential pin-init syntax once available. diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index aed992488db3..38c75df77e16 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -2,6 +2,7 @@ use kernel::{ device, + dma::Device, fmt, io::Io, num::Bounded, @@ -269,7 +270,7 @@ pub(crate) struct Gpu<'gpu> { impl<'gpu> Gpu<'gpu> { pub(crate) fn new( - pdev: &'gpu pci::Device, + pdev: &'gpu pci::Device>, bar: &'gpu Bar0, ) -> impl PinInit + 'gpu { try_pin_init!(Self { @@ -280,7 +281,14 @@ pub(crate) fn new( // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. _: { - hal::gpu_hal(spec.chipset).wait_gfw_boot_completion(bar) + let hal = hal::gpu_hal(spec.chipset); + let dma_mask = hal.dma_mask(); + + // SAFETY: `Gpu` owns all DMA allocations for this device, and we are + // still constructing it, so no concurrent DMA allocations can exist. + unsafe { pdev.dma_set_mask_and_coherent(dma_mask)? }; + + hal.wait_gfw_boot_completion(bar) .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?; }, diff --git a/drivers/gpu/nova-core/gpu/hal.rs b/drivers/gpu/nova-core/gpu/hal.rs index 788de20ab5d3..0b636b713593 100644 --- a/drivers/gpu/nova-core/gpu/hal.rs +++ b/drivers/gpu/nova-core/gpu/hal.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 -use kernel::prelude::*; +use kernel::{ + dma::DmaMask, + prelude::*, // +}; use crate::{ driver::Bar0, @@ -16,6 +19,9 @@ pub(crate) trait GpuHal { /// Waits for GFW_BOOT completion if required by this hardware family. fn wait_gfw_boot_completion(&self, bar: &Bar0) -> Result; + + /// Returns the DMA mask for the current architecture. + fn dma_mask(&self) -> DmaMask; } pub(super) fn gpu_hal(chipset: Chipset) -> &'static dyn GpuHal { diff --git a/drivers/gpu/nova-core/gpu/hal/gh100.rs b/drivers/gpu/nova-core/gpu/hal/gh100.rs index 1ed5bccdda1d..41fbabb04ff8 100644 --- a/drivers/gpu/nova-core/gpu/hal/gh100.rs +++ b/drivers/gpu/nova-core/gpu/hal/gh100.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 -use kernel::prelude::*; +use kernel::{ + dma::DmaMask, + prelude::*, // +}; use crate::driver::Bar0; @@ -12,6 +15,10 @@ impl GpuHal for Gh100 { fn wait_gfw_boot_completion(&self, _bar: &Bar0) -> Result { Ok(()) } + + fn dma_mask(&self) -> DmaMask { + DmaMask::new::<52>() + } } const GH100: Gh100 = Gh100; diff --git a/drivers/gpu/nova-core/gpu/hal/tu102.rs b/drivers/gpu/nova-core/gpu/hal/tu102.rs index 08dd4434bd72..2881ab03dbcd 100644 --- a/drivers/gpu/nova-core/gpu/hal/tu102.rs +++ b/drivers/gpu/nova-core/gpu/hal/tu102.rs @@ -19,6 +19,7 @@ //! Note that the devinit sequence also needs to run during suspend/resume. use kernel::{ + dma::DmaMask, io::{ poll::read_poll_timeout, Io, // @@ -80,6 +81,10 @@ fn wait_gfw_boot_completion(&self, bar: &Bar0) -> Result { ) .map(|_| ()) } + + fn dma_mask(&self) -> DmaMask { + DmaMask::new::<47>() + } } const TU102: Tu102 = Tu102; From ee9414e6055bf3249f535bfe0f7d4e3c5a3e4b13 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:20:50 -0700 Subject: [PATCH 105/131] gpu: nova-core: Hopper/Blackwell: new location for PCI config mirror Hopper and Blackwell GPUs moved the PCI config space mirror from 0x088000 to 0x092000. Select the correct address per architecture when building the GSP system info command. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-3-jhubbard@nvidia.com Co-developed-by: Alexandre Courbot Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 7 +++++++ drivers/gpu/nova-core/gpu/hal.rs | 5 +++++ drivers/gpu/nova-core/gpu/hal/gh100.rs | 9 +++++++++ drivers/gpu/nova-core/gpu/hal/tu102.rs | 9 +++++++++ drivers/gpu/nova-core/gsp/boot.rs | 2 +- drivers/gpu/nova-core/gsp/commands.rs | 8 +++++--- drivers/gpu/nova-core/gsp/fw/commands.rs | 15 +++++++++++---- 7 files changed, 47 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 38c75df77e16..7dd736e5b190 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 +use core::ops::Range; + use kernel::{ device, dma::Device, @@ -134,6 +136,11 @@ pub(crate) const fn arch(self) -> Architecture { pub(crate) const fn needs_fwsec_bootloader(self) -> bool { matches!(self.arch(), Architecture::Turing) || matches!(self, Self::GA100) } + + /// Returns the address range of the PCI config mirror space. + pub(crate) fn pci_config_mirror_range(self) -> Range { + hal::gpu_hal(self).pci_config_mirror_range() + } } // TODO diff --git a/drivers/gpu/nova-core/gpu/hal.rs b/drivers/gpu/nova-core/gpu/hal.rs index 0b636b713593..cd833bd49b9b 100644 --- a/drivers/gpu/nova-core/gpu/hal.rs +++ b/drivers/gpu/nova-core/gpu/hal.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 +use core::ops::Range; + use kernel::{ dma::DmaMask, prelude::*, // @@ -22,6 +24,9 @@ pub(crate) trait GpuHal { /// Returns the DMA mask for the current architecture. fn dma_mask(&self) -> DmaMask; + + /// Returns the address range of the PCI config mirror space. + fn pci_config_mirror_range(&self) -> Range; } pub(super) fn gpu_hal(chipset: Chipset) -> &'static dyn GpuHal { diff --git a/drivers/gpu/nova-core/gpu/hal/gh100.rs b/drivers/gpu/nova-core/gpu/hal/gh100.rs index 41fbabb04ff8..17778a618900 100644 --- a/drivers/gpu/nova-core/gpu/hal/gh100.rs +++ b/drivers/gpu/nova-core/gpu/hal/gh100.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 +use core::ops::Range; + use kernel::{ dma::DmaMask, prelude::*, // @@ -19,6 +21,13 @@ fn wait_gfw_boot_completion(&self, _bar: &Bar0) -> Result { fn dma_mask(&self) -> DmaMask { DmaMask::new::<52>() } + + fn pci_config_mirror_range(&self) -> Range { + const PCI_CONFIG_MIRROR_START: u32 = 0x092000; + const PCI_CONFIG_MIRROR_SIZE: u32 = 0x001000; + + PCI_CONFIG_MIRROR_START..PCI_CONFIG_MIRROR_START + PCI_CONFIG_MIRROR_SIZE + } } const GH100: Gh100 = Gh100; diff --git a/drivers/gpu/nova-core/gpu/hal/tu102.rs b/drivers/gpu/nova-core/gpu/hal/tu102.rs index 2881ab03dbcd..125478bfe07a 100644 --- a/drivers/gpu/nova-core/gpu/hal/tu102.rs +++ b/drivers/gpu/nova-core/gpu/hal/tu102.rs @@ -18,6 +18,8 @@ //! //! Note that the devinit sequence also needs to run during suspend/resume. +use core::ops::Range; + use kernel::{ dma::DmaMask, io::{ @@ -85,6 +87,13 @@ fn wait_gfw_boot_completion(&self, bar: &Bar0) -> Result { fn dma_mask(&self) -> DmaMask { DmaMask::new::<47>() } + + fn pci_config_mirror_range(&self) -> Range { + const PCI_CONFIG_MIRROR_START: u32 = 0x088000; + const PCI_CONFIG_MIRROR_SIZE: u32 = 0x001000; + + PCI_CONFIG_MIRROR_START..PCI_CONFIG_MIRROR_START + PCI_CONFIG_MIRROR_SIZE + } } const TU102: Tu102 = Tu102; diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 087ee59da6d9..8c316fa2e585 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -144,7 +144,7 @@ pub(crate) fn boot( dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(bar),); self.cmdq - .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev))?; + .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; self.cmdq .send_command_no_wait(bar, commands::SetRegistry::new())?; diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 3a365455d10c..f84de9f4f045 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -19,6 +19,7 @@ }; use crate::{ + gpu::Chipset, gsp::{ cmdq::{ Cmdq, @@ -37,12 +38,13 @@ /// The `GspSetSystemInfo` command. pub(crate) struct SetSystemInfo<'a> { pdev: &'a pci::Device, + chipset: Chipset, } impl<'a> SetSystemInfo<'a> { /// Creates a new `GspSetSystemInfo` command using the parameters of `pdev`. - pub(crate) fn new(pdev: &'a pci::Device) -> Self { - Self { pdev } + pub(crate) fn new(pdev: &'a pci::Device, chipset: Chipset) -> Self { + Self { pdev, chipset } } } @@ -53,7 +55,7 @@ impl<'a> CommandToGsp for SetSystemInfo<'a> { type InitError = Error; fn init(&self) -> impl Init { - Self::Command::init(self.pdev) + Self::Command::init(self.pdev, self.chipset) } } diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index 42985d446bae..7bcc41fc7fa0 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -11,7 +11,10 @@ }, // }; -use crate::gsp::GSP_PAGE_SIZE; +use crate::{ + gpu::Chipset, + gsp::GSP_PAGE_SIZE, // +}; use super::bindings; @@ -25,8 +28,12 @@ pub(crate) struct GspSetSystemInfo { impl GspSetSystemInfo { /// Returns an in-place initializer for the `GspSetSystemInfo` command. #[allow(non_snake_case)] - pub(crate) fn init<'a>(dev: &'a pci::Device) -> impl Init + 'a { + pub(crate) fn init<'a>( + dev: &'a pci::Device, + chipset: Chipset, + ) -> impl Init + 'a { type InnerGspSystemInfo = bindings::GspSystemInfo; + let pci_config_mirror_range = chipset.pci_config_mirror_range(); let init_inner = try_init!(InnerGspSystemInfo { gpuPhysAddr: dev.resource_start(0)?, gpuPhysFbAddr: dev.resource_start(1)?, @@ -36,8 +43,8 @@ pub(crate) fn init<'a>(dev: &'a pci::Device) -> impl Init Date: Mon, 1 Jun 2026 20:20:51 -0700 Subject: [PATCH 106/131] gpu: nova-core: Blackwell: compute PMU-reserved framebuffer size GSP boot needs to know how much framebuffer memory is reserved for the PMU. Compute it per architecture: Blackwell dGPUs reserve a non-zero amount, earlier architectures leave it at zero, matching Open RM behavior. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-4-jhubbard@nvidia.com Co-developed-by: Alexandre Courbot Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 3 ++ drivers/gpu/nova-core/fb/hal.rs | 12 +++--- drivers/gpu/nova-core/fb/hal/ga100.rs | 5 +++ drivers/gpu/nova-core/fb/hal/ga102.rs | 7 +++- drivers/gpu/nova-core/fb/hal/gb100.rs | 57 +++++++++++++++++++++++++++ drivers/gpu/nova-core/fb/hal/tu102.rs | 9 +++++ drivers/gpu/nova-core/gsp/fw.rs | 1 + 7 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 drivers/gpu/nova-core/fb/hal/gb100.rs diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 1fb65d4eb290..d7a4dc944131 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -165,6 +165,8 @@ pub(crate) struct FbLayout { pub(crate) wpr2: FbRange, pub(crate) heap: FbRange, pub(crate) vf_partition_count: u8, + /// PMU reserved memory size, in bytes. + pub(crate) pmu_reserved_size: u32, } impl FbLayout { @@ -265,6 +267,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result< wpr2, heap, vf_partition_count: 0, + pmu_reserved_size: hal.pmu_reserved_size(), }) } } diff --git a/drivers/gpu/nova-core/fb/hal.rs b/drivers/gpu/nova-core/fb/hal.rs index 8b192a503363..b45784ad5f2e 100644 --- a/drivers/gpu/nova-core/fb/hal.rs +++ b/drivers/gpu/nova-core/fb/hal.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::prelude::*; @@ -12,6 +13,7 @@ mod ga100; mod ga102; +mod gb100; mod tu102; pub(crate) trait FbHal { @@ -29,6 +31,9 @@ pub(crate) trait FbHal { /// Returns the VRAM size, in bytes. fn vidmem_size(&self, bar: &Bar0) -> u64; + /// Returns the amount of VRAM to reserve for the PMU. + fn pmu_reserved_size(&self) -> u32; + /// Returns the FRTS size, in bytes. fn frts_size(&self) -> u64; } @@ -38,10 +43,7 @@ pub(super) fn fb_hal(chipset: Chipset) -> &'static dyn FbHal { match chipset.arch() { Architecture::Turing => tu102::TU102_HAL, Architecture::Ampere if chipset == Chipset::GA100 => ga100::GA100_HAL, - Architecture::Ampere => ga102::GA102_HAL, - Architecture::Ada - | Architecture::Hopper - | Architecture::BlackwellGB10x - | Architecture::BlackwellGB20x => ga102::GA102_HAL, + Architecture::Ampere | Architecture::Ada | Architecture::Hopper => ga102::GA102_HAL, + Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => gb100::GB100_HAL, } } diff --git a/drivers/gpu/nova-core/fb/hal/ga100.rs b/drivers/gpu/nova-core/fb/hal/ga100.rs index 2f5871d915c3..0f5132aa9c31 100644 --- a/drivers/gpu/nova-core/fb/hal/ga100.rs +++ b/drivers/gpu/nova-core/fb/hal/ga100.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ io::Io, @@ -67,6 +68,10 @@ fn vidmem_size(&self, bar: &Bar0) -> u64 { super::tu102::vidmem_size_gp102(bar) } + fn pmu_reserved_size(&self) -> u32 { + super::tu102::pmu_reserved_size_tu102() + } + // GA100 is a special case where its FRTS region exists, but is empty. We // return a size of 0 because we still need to record where the region starts. fn frts_size(&self) -> u64 { diff --git a/drivers/gpu/nova-core/fb/hal/ga102.rs b/drivers/gpu/nova-core/fb/hal/ga102.rs index 3bb66f64bef7..17a2fef1ad44 100644 --- a/drivers/gpu/nova-core/fb/hal/ga102.rs +++ b/drivers/gpu/nova-core/fb/hal/ga102.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ io::Io, @@ -11,7 +12,7 @@ regs, // }; -fn vidmem_size_ga102(bar: &Bar0) -> u64 { +pub(super) fn vidmem_size_ga102(bar: &Bar0) -> u64 { bar.read(regs::NV_USABLE_FB_SIZE_IN_MB).usable_fb_size() } @@ -36,6 +37,10 @@ fn vidmem_size(&self, bar: &Bar0) -> u64 { vidmem_size_ga102(bar) } + fn pmu_reserved_size(&self) -> u32 { + super::tu102::pmu_reserved_size_tu102() + } + fn frts_size(&self) -> u64 { super::tu102::frts_size_tu102() } diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs new file mode 100644 index 000000000000..c78027c26a9e --- /dev/null +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Blackwell framebuffer HAL. + +use kernel::{ + prelude::*, + ptr::{ + const_align_up, + Alignment, // + }, + sizes::*, // +}; + +use crate::{ + driver::Bar0, + fb::hal::FbHal, + num::usize_into_u32, // +}; + +struct Gb100; + +const fn pmu_reserved_size_gb100() -> u32 { + usize_into_u32::<{ const_align_up(SZ_8M + SZ_16M + SZ_4K, Alignment::new::()).unwrap() }>( + ) +} + +impl FbHal for Gb100 { + fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + super::ga100::read_sysmem_flush_page_ga100(bar) + } + + fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + super::ga100::write_sysmem_flush_page_ga100(bar, addr); + + Ok(()) + } + + fn supports_display(&self, bar: &Bar0) -> bool { + super::ga100::display_enabled_ga100(bar) + } + + fn vidmem_size(&self, bar: &Bar0) -> u64 { + super::ga102::vidmem_size_ga102(bar) + } + + fn pmu_reserved_size(&self) -> u32 { + pmu_reserved_size_gb100() + } + + fn frts_size(&self) -> u64 { + super::tu102::frts_size_tu102() + } +} + +const GB100: Gb100 = Gb100; +pub(super) const GB100_HAL: &dyn FbHal = &GB100; diff --git a/drivers/gpu/nova-core/fb/hal/tu102.rs b/drivers/gpu/nova-core/fb/hal/tu102.rs index 22c174bf1472..1755bbc27866 100644 --- a/drivers/gpu/nova-core/fb/hal/tu102.rs +++ b/drivers/gpu/nova-core/fb/hal/tu102.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ io::Io, @@ -39,6 +40,10 @@ pub(super) fn vidmem_size_gp102(bar: &Bar0) -> u64 { .usable_fb_size() } +pub(super) const fn pmu_reserved_size_tu102() -> u32 { + 0 +} + pub(super) const fn frts_size_tu102() -> u64 { u64::SZ_1M } @@ -62,6 +67,10 @@ fn vidmem_size(&self, bar: &Bar0) -> u64 { vidmem_size_gp102(bar) } + fn pmu_reserved_size(&self) -> u32 { + pmu_reserved_size_tu102() + } + fn frts_size(&self) -> u64 { frts_size_tu102() } diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 33c9f5860771..919d3ab00075 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -247,6 +247,7 @@ pub(crate) fn new<'a>( fbSize: fb_layout.fb.end - fb_layout.fb.start, vgaWorkspaceOffset: fb_layout.vga_workspace.start, vgaWorkspaceSize: fb_layout.vga_workspace.end - fb_layout.vga_workspace.start, + pmuReservedSize: fb_layout.pmu_reserved_size, ..Zeroable::init_zeroed() }); From fcdd74aa8ca6ca7f4922d92da932891996734d15 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:20:52 -0700 Subject: [PATCH 107/131] gpu: nova-core: Hopper/Blackwell: larger non-WPR heap Hopper and Blackwell need a larger non-WPR heap than the 1 MiB that earlier architectures use. Hopper and Blackwell GB10x need 2 MiB, while Blackwell GB20x needs 2 MiB + 128 KiB. These sizes diverge by family, so give Hopper and each Blackwell family its own framebuffer HAL and select the non-WPR heap size per chipset family. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-5-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb.rs | 5 ++- drivers/gpu/nova-core/fb/hal.rs | 11 ++++-- drivers/gpu/nova-core/fb/hal/ga100.rs | 4 +++ drivers/gpu/nova-core/fb/hal/ga102.rs | 4 +++ drivers/gpu/nova-core/fb/hal/gb100.rs | 9 +++-- drivers/gpu/nova-core/fb/hal/gb202.rs | 52 +++++++++++++++++++++++++++ drivers/gpu/nova-core/fb/hal/gh100.rs | 50 ++++++++++++++++++++++++++ drivers/gpu/nova-core/fb/hal/tu102.rs | 8 +++++ 8 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 drivers/gpu/nova-core/fb/hal/gb202.rs create mode 100644 drivers/gpu/nova-core/fb/hal/gh100.rs diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index d7a4dc944131..0aaee718c2c3 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -252,9 +252,8 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result< }; let heap = { - const HEAP_SIZE: u64 = u64::SZ_1M; - - FbRange(wpr2.start - HEAP_SIZE..wpr2.start) + let heap_size = u64::from(hal.non_wpr_heap_size()); + FbRange(wpr2.start - heap_size..wpr2.start) }; Ok(Self { diff --git a/drivers/gpu/nova-core/fb/hal.rs b/drivers/gpu/nova-core/fb/hal.rs index b45784ad5f2e..be9e75f990f0 100644 --- a/drivers/gpu/nova-core/fb/hal.rs +++ b/drivers/gpu/nova-core/fb/hal.rs @@ -14,6 +14,8 @@ mod ga100; mod ga102; mod gb100; +mod gb202; +mod gh100; mod tu102; pub(crate) trait FbHal { @@ -34,6 +36,9 @@ pub(crate) trait FbHal { /// Returns the amount of VRAM to reserve for the PMU. fn pmu_reserved_size(&self) -> u32; + /// Returns the non-WPR heap size for this chipset, in bytes. + fn non_wpr_heap_size(&self) -> u32; + /// Returns the FRTS size, in bytes. fn frts_size(&self) -> u64; } @@ -43,7 +48,9 @@ pub(super) fn fb_hal(chipset: Chipset) -> &'static dyn FbHal { match chipset.arch() { Architecture::Turing => tu102::TU102_HAL, Architecture::Ampere if chipset == Chipset::GA100 => ga100::GA100_HAL, - Architecture::Ampere | Architecture::Ada | Architecture::Hopper => ga102::GA102_HAL, - Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => gb100::GB100_HAL, + Architecture::Ampere | Architecture::Ada => ga102::GA102_HAL, + Architecture::Hopper => gh100::GH100_HAL, + Architecture::BlackwellGB10x => gb100::GB100_HAL, + Architecture::BlackwellGB20x => gb202::GB202_HAL, } } diff --git a/drivers/gpu/nova-core/fb/hal/ga100.rs b/drivers/gpu/nova-core/fb/hal/ga100.rs index 0f5132aa9c31..af95f1bdd273 100644 --- a/drivers/gpu/nova-core/fb/hal/ga100.rs +++ b/drivers/gpu/nova-core/fb/hal/ga100.rs @@ -72,6 +72,10 @@ fn pmu_reserved_size(&self) -> u32 { super::tu102::pmu_reserved_size_tu102() } + fn non_wpr_heap_size(&self) -> u32 { + super::tu102::non_wpr_heap_size_tu102() + } + // GA100 is a special case where its FRTS region exists, but is empty. We // return a size of 0 because we still need to record where the region starts. fn frts_size(&self) -> u64 { diff --git a/drivers/gpu/nova-core/fb/hal/ga102.rs b/drivers/gpu/nova-core/fb/hal/ga102.rs index 17a2fef1ad44..e06dbb08349e 100644 --- a/drivers/gpu/nova-core/fb/hal/ga102.rs +++ b/drivers/gpu/nova-core/fb/hal/ga102.rs @@ -41,6 +41,10 @@ fn pmu_reserved_size(&self) -> u32 { super::tu102::pmu_reserved_size_tu102() } + fn non_wpr_heap_size(&self) -> u32 { + super::tu102::non_wpr_heap_size_tu102() + } + fn frts_size(&self) -> u64 { super::tu102::frts_size_tu102() } diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index c78027c26a9e..8d63350abf8a 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -//! Blackwell framebuffer HAL. +//! Blackwell GB10x framebuffer HAL. use kernel::{ prelude::*, @@ -20,7 +20,7 @@ struct Gb100; -const fn pmu_reserved_size_gb100() -> u32 { +pub(super) const fn pmu_reserved_size_gb100() -> u32 { usize_into_u32::<{ const_align_up(SZ_8M + SZ_16M + SZ_4K, Alignment::new::()).unwrap() }>( ) } @@ -48,6 +48,11 @@ fn pmu_reserved_size(&self) -> u32 { pmu_reserved_size_gb100() } + fn non_wpr_heap_size(&self) -> u32 { + // Non-WPR heap for GB10x (see Open RM: kgspGetNonWprHeapSize, GB100/GB102). + u32::SZ_2M + } + fn frts_size(&self) -> u64 { super::tu102::frts_size_tu102() } diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs new file mode 100644 index 000000000000..542c1d7429e9 --- /dev/null +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Blackwell GB20x framebuffer HAL. + +use kernel::{ + prelude::*, + sizes::SizeConstants, // +}; + +use crate::{ + driver::Bar0, + fb::hal::FbHal, // +}; + +struct Gb202; + +impl FbHal for Gb202 { + fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + super::ga100::read_sysmem_flush_page_ga100(bar) + } + + fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + super::ga100::write_sysmem_flush_page_ga100(bar, addr); + + Ok(()) + } + + fn supports_display(&self, bar: &Bar0) -> bool { + super::ga100::display_enabled_ga100(bar) + } + + fn vidmem_size(&self, bar: &Bar0) -> u64 { + super::ga102::vidmem_size_ga102(bar) + } + + fn pmu_reserved_size(&self) -> u32 { + super::gb100::pmu_reserved_size_gb100() + } + + fn non_wpr_heap_size(&self) -> u32 { + // Non-WPR heap for GB20x (see Open RM: kgspGetNonWprHeapSize, GB202+). + u32::SZ_2M + u32::SZ_128K + } + + fn frts_size(&self) -> u64 { + super::tu102::frts_size_tu102() + } +} + +const GB202: Gb202 = Gb202; +pub(super) const GB202_HAL: &dyn FbHal = &GB202; diff --git a/drivers/gpu/nova-core/fb/hal/gh100.rs b/drivers/gpu/nova-core/fb/hal/gh100.rs new file mode 100644 index 000000000000..8f79c72b1823 --- /dev/null +++ b/drivers/gpu/nova-core/fb/hal/gh100.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::{ + prelude::*, + sizes::SizeConstants, // +}; + +use crate::{ + driver::Bar0, + fb::hal::FbHal, // +}; + +struct Gh100; + +impl FbHal for Gh100 { + fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + super::ga100::read_sysmem_flush_page_ga100(bar) + } + + fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + super::ga100::write_sysmem_flush_page_ga100(bar, addr); + + Ok(()) + } + + fn supports_display(&self, bar: &Bar0) -> bool { + super::ga100::display_enabled_ga100(bar) + } + + fn vidmem_size(&self, bar: &Bar0) -> u64 { + super::ga102::vidmem_size_ga102(bar) + } + + fn pmu_reserved_size(&self) -> u32 { + super::tu102::pmu_reserved_size_tu102() + } + + fn non_wpr_heap_size(&self) -> u32 { + // Non-WPR heap for Hopper (see Open RM: kgspCalculateFbLayout_GH100). + u32::SZ_2M + } + + fn frts_size(&self) -> u64 { + super::tu102::frts_size_tu102() + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn FbHal = &GH100; diff --git a/drivers/gpu/nova-core/fb/hal/tu102.rs b/drivers/gpu/nova-core/fb/hal/tu102.rs index 1755bbc27866..62d9357987f7 100644 --- a/drivers/gpu/nova-core/fb/hal/tu102.rs +++ b/drivers/gpu/nova-core/fb/hal/tu102.rs @@ -44,6 +44,10 @@ pub(super) const fn pmu_reserved_size_tu102() -> u32 { 0 } +pub(super) const fn non_wpr_heap_size_tu102() -> u32 { + u32::SZ_1M +} + pub(super) const fn frts_size_tu102() -> u64 { u64::SZ_1M } @@ -71,6 +75,10 @@ fn pmu_reserved_size(&self) -> u32 { pmu_reserved_size_tu102() } + fn non_wpr_heap_size(&self) -> u32 { + non_wpr_heap_size_tu102() + } + fn frts_size(&self) -> u64 { frts_size_tu102() } From f66287cf155e47b604118ee1e7731ff634d8dbe9 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:20:53 -0700 Subject: [PATCH 108/131] gpu: nova-core: Hopper/Blackwell: larger WPR2 (GSP) heap The GSP-RM boot working memory portion of the WPR2 heap must be larger on Hopper and later GPUs than on Turing, Ampere, and Ada. Select the larger value for those generations. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-6-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/fw.rs | 20 +++++++++++++------ .../gpu/nova-core/gsp/fw/r570_144/bindings.rs | 1 + 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 919d3ab00075..0c54e8bf4bb3 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. pub(crate) mod commands; mod r570_144; @@ -29,7 +30,10 @@ use crate::{ fb::FbLayout, firmware::gsp::GspFirmware, - gpu::Chipset, + gpu::{ + Architecture, + Chipset, // + }, gsp::{ cmdq::Cmdq, // GSP_PAGE_SIZE, @@ -106,11 +110,15 @@ enum GspFwHeapParams {} impl GspFwHeapParams { /// Returns the amount of GSP-RM heap memory used during GSP-RM boot and initialization (up to /// and including the first client subdevice allocation). - fn base_rm_size(_chipset: Chipset) -> u64 { - // TODO: this needs to be updated to return the correct value for Hopper+ once support for - // them is added: - // u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100) - u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X) + fn base_rm_size(chipset: Chipset) -> u64 { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => { + u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X) + } + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100) + } + } } /// Returns the amount of heap memory required to support a single channel allocation. diff --git a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs index f82ed097b283..1d592bd3f9ed 100644 --- a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs +++ b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs @@ -37,6 +37,7 @@ fn fmt(&self, fmt: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { pub const GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2: u32 = 0; pub const GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL: u32 = 23068672; pub const GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X: u32 = 8388608; +pub const GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100: u32 = 14680064; pub const GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB: u32 = 98304; pub const GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE: u32 = 100663296; pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB: u32 = 64; From a5bf742bc28a4e064059c8d137970e56669a21a0 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:20:54 -0700 Subject: [PATCH 109/131] gpu: nova-core: Blackwell: use correct sysmem flush registers Blackwell GPUs moved the sysmem flush page registers away from the Ampere/Ada location. GB10x routes the flush through a pair of HSHUB0 register sets (primary and egress) that must both be programmed to the same address. GB20x routes it through FBHUB0. Define these registers relative to their HSHUB0 and FBHUB0 bases, as Open RM does, and implement the flush paths in the GB10x and GB20x framebuffer HALs. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-7-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fb/hal/gb100.rs | 66 +++++++++++++++++++++++++-- drivers/gpu/nova-core/fb/hal/gb202.rs | 49 ++++++++++++++++++-- drivers/gpu/nova-core/regs.rs | 45 ++++++++++++++++++ 3 files changed, 154 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index 8d63350abf8a..ecea4ff446ff 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -4,6 +4,14 @@ //! Blackwell GB10x framebuffer HAL. use kernel::{ + io::{ + register::{ + RegisterBase, + WithBase, // + }, + Io, // + }, + num::Bounded, prelude::*, ptr::{ const_align_up, @@ -15,11 +23,61 @@ use crate::{ driver::Bar0, fb::hal::FbHal, - num::usize_into_u32, // + num::usize_into_u32, + regs, // }; struct Gb100; +impl RegisterBase for Gb100 { + const BASE: usize = 0x0087_0000; +} + +fn read_sysmem_flush_page_gb100(bar: &Bar0) -> u64 { + let lo = u64::from( + bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::()) + .adr(), + ); + let hi = u64::from( + bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::()) + .adr(), + ); + + lo | (hi << 32) +} + +/// Write the sysmem flush page address through the GB10x HSHUB0 registers. +/// +/// Both the primary and EG (egress) register pairs must be programmed to the same address, +/// as required by hardware. +fn write_sysmem_flush_page_gb100(bar: &Bar0, addr: Bounded) { + // CAST: lower 32 bits. Hardware ignores bits 7:0. + let addr_lo = *addr as u32; + let addr_hi = addr.shr::<32, 20>().cast::(); + + // Write HI first. The hardware will trigger the flush on the LO write. + + // Primary HSHUB pair. + bar.write( + regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::(), + regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi), + ); + bar.write( + regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::(), + regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo), + ); + + // EG (egress) pair -- must match the primary pair. + bar.write( + regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::of::(), + regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi), + ); + bar.write( + regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::of::(), + regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo), + ); +} + pub(super) const fn pmu_reserved_size_gb100() -> u32 { usize_into_u32::<{ const_align_up(SZ_8M + SZ_16M + SZ_4K, Alignment::new::()).unwrap() }>( ) @@ -27,11 +85,13 @@ pub(super) const fn pmu_reserved_size_gb100() -> u32 { impl FbHal for Gb100 { fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { - super::ga100::read_sysmem_flush_page_ga100(bar) + read_sysmem_flush_page_gb100(bar) } fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { - super::ga100::write_sysmem_flush_page_ga100(bar, addr); + let addr = Bounded::::try_new(addr).ok_or(EINVAL)?; + + write_sysmem_flush_page_gb100(bar, addr); Ok(()) } diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs index 542c1d7429e9..fa5c3f7f2b2e 100644 --- a/drivers/gpu/nova-core/fb/hal/gb202.rs +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -4,24 +4,67 @@ //! Blackwell GB20x framebuffer HAL. use kernel::{ + io::{ + register::{ + RegisterBase, + WithBase, // + }, + Io, // + }, + num::Bounded, prelude::*, sizes::SizeConstants, // }; use crate::{ driver::Bar0, - fb::hal::FbHal, // + fb::hal::FbHal, + regs, // }; struct Gb202; +impl RegisterBase for Gb202 { + const BASE: usize = 0x008a_0000; +} + +fn read_sysmem_flush_page_gb202(bar: &Bar0) -> u64 { + let lo = u64::from( + bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::()) + .adr(), + ); + let hi = u64::from( + bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::()) + .adr(), + ); + + lo | (hi << 32) +} + +/// Write the sysmem flush page address through the GB20x FBHUB0 registers. +fn write_sysmem_flush_page_gb202(bar: &Bar0, addr: Bounded) { + // Write HI first. The hardware will trigger the flush on the LO write. + bar.write( + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::(), + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed() + .with_adr(addr.shr::<32, 20>().cast::()), + ); + bar.write( + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::(), + // CAST: lower 32 bits. Hardware ignores bits 7:0. + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32), + ); +} + impl FbHal for Gb202 { fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { - super::ga100::read_sysmem_flush_page_ga100(bar) + read_sysmem_flush_page_gb202(bar) } fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { - super::ga100::write_sysmem_flush_page_ga100(bar, addr); + let addr = Bounded::::try_new(addr).ok_or(EINVAL)?; + + write_sysmem_flush_page_gb202(bar, addr); Ok(()) } diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 356fbf364ea5..b39647684dd1 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ io::{ @@ -147,6 +148,50 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { } } +/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM). +/// +/// The base is provided by the GB10x framebuffer HAL. +pub(crate) struct Hshub0Base(()); + +/// Base of the GB20x FBHUB0 register window (`NV_FBHUB0_PRI_BASE` in Open RM). +/// +/// The base is provided by the GB20x framebuffer HAL. +pub(crate) struct Fbhub0Base(()); + +register! { + // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar + // through a primary and an EG (egress) pair that must both be programmed to the same + // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed + // HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here. + pub(crate) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 { + 31:0 adr => u32; + } + + pub(crate) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 { + 19:0 adr; + } + + pub(crate) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 { + 31:0 adr => u32; + } + + pub(crate) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 { + 19:0 adr; + } + + // GB20x sysmem flush registers, relative to the FBHUB0 base. Unlike the older + // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers which encode the address with an 8-bit + // right-shift, these take the raw address split into lower and upper halves. Hardware + // ignores bits 7:0 of the LO register. + pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Fbhub0Base + 0x00001d58 { + 31:0 adr => u32; + } + + pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Fbhub0Base + 0x00001d5c { + 19:0 adr; + } +} + impl NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE { /// Returns the usable framebuffer size, in bytes. pub(crate) fn usable_fb_size(self) -> u64 { From bd581ff381443db71c86d5be56f3f70b0030058f Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:20:55 -0700 Subject: [PATCH 110/131] gpu: nova-core: don't assume 64-bit firmware images Introduce a single ELF format abstraction that ties each ELF header type to its matching section-header type. This keeps the shared section parser ready for upcoming ELF32 support and avoids mixing 32-bit and 64-bit ELF layouts by mistake. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-8-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware.rs | 108 +++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 3aac073efee2..38088e950980 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. //! Contains structures and functions dedicated to the parsing, building and patching of firmwares //! to be loaded into a given execution unit. @@ -467,17 +468,72 @@ mod elf { transmute::FromBytes, // }; + /// Trait to abstract over ELF header differences. + trait ElfHeader: FromBytes { + fn shnum(&self) -> u16; + fn shoff(&self) -> u64; + fn shstrndx(&self) -> u16; + } + + /// Trait to abstract over ELF section-header differences. + trait ElfSectionHeader: FromBytes { + fn name(&self) -> u32; + fn offset(&self) -> u64; + fn size(&self) -> u64; + } + + /// Trait describing a matching ELF header and section-header format. + trait ElfFormat { + type Header: ElfHeader; + type SectionHeader: ElfSectionHeader; + } + /// Newtype to provide a [`FromBytes`] implementation. #[repr(transparent)] struct Elf64Hdr(bindings::elf64_hdr); // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. unsafe impl FromBytes for Elf64Hdr {} + impl ElfHeader for Elf64Hdr { + fn shnum(&self) -> u16 { + self.0.e_shnum + } + + fn shoff(&self) -> u64 { + self.0.e_shoff + } + + fn shstrndx(&self) -> u16 { + self.0.e_shstrndx + } + } + #[repr(transparent)] struct Elf64SHdr(bindings::elf64_shdr); // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. unsafe impl FromBytes for Elf64SHdr {} + impl ElfSectionHeader for Elf64SHdr { + fn name(&self) -> u32 { + self.0.sh_name + } + + fn offset(&self) -> u64 { + self.0.sh_offset + } + + fn size(&self) -> u64 { + self.0.sh_size + } + } + + struct Elf64Format; + + impl ElfFormat for Elf64Format { + type Header = Elf64Hdr; + type SectionHeader = Elf64SHdr; + } + /// Returns a NULL-terminated string from the ELF image at `offset`. fn elf_str(elf: &[u8], offset: u64) -> Option<&str> { let idx = usize::try_from(offset).ok()?; @@ -485,47 +541,49 @@ fn elf_str(elf: &[u8], offset: u64) -> Option<&str> { CStr::from_bytes_until_nul(bytes).ok()?.to_str().ok() } - /// Tries to extract section with name `name` from the ELF64 image `elf`, and returns it. - pub(super) fn elf64_section<'a, 'b>(elf: &'a [u8], name: &'b str) -> Option<&'a [u8]> { - let hdr = &elf - .get(0..size_of::()) - .and_then(Elf64Hdr::from_bytes)? - .0; + fn elf_section_generic<'a, F>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> + where + F: ElfFormat, + { + let hdr = F::Header::from_bytes(elf.get(0..size_of::())?)?; - // Get all the section headers. - let mut shdr = { - let shdr_num = usize::from(hdr.e_shnum); - let shdr_start = usize::try_from(hdr.e_shoff).ok()?; - let shdr_end = shdr_num - .checked_mul(size_of::()) - .and_then(|v| v.checked_add(shdr_start))?; + let shdr_num = usize::from(hdr.shnum()); + let shdr_start = usize::try_from(hdr.shoff()).ok()?; + let shdr_end = shdr_num + .checked_mul(size_of::()) + .and_then(|v| v.checked_add(shdr_start))?; - elf.get(shdr_start..shdr_end) - .map(|slice| slice.chunks_exact(size_of::()))? - }; + // Get all the section headers as an iterator over byte chunks. + let shdr_bytes = elf.get(shdr_start..shdr_end)?; + let mut shdr_iter = shdr_bytes.chunks_exact(size_of::()); // Get the strings table. - let strhdr = shdr + let strhdr = shdr_iter .clone() - .nth(usize::from(hdr.e_shstrndx)) - .and_then(Elf64SHdr::from_bytes)?; + .nth(usize::from(hdr.shstrndx())) + .and_then(F::SectionHeader::from_bytes)?; // Find the section which name matches `name` and return it. - shdr.find_map(|sh| { - let hdr = Elf64SHdr::from_bytes(sh)?; - let name_offset = strhdr.0.sh_offset.checked_add(u64::from(hdr.0.sh_name))?; + shdr_iter.find_map(|sh_bytes| { + let sh = F::SectionHeader::from_bytes(sh_bytes)?; + let name_offset = strhdr.offset().checked_add(u64::from(sh.name()))?; let section_name = elf_str(elf, name_offset)?; if section_name != name { return None; } - let start = usize::try_from(hdr.0.sh_offset).ok()?; - let end = usize::try_from(hdr.0.sh_size) + let start = usize::try_from(sh.offset()).ok()?; + let end = usize::try_from(sh.size()) .ok() - .and_then(|sh_size| start.checked_add(sh_size))?; + .and_then(|sz| start.checked_add(sz))?; elf.get(start..end) }) } + + /// Tries to extract section with name `name` from the ELF64 image `elf`, and returns it. + pub(super) fn elf64_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { + elf_section_generic::(elf, name) + } } From 258a6f9ab13b809726d2f42da54b8d830e57f1dd Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:20:56 -0700 Subject: [PATCH 111/131] gpu: nova-core: add support for 32-bit firmware images Some GPU firmware images are packaged as 32-bit ELF rather than 64-bit. Add a 32-bit implementation of the shared ELF section-parsing abstraction so those images can be parsed alongside the existing 64-bit path. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-9-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware.rs | 53 +++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 38088e950980..e4dcc9a87b7e 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -534,6 +534,53 @@ impl ElfFormat for Elf64Format { type SectionHeader = Elf64SHdr; } + /// Newtype to provide [`FromBytes`] and [`ElfHeader`] implementations for ELF32. + #[repr(transparent)] + struct Elf32Hdr(bindings::elf32_hdr); + // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. + unsafe impl FromBytes for Elf32Hdr {} + + impl ElfHeader for Elf32Hdr { + fn shnum(&self) -> u16 { + self.0.e_shnum + } + + fn shoff(&self) -> u64 { + u64::from(self.0.e_shoff) + } + + fn shstrndx(&self) -> u16 { + self.0.e_shstrndx + } + } + + /// Newtype to provide [`FromBytes`] and [`ElfSectionHeader`] implementations for ELF32. + #[repr(transparent)] + struct Elf32SHdr(bindings::elf32_shdr); + // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. + unsafe impl FromBytes for Elf32SHdr {} + + impl ElfSectionHeader for Elf32SHdr { + fn name(&self) -> u32 { + self.0.sh_name + } + + fn offset(&self) -> u64 { + u64::from(self.0.sh_offset) + } + + fn size(&self) -> u64 { + u64::from(self.0.sh_size) + } + } + + struct Elf32Format; + + impl ElfFormat for Elf32Format { + type Header = Elf32Hdr; + type SectionHeader = Elf32SHdr; + } + /// Returns a NULL-terminated string from the ELF image at `offset`. fn elf_str(elf: &[u8], offset: u64) -> Option<&str> { let idx = usize::try_from(offset).ok()?; @@ -586,4 +633,10 @@ fn elf_section_generic<'a, F>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> pub(super) fn elf64_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { elf_section_generic::(elf, name) } + + /// Extract the section with name `name` from the ELF32 image `elf`. + #[expect(dead_code)] + pub(super) fn elf32_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { + elf_section_generic::(elf, name) + } } From ad5f9977a9a0070526d3d7f8f18e4652877a9def Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:20:57 -0700 Subject: [PATCH 112/131] gpu: nova-core: add auto-detection of 32-bit, 64-bit firmware images A firmware image may be either a 32-bit or a 64-bit ELF, and callers should not have to know which. Detect the ELF class from the image header at parse time and dispatch to the matching parser, so a single entry point handles both layouts. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-10-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware.rs | 27 +++++++++++++++++++++++---- drivers/gpu/nova-core/firmware/gsp.rs | 4 ++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index e4dcc9a87b7e..87588cb24f11 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -629,14 +629,33 @@ fn elf_section_generic<'a, F>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> }) } - /// Tries to extract section with name `name` from the ELF64 image `elf`, and returns it. - pub(super) fn elf64_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { + /// Extract the section with name `name` from the ELF64 image `elf`. + fn elf64_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { elf_section_generic::(elf, name) } /// Extract the section with name `name` from the ELF32 image `elf`. - #[expect(dead_code)] - pub(super) fn elf32_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { + fn elf32_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { elf_section_generic::(elf, name) } + + /// Automatically detects ELF32 vs ELF64 based on the ELF header. + pub(super) fn elf_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { + // ELF identification: a 4-byte magic followed by a class byte (32- vs 64-bit). + const ELFMAG: &[u8] = b"\x7fELF"; + const SELFMAG: usize = ELFMAG.len(); + const EI_CLASS: usize = 4; + const ELFCLASS32: u8 = 1; + const ELFCLASS64: u8 = 2; + + if elf.get(0..SELFMAG) != Some(ELFMAG) { + return None; + } + + match *elf.get(EI_CLASS)? { + ELFCLASS32 => elf32_section(elf, name), + ELFCLASS64 => elf64_section(elf, name), + _ => None, + } + } } diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index e576bc8a9b1c..99a302bae567 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -88,7 +88,7 @@ pub(crate) fn new<'a>( pin_init::pin_init_scope(move || { let firmware = super::request_firmware(dev, chipset, "gsp", ver)?; - let fw_section = elf::elf64_section(firmware.data(), ".fwimage").ok_or(EINVAL)?; + let fw_section = elf::elf_section(firmware.data(), ".fwimage").ok_or(EINVAL)?; let size = fw_section.len(); @@ -148,7 +148,7 @@ pub(crate) fn new<'a>( signatures: { let sigs_section = Self::find_gsp_sigs_section(chipset); - elf::elf64_section(firmware.data(), sigs_section) + elf::elf_section(firmware.data(), sigs_section) .ok_or(EINVAL) .and_then(|data| Coherent::from_slice(dev, data, GFP_KERNEL))? }, From 34599e07389fb3f44a79d5a4a4b64a04f4304271 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:20:58 -0700 Subject: [PATCH 113/131] gpu: nova-core: Hopper/Blackwell: add FSP falcon engine stub Add the FSP (Foundation Security Processor) falcon engine type that will handle secure boot and Chain of Trust operations on Hopper and Blackwell architectures. The FSP falcon replaces SEC2's role in the boot sequence for these newer architectures. This initial stub just defines the falcon type and its base address. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-11-jhubbard@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon.rs | 1 + drivers/gpu/nova-core/falcon/fsp.rs | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 drivers/gpu/nova-core/falcon/fsp.rs diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 24cc2c26e28d..053ce5bea6cd 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -40,6 +40,7 @@ regs, }; +pub(crate) mod fsp; pub(crate) mod gsp; mod hal; pub(crate) mod sec2; diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs new file mode 100644 index 000000000000..c4a9ce8a47f8 --- /dev/null +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! FSP (Foundation Security Processor) falcon engine for Hopper/Blackwell GPUs. +//! +//! The FSP falcon handles secure boot and Chain of Trust operations +//! on Hopper and Blackwell architectures, replacing SEC2's role. + +use kernel::io::register::RegisterBase; + +use crate::falcon::{ + FalconEngine, + PFalcon2Base, + PFalconBase, // +}; + +/// Type specifying the `Fsp` falcon engine. Cannot be instantiated. +#[expect(dead_code)] +pub(crate) struct Fsp(()); + +impl RegisterBase for Fsp { + const BASE: usize = 0x8f2000; +} + +impl RegisterBase for Fsp { + const BASE: usize = 0x8f3000; +} + +impl FalconEngine for Fsp {} From 4257d3179384f8ccb085b64f5ca1373be637b77c Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:20:59 -0700 Subject: [PATCH 114/131] gpu: nova-core: Hopper/Blackwell: add FMC firmware image FSP is the Falcon that runs FMC firmware on Hopper and Blackwell. Load the FMC ELF in two forms: the image section that FSP boots from, and the full Firmware object for later signature extraction during Chain of Trust verification. Declare the FMC image in the module's firmware table so it is bundled for FSP-based chipsets. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-12-jhubbard@nvidia.com Co-developed-by: Alexandre Courbot Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware.rs | 9 ++++- drivers/gpu/nova-core/firmware/fsp.rs | 47 ++++++++++++++++++++++++++ drivers/gpu/nova-core/gpu.rs | 9 +++++ drivers/gpu/nova-core/gsp/hal/gh100.rs | 10 ++++-- 4 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/nova-core/firmware/fsp.rs diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 87588cb24f11..366d3b76360e 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -28,6 +28,7 @@ }; pub(crate) mod booter; +pub(crate) mod fsp; pub(crate) mod fwsec; pub(crate) mod gsp; pub(crate) mod riscv; @@ -431,10 +432,16 @@ const fn make_entry_chipset(self, chipset: gpu::Chipset) -> Self { .make_entry_file(name, "bootloader") .make_entry_file(name, "gsp"); - if chipset.needs_fwsec_bootloader() { + let this = if chipset.needs_fwsec_bootloader() { this.make_entry_file(name, "gen_bootloader") } else { this + }; + + if chipset.uses_fsp() { + this.make_entry_file(name, "fmc") + } else { + this } } diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs new file mode 100644 index 000000000000..011be1e571c2 --- /dev/null +++ b/drivers/gpu/nova-core/firmware/fsp.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! FSP is a hardware unit that runs FMC firmware. + +use kernel::{ + device, + dma::Coherent, + firmware::Firmware, + prelude::*, // +}; + +use crate::{ + firmware::elf, + gpu::Chipset, // +}; + +pub(crate) struct FspFirmware { + /// FMC firmware image data (only the "image" ELF section). + #[expect(dead_code)] + pub(crate) fmc_image: Coherent<[u8]>, + /// Full FMC ELF for signature extraction. + #[expect(dead_code)] + pub(crate) fmc_elf: Firmware, +} + +impl FspFirmware { + pub(crate) fn new( + dev: &device::Device, + chipset: Chipset, + ver: &str, + ) -> Result { + let fw = super::request_firmware(dev, chipset, "fmc", ver)?; + + // FSP expects only the "image" section, not the entire ELF file. + let fmc_image_data = elf::elf_section(fw.data(), "image").ok_or_else(|| { + dev_err!(dev, "FMC ELF file missing 'image' section\n"); + EINVAL + })?; + let fmc_image = Coherent::from_slice(dev, fmc_image_data, GFP_KERNEL)?; + + Ok(Self { + fmc_image, + fmc_elf: fw, + }) + } +} diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 7dd736e5b190..b7341bde04be 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -137,6 +137,15 @@ pub(crate) const fn needs_fwsec_bootloader(self) -> bool { matches!(self.arch(), Architecture::Turing) || matches!(self, Self::GA100) } + /// Returns `true` if this chipset boots via FSP (Hopper and later), which requires the FMC + /// firmware image. + pub(crate) const fn uses_fsp(self) -> bool { + matches!( + self.arch(), + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x + ) + } + /// Returns the address range of the PCI config mirror space. pub(crate) fn pci_config_mirror_range(self) -> Range { hal::gpu_hal(self).pci_config_mirror_range() diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 9a4bb22578b3..9681f9a73e86 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -16,6 +16,10 @@ Falcon, // }, fb::FbLayout, + firmware::{ + fsp::FspFirmware, + FIRMWARE_VERSION, // + }, gpu::Chipset, gsp::{ boot::BootUnloadGuard, @@ -35,14 +39,16 @@ impl GspHal for Gh100 { fn boot<'a>( &self, _gsp: &'a Gsp, - _dev: &'a device::Device, + dev: &'a device::Device, _bar: &'a Bar0, - _chipset: Chipset, + chipset: Chipset, _fb_layout: &FbLayout, _wpr_meta: &Coherent, _gsp_falcon: &'a Falcon, _sec2_falcon: &'a Falcon, ) -> Result> { + let _fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; + Err(ENOTSUPP) } } From 82eaa14e7efcbb3933083b4c27fd5496fbb1a4be Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:21:00 -0700 Subject: [PATCH 115/131] gpu: nova-core: Hopper/Blackwell: add FSP secure boot completion waiting Hopper and Blackwell use FSP instead of SEC2 for secure boot. The driver must wait for FSP secure boot to complete before continuing with GSP bring-up. Poll for boot success with a 5-second timeout, and return the FSP interface only on success so that later Chain of Trust operations cannot run before FSP is ready. The interface owns the FSP falcon and the FMC firmware. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-13-jhubbard@nvidia.com [acourbot: use `inspect_err` instead of `map_err` and display actual error] [acourbot: limit visibility of `fsp_hal` to `super``] Co-developed-by: Alexandre Courbot Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/fsp.rs | 1 - drivers/gpu/nova-core/fsp.rs | 72 ++++++++++++++++++++++++++ drivers/gpu/nova-core/fsp/hal.rs | 27 ++++++++++ drivers/gpu/nova-core/fsp/hal/gb202.rs | 23 ++++++++ drivers/gpu/nova-core/fsp/hal/gh100.rs | 23 ++++++++ drivers/gpu/nova-core/gsp/hal/gh100.rs | 6 ++- drivers/gpu/nova-core/nova_core.rs | 1 + drivers/gpu/nova-core/regs.rs | 36 +++++++++++++ 8 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/nova-core/fsp.rs create mode 100644 drivers/gpu/nova-core/fsp/hal.rs create mode 100644 drivers/gpu/nova-core/fsp/hal/gb202.rs create mode 100644 drivers/gpu/nova-core/fsp/hal/gh100.rs diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index c4a9ce8a47f8..d9f87262e8b1 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -15,7 +15,6 @@ }; /// Type specifying the `Fsp` falcon engine. Cannot be instantiated. -#[expect(dead_code)] pub(crate) struct Fsp(()); impl RegisterBase for Fsp { diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs new file mode 100644 index 000000000000..908dc112aa6f --- /dev/null +++ b/drivers/gpu/nova-core/fsp.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! FSP (Foundation Security Processor) interface for Hopper/Blackwell GPUs. +//! +//! Hopper/Blackwell use a simplified firmware boot sequence: FMC, then FSP, then GSP. +//! Unlike Turing/Ampere/Ada, there is no SEC2 (Security Engine 2) usage. +//! FSP handles secure boot directly using FMC firmware and Chain of Trust. + +use kernel::{ + device, + io::poll::read_poll_timeout, + prelude::*, + time::Delta, // +}; + +use crate::{ + driver::Bar0, + falcon::{ + fsp::Fsp as FspEngine, + Falcon, // + }, + firmware::fsp::FspFirmware, + gpu::Chipset, + regs, // +}; + +mod hal; + +/// FSP interface for Hopper/Blackwell GPUs. +/// +/// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot +/// has completed. It owns the FSP falcon and the FMC firmware, which are used for the subsequent +/// Chain of Trust boot. +pub(crate) struct Fsp { + #[expect(dead_code)] + falcon: Falcon, + #[expect(dead_code)] + fsp_fw: FspFirmware, +} + +impl Fsp { + /// Waits for FSP secure boot completion, then returns the [`Fsp`] interface. + /// + /// Polls the thermal scratch register until FSP signals boot completion or the timeout + /// elapses. Returning an [`Fsp`] only on success guarantees, at the API level, that the + /// interface is not used before secure boot has completed. + pub(crate) fn wait_secure_boot( + dev: &device::Device, + bar: &Bar0, + chipset: Chipset, + fsp_fw: FspFirmware, + ) -> Result { + /// FSP secure boot completion timeout in milliseconds. + const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000; + + let hal = hal::fsp_hal(chipset).ok_or(ENOTSUPP)?; + let falcon = Falcon::::new(dev, chipset)?; + + read_poll_timeout( + || Ok(hal.fsp_boot_status(bar)), + |&status| status == regs::NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE_STATUS_SUCCESS, + Delta::from_millis(10), + Delta::from_millis(FSP_SECURE_BOOT_TIMEOUT_MS), + ) + .inspect_err(|e| { + dev_err!(dev, "FSP secure boot completion error: {:?}\n", e); + })?; + + Ok(Fsp { falcon, fsp_fw }) + } +} diff --git a/drivers/gpu/nova-core/fsp/hal.rs b/drivers/gpu/nova-core/fsp/hal.rs new file mode 100644 index 000000000000..fc5ebb749c59 --- /dev/null +++ b/drivers/gpu/nova-core/fsp/hal.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::{ + driver::Bar0, + gpu::{ + Architecture, + Chipset, // + }, +}; + +mod gb202; +mod gh100; + +pub(super) trait FspHal { + /// Returns the secure boot status from the architecture-specific `NV_THERM_I2CS_SCRATCH` register. + fn fsp_boot_status(&self, bar: &Bar0) -> u32; +} + +/// Returns the FSP HAL, or `None` if the architecture doesn't support FSP. +pub(super) fn fsp_hal(chipset: Chipset) -> Option<&'static dyn FspHal> { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => None, + Architecture::Hopper | Architecture::BlackwellGB10x => Some(gh100::GH100_HAL), + Architecture::BlackwellGB20x => Some(gb202::GB202_HAL), + } +} diff --git a/drivers/gpu/nova-core/fsp/hal/gb202.rs b/drivers/gpu/nova-core/fsp/hal/gb202.rs new file mode 100644 index 000000000000..2f08b6c9f308 --- /dev/null +++ b/drivers/gpu/nova-core/fsp/hal/gb202.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::io::Io; + +use crate::{ + driver::Bar0, + fsp::hal::FspHal, + regs, // +}; + +struct Gb202; + +impl FspHal for Gb202 { + fn fsp_boot_status(&self, bar: &Bar0) -> u32 { + bar.read(regs::gb202::NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE) + .fsp_boot_complete() + .into() + } +} + +const GB202: Gb202 = Gb202; +pub(super) const GB202_HAL: &dyn FspHal = &GB202; diff --git a/drivers/gpu/nova-core/fsp/hal/gh100.rs b/drivers/gpu/nova-core/fsp/hal/gh100.rs new file mode 100644 index 000000000000..290fb55a81da --- /dev/null +++ b/drivers/gpu/nova-core/fsp/hal/gh100.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::io::Io; + +use crate::{ + driver::Bar0, + fsp::hal::FspHal, + regs, // +}; + +struct Gh100; + +impl FspHal for Gh100 { + fn fsp_boot_status(&self, bar: &Bar0) -> u32 { + bar.read(regs::gh100::NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE) + .fsp_boot_complete() + .into() + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn FspHal = &GH100; diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 9681f9a73e86..b25970dd4561 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -20,6 +20,7 @@ fsp::FspFirmware, FIRMWARE_VERSION, // }, + fsp::Fsp, gpu::Chipset, gsp::{ boot::BootUnloadGuard, @@ -40,14 +41,15 @@ fn boot<'a>( &self, _gsp: &'a Gsp, dev: &'a device::Device, - _bar: &'a Bar0, + bar: &'a Bar0, chipset: Chipset, _fb_layout: &FbLayout, _wpr_meta: &Coherent, _gsp_falcon: &'a Falcon, _sec2_falcon: &'a Falcon, ) -> Result> { - let _fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; + let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; + let _fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?; Err(ENOTSUPP) } diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 5a260062295f..7b6c331da10e 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -17,6 +17,7 @@ mod falcon; mod fb; mod firmware; +mod fsp; mod gpu; mod gsp; #[macro_use] diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index b39647684dd1..2cb1f02f35a4 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -587,3 +587,39 @@ pub(crate) mod ga100 { } } } + +pub(crate) const NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE_STATUS_SUCCESS: u32 = 0xff; + +pub(crate) mod gh100 { + use kernel::io::register; + + // PTHERM + + register! { + pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x000200bc { + 31:0 data; + } + + // Alias to `NV_THERM_I2CS_SCRATCH` when used to check for FSP boot completion. + pub(crate) NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE(u32) => NV_THERM_I2CS_SCRATCH { + 31:0 fsp_boot_complete; + } + } +} + +pub(crate) mod gb202 { + use kernel::io::register; + + // PTHERM + + register! { + pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x00ad00bc { + 31:0 data; + } + + // Alias to `NV_THERM_I2CS_SCRATCH` when used to check for FSP boot completion. + pub(crate) NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE(u32) => NV_THERM_I2CS_SCRATCH { + 31:0 fsp_boot_complete; + } + } +} From 4d789488d3bec3e2c6b7f282e29cfb1bec6b2c25 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Mon, 1 Jun 2026 20:21:01 -0700 Subject: [PATCH 116/131] gpu: nova-core: Hopper/Blackwell: add FMC signature extraction Extract the SHA-384 hash, RSA public key, and RSA signature from the FMC ELF32 firmware sections. FSP Chain of Trust verification needs these to validate the FMC image during boot. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602032111.224790-14-jhubbard@nvidia.com [acourbot: derive `Zeroable` on `FmcSignature` for in-place initialization] Co-developed-by: Alexandre Courbot Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware/fsp.rs | 89 ++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs index 011be1e571c2..9b211426a75a 100644 --- a/drivers/gpu/nova-core/firmware/fsp.rs +++ b/drivers/gpu/nova-core/firmware/fsp.rs @@ -15,13 +15,35 @@ gpu::Chipset, // }; +/// Size of the FSP SHA-384 hash, in bytes. +const FSP_HASH_SIZE: usize = 48; +/// Maximum size of the FSP public key (RSA-3072), in bytes. +/// +/// The FMC ELF `publickey` section may be shorter, so the remaining bytes are zero-padded. +const FSP_PKEY_SIZE: usize = 384; +/// Maximum size of the FSP signature (RSA-3072), in bytes. +/// +/// The FMC ELF `signature` section may be shorter, so the remaining bytes are zero-padded. +const FSP_SIG_SIZE: usize = 384; + +/// Structure to hold FMC signatures. +/// +/// C representation is used because this type is used for communication with the FSP. +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(C)] +pub(crate) struct FmcSignatures { + pub(crate) hash384: [u8; FSP_HASH_SIZE], + pub(crate) public_key: [u8; FSP_PKEY_SIZE], + pub(crate) signature: [u8; FSP_SIG_SIZE], +} + pub(crate) struct FspFirmware { /// FMC firmware image data (only the "image" ELF section). #[expect(dead_code)] pub(crate) fmc_image: Coherent<[u8]>, - /// Full FMC ELF for signature extraction. + /// FMC firmware signatures. #[expect(dead_code)] - pub(crate) fmc_elf: Firmware, + pub(crate) fmc_sigs: KBox, } impl FspFirmware { @@ -41,7 +63,68 @@ pub(crate) fn new( Ok(Self { fmc_image, - fmc_elf: fw, + fmc_sigs: Self::extract_fmc_signatures(&fw, dev)?, }) } + + /// Extract FMC firmware signatures for Chain of Trust verification. + /// + /// Extracts real cryptographic signatures from FMC ELF32 firmware sections. + /// Returns signatures in a heap-allocated structure to prevent stack overflow. + fn extract_fmc_signatures( + fmc_fw: &Firmware, + dev: &device::Device, + ) -> Result> { + let get_section = |name: &str, max_len: usize| { + elf::elf_section(fmc_fw.data(), name) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "FMC firmware missing '{}' section\n", name)) + .and_then(|section| { + if section.len() > max_len { + dev_err!( + dev, + "FMC {} section size {} > maximum {}\n", + name, + section.len(), + max_len + ); + Err(EINVAL) + } else { + Ok(section) + } + }) + }; + + let hash_section = get_section("hash", FSP_HASH_SIZE)?; + let pkey_section = get_section("publickey", FSP_PKEY_SIZE)?; + let sig_section = get_section("signature", FSP_SIG_SIZE)?; + + // The hash section is a SHA-384 output: it must be exactly FSP_HASH_SIZE bytes. + if hash_section.len() != FSP_HASH_SIZE { + dev_err!( + dev, + "FMC hash section size {} != expected {}\n", + hash_section.len(), + FSP_HASH_SIZE + ); + return Err(EINVAL); + } + + // Initialize the signatures in place to avoid building the large `FmcSignatures` on the + // stack, then fill each section from the firmware. + let signatures = KBox::init( + pin_init::init_zeroed::().chain(|sigs| { + // PANIC: src and dst lengths are both FSP_HASH_SIZE (verified above). + sigs.hash384.copy_from_slice(hash_section); + // PANIC: dst is sliced to src.len(); src.len() <= FSP_PKEY_SIZE per `get_section`. + sigs.public_key[..pkey_section.len()].copy_from_slice(pkey_section); + // PANIC: dst is sliced to src.len(); src.len() <= FSP_SIG_SIZE per `get_section`. + sigs.signature[..sig_section.len()].copy_from_slice(sig_section); + Ok(()) + }), + GFP_KERNEL, + )?; + + Ok(signatures) + } } From 04b325fb3456df0ffa7d6a8ac5cb4f94b860b575 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Tue, 28 Apr 2026 15:03:41 -0400 Subject: [PATCH 117/131] rust: drm: gem: s/device::Device/Device/ for shmem.rs We're about to start explicitly mentioning kernel devices as well in this file, so this makes it easier to differentiate the two by allowing us to import `device` as `kernel::device`. Signed-off-by: Lyude Paul Reviewed-by: Alexandre Courbot Reviewed-by: Alice Ryhl Link: https://patch.msgid.link/20260428190605.3355690-2-lyude@redhat.com --- rust/kernel/drm/gem/shmem.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index e1b648920d2f..35d7523e164f 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -12,10 +12,10 @@ use crate::{ container_of, drm::{ - device, driver, gem, - private::Sealed, // + private::Sealed, + Device, // }, error::to_result, prelude::*, @@ -106,7 +106,7 @@ fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { /// /// Additional config options can be specified using `config`. pub fn new( - dev: &device::Device, + dev: &Device, size: usize, config: ObjectConfig<'_, T>, args: T::Args, @@ -148,9 +148,9 @@ pub fn new( } /// Returns the `Device` that owns this GEM object. - pub fn dev(&self) -> &device::Device { + pub fn dev(&self) -> &Device { // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. - unsafe { device::Device::from_raw((*self.as_raw()).dev) } + unsafe { Device::from_raw((*self.as_raw()).dev) } } extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { From b78dab829760aee9b83f5cf15550a0fe36c6f4b0 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Fri, 29 May 2026 14:34:03 -0400 Subject: [PATCH 118/131] drm/gem/shmem: Introduce __drm_gem_shmem_free_sgt_locked() One of the complications of trying to use the shmem helpers to create a scatterlist for shmem objects is that we need to be able to provide a guarantee that the driver cannot be unbound for the lifetime of the scatterlist. The easiest way of handling this seems to be just hooking up an unmap operation to devres the first time we create a scatterlist, which allows us to still take advantage of gem shmem facilities without breaking that guarantee. To allow for this, we extract __drm_gem_shmem_free_sgt_locked() - which allows a caller (e.g. the rust bindings) to manually unmap the sgt for a gem object as needed. Signed-off-by: Lyude Paul Reviewed-by: Alexandre Courbot Link: https://patch.msgid.link/20260529183702.677677-6-lyude@redhat.com --- drivers/gpu/drm/drm_gem_shmem_helper.c | 32 +++++++++++++++++++++----- include/drm/drm_gem_shmem_helper.h | 1 + 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 545933c7f712..c989459eb215 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -158,6 +158,30 @@ struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t } EXPORT_SYMBOL_GPL(drm_gem_shmem_create); +/** + * __drm_gem_shmem_release_sgt_locked - Unpin and DMA unmap pages, and release the + * cached scatter/gather table for an shmem GEM object. + * @shmem: shmem GEM object + * + * If the passed shmem object has an active scatter/gather table for driver + * usage, this function will unmap it and release the memory associated with it. + * It is the responsibility of the caller to ensure it holds the dma_resv_lock + * for this object. + * + * Drivers should not need to call this function themselves, it is mainly + * intended for usage in the Rust shmem bindings. + */ +void __drm_gem_shmem_free_sgt_locked(struct drm_gem_shmem_object *shmem) +{ + dma_resv_assert_held(shmem->base.resv); + + dma_unmap_sgtable(shmem->base.dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); + sg_free_table(shmem->sgt); + kfree(shmem->sgt); + shmem->sgt = NULL; +} +EXPORT_SYMBOL_GPL(__drm_gem_shmem_free_sgt_locked); + /** * drm_gem_shmem_release - Release resources associated with a shmem GEM object. * @shmem: shmem GEM object @@ -176,12 +200,8 @@ void drm_gem_shmem_release(struct drm_gem_shmem_object *shmem) drm_WARN_ON(obj->dev, refcount_read(&shmem->vmap_use_count)); - if (shmem->sgt) { - dma_unmap_sgtable(obj->dev->dev, shmem->sgt, - DMA_BIDIRECTIONAL, 0); - sg_free_table(shmem->sgt); - kfree(shmem->sgt); - } + if (shmem->sgt) + __drm_gem_shmem_free_sgt_locked(shmem); if (shmem->pages) drm_gem_shmem_put_pages_locked(shmem); diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index 5ccdae21b94a..b2c23af628e1 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -111,6 +111,7 @@ int drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_object *shme struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size); void drm_gem_shmem_release(struct drm_gem_shmem_object *shmem); void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem); +void __drm_gem_shmem_free_sgt_locked(struct drm_gem_shmem_object *shmem); void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem); int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem); From 571d4242c466e558c9fd57a9b3d9a045b1f400d1 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Thu, 7 May 2026 17:59:19 -0400 Subject: [PATCH 119/131] rust/drm: Introduce DeviceContext One of the tricky things about DRM bindings in Rust is the fact that initialization of a DRM device is a multi-step process. It's quite normal for a device driver to start making use of its DRM device for tasks like creating GEM objects before userspace registration happens. This is an issue in rust though, since prior to userspace registration the device is only partly initialized. This means there's a plethora of DRM device operations we can't yet expose without opening up the door to UB if the DRM device in question isn't yet registered. Additionally, this isn't something we can reliably check at runtime. And even if we could, performing an operation which requires the device be registered when the device isn't actually registered is a programmer bug, meaning there's no real way to gracefully handle such a mistake at runtime. And even if that wasn't the case, it would be horrendously annoying and noisy to have to check if a device is registered constantly throughout a driver. In order to solve this, we first take inspiration from `kernel::device::DeviceContext` and introduce `kernel::drm::DeviceContext`. This provides us with a ZST type that we can generalize over to represent contexts where a device is known to have been registered with userspace at some point in time (`Registered`), along with contexts where we can't make such a guarantee (`Uninit`). It's important to note we intentionally do not provide a `DeviceContext` which represents an unregistered device. This is because there's no reasonable way to guarantee that a device with long-living references to itself will not be registered eventually with userspace. Instead, we provide a new-type for this: `UnregisteredDevice` which can provide a guarantee that the `Device` has never been registered with userspace. To ensure this, we modify `Registration` so that creating a new `Registration` requires passing ownership of an `UnregisteredDevice`. Signed-off-by: Lyude Paul Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260507220044.3204919-2-lyude@redhat.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 8 +- drivers/gpu/drm/tyr/driver.rs | 10 +- rust/kernel/drm/device.rs | 222 +++++++++++++++++++++++++-------- rust/kernel/drm/driver.rs | 35 ++++-- rust/kernel/drm/mod.rs | 4 + 5 files changed, 214 insertions(+), 65 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 4289df7de01c..33fe088dd3b9 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -23,7 +23,7 @@ pub(crate) struct Nova { } /// Convienence type alias for the DRM device type for this driver -pub(crate) type NovaDevice = drm::Device; +pub(crate) type NovaDevice = drm::Device; #[pin_data] pub(crate) struct NovaData { @@ -62,10 +62,10 @@ fn probe<'bound>( ) -> impl PinInit, Error> + 'bound { let data = try_pin_init!(NovaData { adev: adev.into() }); - let drm = drm::Device::::new(adev.as_ref(), data)?; - drm::Registration::new_foreign_owned(&drm, adev.as_ref(), 0)?; + let drm = drm::UnregisteredDevice::::new(adev.as_ref(), data)?; + let drm = drm::Registration::new_foreign_owned(drm, adev.as_ref(), 0)?; - Ok(Nova { drm }) + Ok(Nova { drm: drm.into() }) } } diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 227ea2adccea..3e53ecd33143 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -46,7 +46,7 @@ pub(crate) struct TyrDrmDriver; /// Convenience type alias for the DRM device type for this driver. -pub(crate) type TyrDrmDevice = drm::Device; +pub(crate) type TyrDrmDevice = drm::Device; pub(crate) struct TyrPlatformDriver; @@ -148,10 +148,12 @@ fn probe<'bound>( gpu_info, }); - let ddev: ARef = drm::Device::new(pdev.as_ref(), data)?; - drm::driver::Registration::new_foreign_owned(&ddev, pdev.as_ref(), 0)?; + let tdev = drm::UnregisteredDevice::::new(pdev.as_ref(), data)?; + let tdev = drm::driver::Registration::new_foreign_owned(tdev, pdev.as_ref(), 0)?; - let driver = TyrPlatformDriverData { _device: ddev }; + let driver = TyrPlatformDriverData { + _device: tdev.into(), + }; // We need this to be dev_info!() because dev_dbg!() does not work at // all in Rust for now, and we need to see whether probe succeeded. diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index dc16738eeb38..a47d81a26724 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -6,10 +6,12 @@ use crate::{ alloc::allocator::Kmalloc, - bindings, device, + bindings, + device, drm::{ self, - driver::AllocImpl, // + driver::AllocImpl, + private::Sealed, // }, error::from_err_ptr, prelude::*, @@ -17,16 +19,20 @@ ARef, AlwaysRefCounted, // }, - types::Opaque, + types::{ + NotThreadSafe, + Opaque, // + }, workqueue::{ HasDelayedWork, HasWork, Work, WorkItem, // - }, + }, // }; use core::{ alloc::Layout, + marker::PhantomData, mem, ops::Deref, ptr::{ @@ -66,20 +72,92 @@ macro_rules! drm_legacy_fields { } } -/// A typed DRM device with a specific `drm::Driver` implementation. +/// A trait implemented by all possible contexts a [`Device`] can be used in. /// -/// The device is always reference-counted. +/// Setting up a new [`Device`] is a multi-stage process. Each step of the process that a user +/// interacts with in Rust has a respective [`DeviceContext`] typestate. For example, +/// `Device` would be a [`Device`] that reached the [`Registered`] [`DeviceContext`]. +/// +/// Each stage of this process is described below: +/// +/// ```text +/// 1 2 3 +/// +--------------+ +------------------+ +-----------------------+ +/// |Device created| → |Device initialized| → |Registered w/ userspace| +/// +--------------+ +------------------+ +-----------------------+ +/// (Uninit) (Registered) +/// ``` +/// +/// 1. The [`Device`] is in the [`Uninit`] context and is not guaranteed to be initialized or +/// registered with userspace. Only a limited subset of DRM core functionality is available. +/// 2. The [`Device`] is guaranteed to be fully initialized, but is not guaranteed to be registered +/// with userspace. All DRM core functionality which doesn't interact with userspace is +/// available. We currently don't have a context for representing this. +/// 3. The [`Device`] is guaranteed to be fully initialized, and is guaranteed to have been +/// registered with userspace at some point - thus putting it in the [`Registered`] context. +/// +/// An important caveat of [`DeviceContext`] which must be kept in mind: when used as a typestate +/// for a reference type, it can only guarantee that a [`Device`] reached a particular stage in the +/// initialization process _at the time the reference was taken_. No guarantee is made in regards to +/// what stage of the process the [`Device`] is currently in. This means for instance that a +/// `&Device` may actually be registered with userspace, it just wasn't known to be +/// registered at the time the reference was taken. +pub trait DeviceContext: Sealed + Send + Sync {} + +/// The [`DeviceContext`] of a [`Device`] that was registered with userspace at some point. +/// +/// This represents a [`Device`] which is guaranteed to have been registered with userspace at +/// some point in time. Such a DRM device is guaranteed to have been fully-initialized. +/// +/// Note: A device in this context is not guaranteed to remain registered with userspace for its +/// entire lifetime, as this is impossible to guarantee at compile-time. /// /// # Invariants /// -/// `self.dev` is a valid instance of a `struct device`. -#[repr(C)] -pub struct Device { - dev: Opaque, - data: T::Data, +/// A [`Device`] in this [`DeviceContext`] is guaranteed to have been registered with userspace +/// at some point in time. +pub struct Registered; + +impl Sealed for Registered {} +impl DeviceContext for Registered {} + +/// The [`DeviceContext`] of a [`Device`] that may be unregistered and partly uninitialized. +/// +/// A [`Device`] in this context is only guaranteed to be partly initialized, and may or may not +/// be registered with userspace. Thus operations which depend on the [`Device`] being fully +/// initialized, or which depend on the [`Device`] being registered with userspace are not +/// available through this [`DeviceContext`]. +/// +/// A [`Device`] in this context can be used to create a +/// [`Registration`](drm::driver::Registration). +pub struct Uninit; + +impl Sealed for Uninit {} +impl DeviceContext for Uninit {} + +/// A [`Device`] which is known at compile-time to be unregistered with userspace. +/// +/// This type allows performing operations which are only safe to do before userspace registration, +/// and can be used to create a [`Registration`](drm::driver::Registration) once the driver is ready +/// to register the device with userspace. +/// +/// Since DRM device initialization must be single-threaded, this object is not thread-safe. +/// +/// # Invariants +/// +/// The device in `self.0` is guaranteed to be a newly created [`Device`] that has not yet been +/// registered with userspace until this type is dropped. +pub struct UnregisteredDevice(ARef>, NotThreadSafe); + +impl Deref for UnregisteredDevice { + type Target = Device; + + fn deref(&self) -> &Self::Target { + &self.0 + } } -impl Device { +impl UnregisteredDevice { const fn compute_features() -> u32 { let mut features = drm::driver::FEAT_GEM; @@ -95,7 +173,7 @@ const fn compute_features() -> u32 { open: Some(drm::File::::open_callback), postclose: Some(drm::File::::postclose_callback), unload: None, - release: Some(Self::release), + release: Some(Device::::release), master_set: None, master_drop: None, debugfs_init: None, @@ -123,11 +201,13 @@ const fn compute_features() -> u32 { const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(); - /// Create a new `drm::Device` for a `drm::Driver`. - pub fn new(dev: &device::Device, data: impl PinInit) -> Result> { + /// Create a new `UnregisteredDevice` for a `drm::Driver`. + /// + /// This can be used to create a [`Registration`](kernel::drm::Registration). + pub fn new(dev: &device::Device, data: impl PinInit) -> Result { // `__drm_dev_alloc` uses `kmalloc()` to allocate memory, hence ensure a `kmalloc()` // compatible `Layout`. - let layout = Kmalloc::aligned_layout(Layout::new::()); + let layout = Kmalloc::aligned_layout(Layout::new::>()); // Use a temporary vtable without a `release` callback until `data` is initialized, so // init failure can release the DRM device without dropping uninitialized fields. @@ -139,12 +219,12 @@ pub fn new(dev: &device::Device, data: impl PinInit) -> Result = unsafe { bindings::__drm_dev_alloc( dev.as_raw(), &alloc_vtable, layout.size(), - mem::offset_of!(Self, dev), + mem::offset_of!(Device, dev), ) } .cast(); @@ -152,7 +232,7 @@ pub fn new(dev: &device::Device, data: impl PinInit) -> Result) -> Result { + dev: Opaque, + data: T::Data, + _ctx: PhantomData, +} + +impl Device { pub(crate) fn as_raw(&self) -> *mut bindings::drm_device { self.dev.get() } @@ -199,13 +309,13 @@ unsafe fn into_drm_device(ptr: NonNull) -> *mut bindings::drm_device { /// /// # Safety /// - /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, - /// i.e. it must be ensured that the reference count of the C `struct drm_device` `ptr` points - /// to can't drop to zero, for the duration of this function call and the entire duration when - /// the returned reference exists. - /// - /// Additionally, callers must ensure that the `struct device`, `ptr` is pointing to, is - /// embedded in `Self`. + /// * Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, + /// i.e. it must be ensured that the reference count of the C `struct drm_device` `ptr` points + /// to can't drop to zero, for the duration of this function call and the entire duration when + /// the returned reference exists. + /// * Additionally, callers must ensure that the `struct device`, `ptr` is pointing to, is + /// embedded in `Self`. + /// * Callers promise that any type invariants of `C` will be upheld. #[doc(hidden)] pub unsafe fn from_raw<'a>(ptr: *const bindings::drm_device) -> &'a Self { // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a @@ -225,9 +335,20 @@ extern "C" fn release(ptr: *mut bindings::drm_device) { // - `this` is valid for dropping. unsafe { core::ptr::drop_in_place(this) }; } + + /// Change the [`DeviceContext`] for a [`Device`]. + /// + /// # Safety + /// + /// The caller promises that `self` fulfills all of the guarantees provided by the given + /// [`DeviceContext`]. + pub(crate) unsafe fn assume_ctx(&self) -> &Device { + // SAFETY: The data layout is identical via our type invariants. + unsafe { mem::transmute(self) } + } } -impl Deref for Device { +impl Deref for Device { type Target = T::Data; fn deref(&self) -> &Self::Target { @@ -237,7 +358,7 @@ fn deref(&self) -> &Self::Target { // SAFETY: DRM device objects are always reference counted and the get/put functions // satisfy the requirements. -unsafe impl AlwaysRefCounted for Device { +unsafe impl AlwaysRefCounted for Device { fn inc_ref(&self) { // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. unsafe { bindings::drm_dev_get(self.as_raw()) }; @@ -252,7 +373,7 @@ unsafe fn dec_ref(obj: NonNull) { } } -impl AsRef for Device { +impl AsRef for Device { fn as_ref(&self) -> &device::Device { // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid, // which is guaranteed by the type invariant. @@ -261,21 +382,22 @@ fn as_ref(&self) -> &device::Device { } // SAFETY: A `drm::Device` can be released from any thread. -unsafe impl Send for Device {} +unsafe impl Send for Device {} // SAFETY: A `drm::Device` can be shared among threads because all immutable methods are protected // by the synchronization in `struct drm_device`. -unsafe impl Sync for Device {} +unsafe impl Sync for Device {} -impl WorkItem for Device +impl WorkItem for Device where T: drm::Driver, - T::Data: WorkItem>>, - T::Data: HasWork, ID>, + T::Data: WorkItem>, + T::Data: HasWork, + C: DeviceContext, { - type Pointer = ARef>; + type Pointer = ARef; - fn run(ptr: ARef>) { + fn run(ptr: ARef) { T::Data::run(ptr); } } @@ -287,40 +409,42 @@ fn run(ptr: ARef>) { // stored inline in `drm::Device`, so the `container_of` call is valid. // // - The two methods are true inverses of each other: given `ptr: *mut -// Device`, `raw_get_work` will return a `*mut Work, ID>` through -// `T::Data::raw_get_work` and given a `ptr: *mut Work, ID>`, -// `work_container_of` will return a `*mut Device` through `container_of`. -unsafe impl HasWork, ID> for Device +// Device`, `raw_get_work` will return a `*mut Work, ID>` through +// `T::Data::raw_get_work` and given a `ptr: *mut Work, ID>`, +// `work_container_of` will return a `*mut Device` through `container_of`. +unsafe impl HasWork for Device where T: drm::Driver, - T::Data: HasWork, ID>, + T::Data: HasWork, + C: DeviceContext, { - unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work, ID> { - // SAFETY: The caller promises that `ptr` points to a valid `Device`. + unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work { + // SAFETY: The caller promises that `ptr` points to a valid `Device`. let data_ptr = unsafe { &raw mut (*ptr).data }; // SAFETY: `data_ptr` is a valid pointer to `T::Data`. unsafe { T::Data::raw_get_work(data_ptr) } } - unsafe fn work_container_of(ptr: *mut Work, ID>) -> *mut Self { + unsafe fn work_container_of(ptr: *mut Work) -> *mut Self { // SAFETY: The caller promises that `ptr` points at a `Work` field in // `T::Data`. let data_ptr = unsafe { T::Data::work_container_of(ptr) }; - // SAFETY: `T::Data` is stored as the `data` field in `Device`. + // SAFETY: `T::Data` is stored as the `data` field in `Device`. unsafe { crate::container_of!(data_ptr, Self, data) } } } // SAFETY: Our `HasWork` implementation returns a `work_struct` that is // stored in the `work` field of a `delayed_work` with the same access rules as -// the `work_struct` owing to the bound on `T::Data: HasDelayedWork, +// the `work_struct` owing to the bound on `T::Data: HasDelayedWork, // ID>`, which requires that `T::Data::raw_get_work` return a `work_struct` that // is inside a `delayed_work`. -unsafe impl HasDelayedWork, ID> for Device +unsafe impl HasDelayedWork for Device where T: drm::Driver, - T::Data: HasDelayedWork, ID>, + T::Data: HasDelayedWork, + C: DeviceContext, { } diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 6886396c8fa7..07af409959a9 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -13,6 +13,10 @@ prelude::*, sync::aref::ARef, // }; +use core::{ + mem, + ptr::NonNull, // +}; /// Driver use the GEM memory manager. This should be set for all modern drivers. pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; @@ -135,21 +139,31 @@ pub trait Driver { pub struct Registration(ARef>); impl Registration { - fn new(drm: &drm::Device, flags: usize) -> Result { + fn new(drm: drm::UnregisteredDevice, flags: usize) -> Result { // SAFETY: `drm.as_raw()` is valid by the invariants of `drm::Device`. to_result(unsafe { bindings::drm_dev_register(drm.as_raw(), flags) })?; - Ok(Self(drm.into())) + // SAFETY: We just called `drm_dev_register` above + let new = NonNull::from(unsafe { drm.assume_ctx() }); + + // Leak the ARef from UnregisteredDevice in preparation for transferring its ownership. + mem::forget(drm); + + // SAFETY: `drm`'s `Drop` constructor was never called, ensuring that there remains at least + // one reference to the device - which we take ownership over here. + let new = unsafe { ARef::from_raw(new) }; + + Ok(Self(new)) } - /// Registers a new [`Device`](drm::Device) with userspace. + /// Registers a new [`UnregisteredDevice`](drm::UnregisteredDevice) with userspace. /// /// Ownership of the [`Registration`] object is passed to [`devres::register`]. - pub fn new_foreign_owned( - drm: &drm::Device, - dev: &device::Device, + pub fn new_foreign_owned<'a>( + drm: drm::UnregisteredDevice, + dev: &'a device::Device, flags: usize, - ) -> Result + ) -> Result<&'a drm::Device> where T: 'static, { @@ -158,8 +172,13 @@ pub fn new_foreign_owned( } let reg = Registration::::new(drm, flags)?; + let drm = NonNull::from(reg.device()); - devres::register(dev, reg, GFP_KERNEL) + devres::register(dev, reg, GFP_KERNEL)?; + + // SAFETY: Since `reg` was passed to devres::register(), the device now owns the lifetime + // of the DRM registration - ensuring that this references lives for at least as long as 'a. + Ok(unsafe { drm.as_ref() }) } /// Returns a reference to the `Device` instance for this registration. diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index a4b6c5430198..a66e7166f66b 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -10,6 +10,10 @@ pub mod ioctl; pub use self::device::Device; +pub use self::device::DeviceContext; +pub use self::device::Registered; +pub use self::device::Uninit; +pub use self::device::UnregisteredDevice; pub use self::driver::Driver; pub use self::driver::DriverInfo; pub use self::driver::Registration; From 43a5d04a743b499dbad31083a62cdcb46ee74391 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Thu, 7 May 2026 17:59:20 -0400 Subject: [PATCH 120/131] rust/drm/gem: Add DriverAllocImpl type alias This is just a type alias that resolves into the AllocImpl for a given T: drm::gem::DriverObject. Signed-off-by: Lyude Paul Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260507220044.3204919-3-lyude@redhat.com Signed-off-by: Danilo Krummrich --- rust/kernel/drm/gem/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 01b5bd47a333..d4b5940ec0df 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -73,6 +73,11 @@ unsafe fn dec_ref(obj: core::ptr::NonNull) { /// [`DriverFile`]: drm::file::DriverFile pub type DriverFile = drm::File<<::Driver as drm::Driver>::File>; +/// A type alias for retrieving the current [`AllocImpl`] for a given [`DriverObject`]. +/// +/// [`Driver`]: drm::Driver +pub type DriverAllocImpl = <::Driver as drm::Driver>::Object; + /// GEM object functions, which must be implemented by drivers. pub trait DriverObject: Sync + Send + Sized { /// Parent `Driver` for this object. @@ -89,12 +94,12 @@ fn new( ) -> impl PinInit; /// Open a new handle to an existing object, associated with a File. - fn open(_obj: &::Object, _file: &DriverFile) -> Result { + fn open(_obj: &DriverAllocImpl, _file: &DriverFile) -> Result { Ok(()) } /// Close a handle to an existing object, associated with a File. - fn close(_obj: &::Object, _file: &DriverFile) {} + fn close(_obj: &DriverAllocImpl, _file: &DriverFile) {} } /// Trait that represents a GEM object subtype From 0023a1e8d01a9d400257d30c851bd16a29568809 Mon Sep 17 00:00:00 2001 From: Lyude Paul Date: Thu, 7 May 2026 17:59:21 -0400 Subject: [PATCH 121/131] rust/drm/gem: Use DeviceContext with GEM objects Now that we have the ability to represent the context in which a DRM device is in at compile-time, we can start carrying around this context with GEM object types in order to allow a driver to safely create GEM objects before a DRM device has registered with userspace. Signed-off-by: Lyude Paul Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260507220044.3204919-4-lyude@redhat.com Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/nova/driver.rs | 2 +- drivers/gpu/drm/nova/gem.rs | 15 +++++--- drivers/gpu/drm/tyr/driver.rs | 2 +- drivers/gpu/drm/tyr/gem.rs | 11 ++++-- rust/kernel/drm/device.rs | 18 ++++++---- rust/kernel/drm/driver.rs | 2 +- rust/kernel/drm/gem/mod.rs | 65 +++++++++++++++++++++++----------- rust/kernel/drm/gem/shmem.rs | 57 +++++++++++++++++------------ 8 files changed, 113 insertions(+), 59 deletions(-) diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 33fe088dd3b9..48933d86ddda 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -73,7 +73,7 @@ fn probe<'bound>( impl drm::Driver for NovaDriver { type Data = NovaData; type File = File; - type Object = gem::Object; + type Object = gem::Object; const INFO: drm::DriverInfo = INFO; diff --git a/drivers/gpu/drm/nova/gem.rs b/drivers/gpu/drm/nova/gem.rs index e073e174e257..9d8ff7de2c0f 100644 --- a/drivers/gpu/drm/nova/gem.rs +++ b/drivers/gpu/drm/nova/gem.rs @@ -2,7 +2,7 @@ use kernel::{ drm, - drm::{gem, gem::BaseObject}, + drm::{gem, gem::BaseObject, DeviceContext}, page, prelude::*, sync::aref::ARef, @@ -21,20 +21,27 @@ impl gem::DriverObject for NovaObject { type Driver = NovaDriver; type Args = (); - fn new(_dev: &NovaDevice, _size: usize, _args: Self::Args) -> impl PinInit { + fn new( + _dev: &NovaDevice, + _size: usize, + _args: Self::Args, + ) -> impl PinInit { try_pin_init!(NovaObject {}) } } impl NovaObject { /// Create a new DRM GEM object. - pub(crate) fn new(dev: &NovaDevice, size: usize) -> Result>> { + pub(crate) fn new( + dev: &NovaDevice, + size: usize, + ) -> Result>> { if size == 0 { return Err(EINVAL); } let aligned_size = page::page_align(size).ok_or(EINVAL)?; - gem::Object::new(dev, aligned_size, ()) + gem::Object::::new(dev, aligned_size, ()) } /// Look up a GEM object handle for a `File` and return an `ObjectRef` for it. diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index 3e53ecd33143..d063bc664cc1 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -181,7 +181,7 @@ fn drop(self: Pin<&mut Self>) {} impl drm::Driver for TyrDrmDriver { type Data = TyrDrmDeviceData; type File = TyrDrmFileData; - type Object = drm::gem::shmem::Object; + type Object = drm::gem::shmem::Object; const INFO: drm::DriverInfo = INFO; const FEAT_RENDER: bool = true; diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs index 1640a161754b..c6d4d6f9bae3 100644 --- a/drivers/gpu/drm/tyr/gem.rs +++ b/drivers/gpu/drm/tyr/gem.rs @@ -5,7 +5,10 @@ //! DRM's GEM subsystem with shmem backing. use kernel::{ - drm::gem, + drm::{ + gem, + DeviceContext, // + }, prelude::*, // }; @@ -30,7 +33,11 @@ impl gem::DriverObject for BoData { type Driver = TyrDrmDriver; type Args = BoCreateArgs; - fn new(_dev: &TyrDrmDevice, _size: usize, args: BoCreateArgs) -> impl PinInit { + fn new( + _dev: &TyrDrmDevice, + _size: usize, + args: BoCreateArgs, + ) -> impl PinInit { try_pin_init!(Self { flags: args.flags }) } } diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index a47d81a26724..477cf771fb10 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -177,13 +177,17 @@ const fn compute_features() -> u32 { master_set: None, master_drop: None, debugfs_init: None, - gem_create_object: T::Object::ALLOC_OPS.gem_create_object, - prime_handle_to_fd: T::Object::ALLOC_OPS.prime_handle_to_fd, - prime_fd_to_handle: T::Object::ALLOC_OPS.prime_fd_to_handle, - gem_prime_import: T::Object::ALLOC_OPS.gem_prime_import, - gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table, - dumb_create: T::Object::ALLOC_OPS.dumb_create, - dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, + + // Ignore the Uninit DeviceContext below. It is only provided because it is required by the + // compiler, and it is not actually used by these functions. + gem_create_object: T::Object::::ALLOC_OPS.gem_create_object, + prime_handle_to_fd: T::Object::::ALLOC_OPS.prime_handle_to_fd, + prime_fd_to_handle: T::Object::::ALLOC_OPS.prime_fd_to_handle, + gem_prime_import: T::Object::::ALLOC_OPS.gem_prime_import, + gem_prime_import_sg_table: T::Object::::ALLOC_OPS.gem_prime_import_sg_table, + dumb_create: T::Object::::ALLOC_OPS.dumb_create, + dumb_map_offset: T::Object::::ALLOC_OPS.dumb_map_offset, + show_fdinfo: None, fbdev_probe: None, diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 07af409959a9..25f7e233884d 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -111,7 +111,7 @@ pub trait Driver { type Data: Sync + Send; /// The type used to manage memory for this driver. - type Object: AllocImpl; + type Object: AllocImpl; /// The type used to represent a DRM File (client) type File: drm::file::DriverFile; diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index d4b5940ec0df..c8b66d816871 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -8,6 +8,10 @@ bindings, drm::{ self, + device::{ + DeviceContext, + Registered, // + }, driver::{ AllocImpl, AllocOps, // @@ -22,6 +26,7 @@ types::Opaque, }; use core::{ + marker::PhantomData, ops::Deref, ptr::NonNull, // }; @@ -76,7 +81,8 @@ unsafe fn dec_ref(obj: core::ptr::NonNull) { /// A type alias for retrieving the current [`AllocImpl`] for a given [`DriverObject`]. /// /// [`Driver`]: drm::Driver -pub type DriverAllocImpl = <::Driver as drm::Driver>::Object; +pub type DriverAllocImpl = + <::Driver as drm::Driver>::Object; /// GEM object functions, which must be implemented by drivers. pub trait DriverObject: Sync + Send + Sized { @@ -87,8 +93,8 @@ pub trait DriverObject: Sync + Send + Sized { type Args; /// Create a new driver data object for a GEM object of a given size. - fn new( - dev: &drm::Device, + fn new( + dev: &drm::Device, size: usize, args: Self::Args, ) -> impl PinInit; @@ -125,9 +131,12 @@ extern "C" fn open_callback( // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`. let file = unsafe { DriverFile::::from_raw(raw_file) }; - // SAFETY: `open_callback` is specified in the AllocOps structure for `DriverObject`, - // ensuring that `raw_obj` is contained within a `DriverObject` - let obj = unsafe { <::Object as IntoGEMObject>::from_raw(raw_obj) }; + // SAFETY: + // * `open_callback` is specified in the AllocOps structure for `DriverObject`, ensuring that + // `raw_obj` is contained within a `DriverAllocImpl` + // * It is only possible for `open_callback` to be called after device registration, ensuring + // that the object's device is in the `Registered` state. + let obj: &DriverAllocImpl = unsafe { IntoGEMObject::from_raw(raw_obj) }; match T::open(obj, file) { Err(e) => e.to_errno(), @@ -144,12 +153,12 @@ extern "C" fn close_callback( // SAFETY: `close_callback` is specified in the AllocOps structure for `Object`, ensuring // that `raw_obj` is indeed contained within a `Object`. - let obj = unsafe { <::Object as IntoGEMObject>::from_raw(raw_obj) }; + let obj: &DriverAllocImpl = unsafe { IntoGEMObject::from_raw(raw_obj) }; T::close(obj, file); } -impl IntoGEMObject for Object { +impl IntoGEMObject for Object { fn as_raw(&self) -> *mut bindings::drm_gem_object { self.obj.get() } @@ -157,7 +166,7 @@ fn as_raw(&self) -> *mut bindings::drm_gem_object { unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self { // SAFETY: `obj` is guaranteed to be in an `Object` via the safety contract of this // function - unsafe { &*crate::container_of!(Opaque::cast_from(self_ptr), Object, obj) } + unsafe { &*crate::container_of!(Opaque::cast_from(self_ptr), Object, obj) } } } @@ -174,7 +183,7 @@ fn size(&self) -> usize { fn create_handle(&self, file: &drm::File) -> Result where Self: AllocImpl, - D: drm::Driver, + D: drm::Driver = Self, File = F>, F: drm::file::DriverFile, { let mut handle: u32 = 0; @@ -189,7 +198,7 @@ fn create_handle(&self, file: &drm::File) -> Result fn lookup_handle(file: &drm::File, handle: u32) -> Result> where Self: AllocImpl, - D: drm::Driver, + D: drm::Driver = Self, File = F>, F: drm::file::DriverFile, { // SAFETY: The arguments are all valid per the type invariants. @@ -241,16 +250,18 @@ impl BaseObjectPrivate for T {} /// /// # Invariants /// -/// - `self.obj` is a valid instance of a `struct drm_gem_object`. +/// * `self.obj` is a valid instance of a `struct drm_gem_object`. +/// * Any type invariants of `Ctx` apply to the parent DRM device for this GEM object. #[repr(C)] #[pin_data] -pub struct Object { +pub struct Object { obj: Opaque, #[pin] data: T, + _ctx: PhantomData, } -impl Object { +impl Object { const OBJECT_FUNCS: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { free: Some(Self::free_callback), open: Some(open_callback::), @@ -270,11 +281,16 @@ impl Object { }; /// Create a new GEM object. - pub fn new(dev: &drm::Device, size: usize, args: T::Args) -> Result> { + pub fn new( + dev: &drm::Device, + size: usize, + args: T::Args, + ) -> Result> { let obj: Pin> = KBox::pin_init( try_pin_init!(Self { obj: Opaque::new(bindings::drm_gem_object::default()), data <- T::new(dev, size, args), + _ctx: PhantomData, }), GFP_KERNEL, )?; @@ -282,6 +298,8 @@ pub fn new(dev: &drm::Device, size: usize, args: T::Args) -> Result, size: usize, args: T::Args) -> Result &drm::Device { + pub fn dev(&self) -> &drm::Device { // SAFETY: // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM // object lives. // - The device we used for creating the gem object is passed as &drm::Device to // Object::::new(), so we know that `T::Driver` is the right generic parameter to use // here. + // - Any type invariants of `Ctx` are upheld by using the same `Ctx` for the `Device` we + // return. unsafe { drm::Device::from_raw((*self.as_raw()).dev) } } @@ -336,11 +356,16 @@ extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { } } -impl_aref_for_gem_obj!(impl for Object where T: DriverObject); +impl_aref_for_gem_obj! { + impl for Object + where + T: DriverObject, + C: DeviceContext +} -impl super::private::Sealed for Object {} +impl super::private::Sealed for Object {} -impl Deref for Object { +impl Deref for Object { type Target = T; fn deref(&self) -> &Self::Target { @@ -348,7 +373,7 @@ fn deref(&self) -> &Self::Target { } } -impl AllocImpl for Object { +impl AllocImpl for Object { type Driver = T::Driver; const ALLOC_OPS: AllocOps = AllocOps { diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 35d7523e164f..34af402899a0 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -15,7 +15,9 @@ driver, gem, private::Sealed, - Device, // + Device, + DeviceContext, + Registered, // }, error::to_result, prelude::*, @@ -23,11 +25,12 @@ types::Opaque, // }; use core::{ + marker::PhantomData, ops::{ Deref, DerefMut, // }, - ptr::NonNull, + ptr::NonNull, // }; use gem::{ BaseObjectPrivate, @@ -40,42 +43,49 @@ /// This is used with [`Object::new()`] to control various properties that can only be set when /// initially creating a shmem-backed GEM object. #[derive(Default)] -pub struct ObjectConfig<'a, T: DriverObject> { +pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Registered> { /// Whether to set the write-combine map flag. pub map_wc: bool, /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. - pub parent_resv_obj: Option<&'a Object>, + pub parent_resv_obj: Option<&'a Object>, } /// A shmem-backed GEM object. /// /// # Invariants /// -/// `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this -/// object. +/// - `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this +/// object. +/// - Any type invariants of `C` apply to the parent DRM device for this GEM object. #[repr(C)] #[pin_data] -pub struct Object { +pub struct Object { #[pin] obj: Opaque, /// Parent object that owns this object's DMA reservation object. - parent_resv_obj: Option>>, + parent_resv_obj: Option>>, #[pin] inner: T, + _ctx: PhantomData, } -super::impl_aref_for_gem_obj!(impl for Object where T: DriverObject); +super::impl_aref_for_gem_obj! { + impl for Object + where + T: DriverObject, + C: DeviceContext +} // SAFETY: All GEM objects are thread-safe. -unsafe impl Send for Object {} +unsafe impl Send for Object {} // SAFETY: All GEM objects are thread-safe. -unsafe impl Sync for Object {} +unsafe impl Sync for Object {} -impl Object { +impl Object { /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { free: Some(Self::free_callback), @@ -106,9 +116,9 @@ fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { /// /// Additional config options can be specified using `config`. pub fn new( - dev: &Device, + dev: &Device, size: usize, - config: ObjectConfig<'_, T>, + config: ObjectConfig<'_, T, C>, args: T::Args, ) -> Result> { let new: Pin> = KBox::try_pin_init( @@ -116,6 +126,7 @@ pub fn new( obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), inner <- T::new(dev, size, args), + _ctx: PhantomData::, }), GFP_KERNEL, )?; @@ -148,7 +159,7 @@ pub fn new( } /// Returns the `Device` that owns this GEM object. - pub fn dev(&self) -> &Device { + pub fn dev(&self) -> &Device { // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. unsafe { Device::from_raw((*self.as_raw()).dev) } } @@ -168,7 +179,7 @@ extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { // SAFETY: // - We verified above that `obj` is valid, which makes `this` valid // - This function is set in AllocOps, so we know that `this` is contained within a - // `Object` + // `Object` let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut(); // SAFETY: We're recovering the Kbox<> we created in gem_create_object() @@ -176,7 +187,7 @@ extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { } } -impl Deref for Object { +impl Deref for Object { type Target = T; fn deref(&self) -> &Self::Target { @@ -184,15 +195,15 @@ fn deref(&self) -> &Self::Target { } } -impl DerefMut for Object { +impl DerefMut for Object { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner } } -impl Sealed for Object {} +impl Sealed for Object {} -impl gem::IntoGEMObject for Object { +impl gem::IntoGEMObject for Object { fn as_raw(&self) -> *mut bindings::drm_gem_object { // SAFETY: // - Our immutable reference is proof that this is safe to dereference. @@ -200,18 +211,18 @@ fn as_raw(&self) -> *mut bindings::drm_gem_object { unsafe { &raw mut (*self.obj.get()).base } } - unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Object { + unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Self { // SAFETY: The safety contract of from_gem_obj() guarantees that `obj` is contained within // `Self` unsafe { let obj = Opaque::cast_from(container_of!(obj, bindings::drm_gem_shmem_object, base)); - &*container_of!(obj, Object, obj) + &*container_of!(obj, Self, obj) } } } -impl driver::AllocImpl for Object { +impl driver::AllocImpl for Object { type Driver = T::Driver; const ALLOC_OPS: driver::AllocOps = driver::AllocOps { From 6d6c3189f9b2b143efe1f536e8ea8b082e054a01 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 3 Jun 2026 16:30:18 +0900 Subject: [PATCH 122/131] gpu: nova-core: Hopper/Blackwell: add FSP falcon EMEM operations Add external memory (EMEM) read/write operations to the GPU's FSP falcon engine. These operations use Falcon PIO (Programmed I/O) to communicate with the FSP through indirect memory access. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260603-b4-blackwell-v13-1-d9f3a06939e0@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/fsp.rs | 80 +++++++++++++++++++++++++++-- drivers/gpu/nova-core/regs.rs | 18 +++++++ 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index d9f87262e8b1..0956a75ef7aa 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -6,12 +6,26 @@ //! The FSP falcon handles secure boot and Chain of Trust operations //! on Hopper and Blackwell architectures, replacing SEC2's role. -use kernel::io::register::RegisterBase; +use kernel::{ + io::{ + register::{ + RegisterBase, + WithBase, // + }, + Io, // + }, + prelude::*, +}; -use crate::falcon::{ - FalconEngine, - PFalcon2Base, - PFalconBase, // +use crate::{ + driver::Bar0, + falcon::{ + Falcon, + FalconEngine, + PFalcon2Base, + PFalconBase, // + }, + regs, }; /// Type specifying the `Fsp` falcon engine. Cannot be instantiated. @@ -26,3 +40,59 @@ impl RegisterBase for Fsp { } impl FalconEngine for Fsp {} + +impl Falcon { + /// Writes `data` to FSP external memory at offset `0`. + /// + /// `data` is interpreted as little-endian 32-bit words. Returns `EINVAL` + /// if the `data` length is not 4-byte aligned. + #[expect(dead_code)] + fn write_emem(&mut self, bar: &Bar0, data: &[u8]) -> Result { + if data.len() % 4 != 0 { + return Err(EINVAL); + } + + // Begin a write burst at offset `0`, auto-incrementing on each write. + bar.write( + WithBase::of::(), + regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true), + ); + + for chunk in data.chunks_exact(4) { + let value = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + + // Write the next 32-bit `value`; hardware advances the offset. + bar.write( + WithBase::of::(), + regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value), + ); + } + + Ok(()) + } + + /// Reads FSP external memory from offset `0` into `data`. + /// + /// `data` is stored as little-endian 32-bit words. Returns `EINVAL` if + /// the `data` length is not 4-byte aligned. + #[expect(dead_code)] + fn read_emem(&mut self, bar: &Bar0, data: &mut [u8]) -> Result { + if data.len() % 4 != 0 { + return Err(EINVAL); + } + + // Begin a read burst at offset `0`, auto-incrementing on each read. + bar.write( + WithBase::of::(), + regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true), + ); + + for chunk in data.chunks_exact_mut(4) { + // Read the next 32-bit word; hardware advances the offset. + let value = bar.read(regs::NV_PFALCON_FALCON_EMEMD::of::()).data(); + chunk.copy_from_slice(&value.to_le_bytes()); + } + + Ok(()) + } +} diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 2cb1f02f35a4..e602c7860459 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -475,6 +475,24 @@ pub(crate) fn vga_workspace_addr(self) -> Option { pub(crate) NV_PFALCON_FBIF_CTL(u32) @ PFalconBase + 0x00000624 { 7:7 allow_phys_no_ctx => bool; } + + // Falcon EMEM PIO registers (used by FSP on Hopper/Blackwell). + // These provide the falcon external memory communication interface. + + pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ PFalconBase + 0x00000ac0 { + /// EMEM byte offset (4-byte aligned) within the block. + 7:2 offs; + /// EMEM block to access. + 15:8 blk; + /// Auto-increment the offset after each write. + 24:24 aincw => bool; + /// Auto-increment the offset after each read. + 25:25 aincr => bool; + } + + pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ PFalconBase + 0x00000ac4 { + 31:0 data => u32; + } } impl NV_PFALCON_FALCON_DMACTL { From ab42c32347ef1229b4ebeae6a10d4202a61c4463 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 3 Jun 2026 16:30:19 +0900 Subject: [PATCH 123/131] gpu: nova-core: Hopper/Blackwell: add FSP message infrastructure FSP communication uses a pair of non-circular queues in the FSP falcon's EMEM, one for messages from the driver to FSP and one for replies, with the driver polling for response data. Add the queue registers and the low-level helpers used by the higher-level FSP message layer. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260603-b4-blackwell-v13-2-d9f3a06939e0@nvidia.com [acourbot: align register fields names with OpenRM.] [acourbot: represent registers as arrays of 8 instances, as per OpenRM.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/fsp.rs | 81 +++++++++++++++++++++++++++-- drivers/gpu/nova-core/regs.rs | 21 ++++++++ 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index 0956a75ef7aa..8fa47a8abb83 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -8,13 +8,16 @@ use kernel::{ io::{ + poll::read_poll_timeout, register::{ + Array, RegisterBase, WithBase, // }, Io, // }, prelude::*, + time::Delta, }; use crate::{ @@ -25,9 +28,13 @@ PFalcon2Base, PFalconBase, // }, - regs, + num, + regs, // }; +/// FSP message timeout in milliseconds. +const FSP_MSG_TIMEOUT_MS: i64 = 2000; + /// Type specifying the `Fsp` falcon engine. Cannot be instantiated. pub(crate) struct Fsp(()); @@ -46,7 +53,6 @@ impl Falcon { /// /// `data` is interpreted as little-endian 32-bit words. Returns `EINVAL` /// if the `data` length is not 4-byte aligned. - #[expect(dead_code)] fn write_emem(&mut self, bar: &Bar0, data: &[u8]) -> Result { if data.len() % 4 != 0 { return Err(EINVAL); @@ -75,7 +81,6 @@ fn write_emem(&mut self, bar: &Bar0, data: &[u8]) -> Result { /// /// `data` is stored as little-endian 32-bit words. Returns `EINVAL` if /// the `data` length is not 4-byte aligned. - #[expect(dead_code)] fn read_emem(&mut self, bar: &Bar0, data: &mut [u8]) -> Result { if data.len() % 4 != 0 { return Err(EINVAL); @@ -95,4 +100,74 @@ fn read_emem(&mut self, bar: &Bar0, data: &mut [u8]) -> Result { Ok(()) } + + /// Poll FSP for incoming data. + /// + /// Returns the size of available data in bytes, or 0 if no data is available. + /// + /// The FSP message queue is not circular. Pointers are reset to 0 after each + /// message exchange, so `tail >= head` is always true when data is present. + fn poll_msgq(&self, bar: &Bar0) -> u32 { + let head = bar.read(regs::NV_PFSP_MSGQ_HEAD::at(0)).val(); + let tail = bar.read(regs::NV_PFSP_MSGQ_TAIL::at(0)).val(); + + if head == tail { + return 0; + } + + // TAIL points at last DWORD written, so add 4 to get total size. + tail.saturating_sub(head).saturating_add(4) + } + + /// Writes `packet` to FSP EMEM and updates the queue pointers to notify FSP. + /// + /// Returns `EINVAL` if `packet` is empty or its length is not 4-byte aligned. + #[expect(dead_code)] + pub(crate) fn send_msg(&mut self, bar: &Bar0, packet: &[u8]) -> Result { + if packet.is_empty() { + return Err(EINVAL); + } + + self.write_emem(bar, packet)?; + + // Update queue pointers. TAIL points at the last DWORD written. + let tail_offset = u32::try_from(packet.len() - 4).map_err(|_| EINVAL)?; + bar.write( + Array::at(0), + regs::NV_PFSP_QUEUE_TAIL::zeroed().with_address(tail_offset), + ); + bar.write( + Array::at(0), + regs::NV_PFSP_QUEUE_HEAD::zeroed().with_address(0), + ); + + Ok(()) + } + + /// Reads the next message from FSP EMEM into a newly-allocated buffer and resets the queue + /// pointers. + /// + /// Returns `ETIMEDOUT` if no message was available until timeout, or a regular error code if a + /// memory allocation error occurred. + #[expect(dead_code)] + pub(crate) fn recv_msg(&mut self, bar: &Bar0) -> Result> { + let msg_size = read_poll_timeout( + || Ok(self.poll_msgq(bar)), + |&size| size > 0, + Delta::from_millis(10), + Delta::from_millis(FSP_MSG_TIMEOUT_MS), + ) + .map(num::u32_as_usize)?; + + let mut buffer = KVec::::new(); + buffer.resize(msg_size, 0, GFP_KERNEL)?; + + self.read_emem(bar, &mut buffer)?; + + // Reset message queue pointers after reading. + bar.write(Array::at(0), regs::NV_PFSP_MSGQ_TAIL::zeroed().with_val(0)); + bar.write(Array::at(0), regs::NV_PFSP_MSGQ_HEAD::zeroed().with_val(0)); + + Ok(buffer) + } } diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index e602c7860459..ce2392ef2f8b 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -579,6 +579,27 @@ pub(crate) fn mem_scrubbing_done(self) -> bool { } } +// FSP (Foundation Security Processor) queue registers for Hopper/Blackwell Chain of Trust. +// These registers manage falcon EMEM communication queues. + +register! { + pub(crate) NV_PFSP_QUEUE_HEAD(u32)[8] @ 0x008f2c00 { + 31:0 address => u32; + } + + pub(crate) NV_PFSP_QUEUE_TAIL(u32)[8] @ 0x008f2c04 { + 31:0 address => u32; + } + + pub(crate) NV_PFSP_MSGQ_HEAD(u32)[8] @ 0x008f2c80 { + 31:0 val => u32; + } + + pub(crate) NV_PFSP_MSGQ_TAIL(u32)[8] @ 0x008f2c84 { + 31:0 val => u32; + } +} + // The modules below provide registers that are not identical on all supported chips. They should // only be used in HAL modules. From 944a0fedca7ed226685d3b965dd1371ff34b3017 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 3 Jun 2026 16:30:20 +0900 Subject: [PATCH 124/131] gpu: nova-core: add MCTP/NVDM protocol types for firmware communication Add the MCTP (Management Component Transport Protocol) and NVDM (NVIDIA Data Model) wire-format types used for communication between the kernel driver and GPU firmware processors. This includes typed MCTP transport headers, NVDM message headers, and NVDM message type identifiers. Both the FSP boot path and the upcoming GSP RPC message queue share this protocol layer. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260603-b4-blackwell-v13-3-d9f3a06939e0@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mctp.rs | 90 ++++++++++++++++++++++++++++++ drivers/gpu/nova-core/nova_core.rs | 1 + 2 files changed, 91 insertions(+) create mode 100644 drivers/gpu/nova-core/mctp.rs diff --git a/drivers/gpu/nova-core/mctp.rs b/drivers/gpu/nova-core/mctp.rs new file mode 100644 index 000000000000..90e289d4c3fe --- /dev/null +++ b/drivers/gpu/nova-core/mctp.rs @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! MCTP/NVDM protocol types for NVIDIA GPU firmware communication. +//! +//! MCTP (Management Component Transport Protocol) carries NVDM (NVIDIA +//! Data Model) messages between the kernel driver and GPU firmware processors +//! such as FSP and GSP. + +#![expect(dead_code)] + +use kernel::pci::Vendor; + +/// NVDM message type identifiers carried over MCTP. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum NvdmType { + #[default] + /// Chain of Trust boot message. + Cot = 0x14, + /// FSP command response. + FspResponse = 0x15, +} + +impl TryFrom for NvdmType { + type Error = u8; + + fn try_from(value: u8) -> Result { + match value { + x if x == u8::from(Self::Cot) => Ok(Self::Cot), + x if x == u8::from(Self::FspResponse) => Ok(Self::FspResponse), + _ => Err(value), + } + } +} + +impl From for u8 { + fn from(value: NvdmType) -> Self { + value as u8 + } +} + +bitfield! { + pub(crate) struct MctpHeader(u32), "MCTP transport header for NVIDIA firmware messages." { + 31:31 som as bool, "Start-of-message bit."; + 30:30 eom as bool, "End-of-message bit."; + 29:28 seq as u8, "Packet sequence number."; + 23:16 seid as u8, "Source endpoint ID."; + } +} + +impl MctpHeader { + /// Builds a single-packet MCTP header (`SOM=1`, `EOM=1`, `SEQ=0`, `SEID=0`). + pub(crate) fn single_packet() -> Self { + Self::default().set_som(true).set_eom(true) + } + + /// Returns whether this is a complete single-packet message (`SOM=1` and `EOM=1`). + pub(crate) fn is_single_packet(self) -> bool { + self.som() && self.eom() + } +} + +/// MCTP message type for PCI vendor-defined messages. +const MSG_TYPE_VENDOR_PCI: u8 = 0x7e; + +bitfield! { + pub(crate) struct NvdmHeader(u32), "NVIDIA Vendor-Defined Message header over MCTP." { + 31:24 nvdm_type as u8 ?=> NvdmType, "NVDM message type."; + 23:8 vendor_id as u16, "PCI vendor ID."; + 6:0 msg_type as u8, "MCTP vendor-defined message type."; + } +} + +impl NvdmHeader { + /// Builds an NVDM header for the given message type. + pub(crate) fn new(nvdm_type: NvdmType) -> Self { + Self::default() + .set_msg_type(MSG_TYPE_VENDOR_PCI) + .set_vendor_id(Vendor::NVIDIA.as_raw()) + .set_nvdm_type(nvdm_type) + } + + /// Validates this header against the expected NVIDIA NVDM format and type. + pub(crate) fn validate(self, expected_type: NvdmType) -> bool { + self.msg_type() == MSG_TYPE_VENDOR_PCI + && self.vendor_id() == Vendor::NVIDIA.as_raw() + && matches!(self.nvdm_type(), Ok(nvdm_type) if nvdm_type == expected_type) + } +} diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 7b6c331da10e..9f0199f7b38c 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -20,6 +20,7 @@ mod fsp; mod gpu; mod gsp; +mod mctp; #[macro_use] mod num; mod regs; From e9e2a24d9493f50a3cc73e112b0dbdf91f319851 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 3 Jun 2026 16:30:21 +0900 Subject: [PATCH 125/131] gpu: nova-core: Hopper/Blackwell: add FSP send/receive messaging FSP exchanges are request/response: the driver sends an MCTP/NVDM message and must match the reply against the request before acting on it. Add the synchronous send-and-wait path that validates the response transport and message headers and confirms the reply corresponds to the request that was sent. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260603-b4-blackwell-v13-4-d9f3a06939e0@nvidia.com [acourbot: make `MessageToFsp` private.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/fsp.rs | 2 - drivers/gpu/nova-core/fsp.rs | 105 +++++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index 8fa47a8abb83..d322b81d7345 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -122,7 +122,6 @@ fn poll_msgq(&self, bar: &Bar0) -> u32 { /// Writes `packet` to FSP EMEM and updates the queue pointers to notify FSP. /// /// Returns `EINVAL` if `packet` is empty or its length is not 4-byte aligned. - #[expect(dead_code)] pub(crate) fn send_msg(&mut self, bar: &Bar0, packet: &[u8]) -> Result { if packet.is_empty() { return Err(EINVAL); @@ -149,7 +148,6 @@ pub(crate) fn send_msg(&mut self, bar: &Bar0, packet: &[u8]) -> Result { /// /// Returns `ETIMEDOUT` if no message was available until timeout, or a regular error code if a /// memory allocation error occurred. - #[expect(dead_code)] pub(crate) fn recv_msg(&mut self, bar: &Bar0) -> Result> { let msg_size = read_poll_timeout( || Ok(self.poll_msgq(bar)), diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 908dc112aa6f..f1f74832bc40 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -11,7 +11,11 @@ device, io::poll::read_poll_timeout, prelude::*, - time::Delta, // + time::Delta, + transmute::{ + AsBytes, + FromBytes, // + }, }; use crate::{ @@ -22,18 +26,52 @@ }, firmware::fsp::FspFirmware, gpu::Chipset, + mctp::{ + MctpHeader, + NvdmHeader, + NvdmType, // + }, regs, // }; mod hal; +/// FSP command response payload (`NVDM_PAYLOAD_COMMAND_RESPONSE`). +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct NvdmPayloadCommandResponse { + task_id: u32, + command_nvdm_type: u32, + error_code: u32, +} + +/// Complete FSP response structure with MCTP and NVDM headers. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspResponse { + mctp_header: MctpHeader, + nvdm_header: NvdmHeader, + response: NvdmPayloadCommandResponse, +} + +// SAFETY: FspResponse is a packed C struct with only integral fields. +unsafe impl FromBytes for FspResponse {} + +/// Trait implemented by types representing a message to send to FSP. +/// +/// This provides [`Fsp::send_sync_fsp`] with the information it needs to send +/// a given message, following the same pattern as GSP's `CommandToGsp`. +trait MessageToFsp: AsBytes { + /// NVDM type identifying this message to FSP. + const NVDM_TYPE: NvdmType; +} + /// FSP interface for Hopper/Blackwell GPUs. /// /// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot /// has completed. It owns the FSP falcon and the FMC firmware, which are used for the subsequent /// Chain of Trust boot. pub(crate) struct Fsp { - #[expect(dead_code)] falcon: Falcon, #[expect(dead_code)] fsp_fw: FspFirmware, @@ -69,4 +107,67 @@ pub(crate) fn wait_secure_boot( Ok(Fsp { falcon, fsp_fw }) } + + /// Sends a message to FSP and waits for the response. + #[expect(dead_code)] + fn send_sync_fsp(&mut self, dev: &device::Device, bar: &Bar0, msg: &M) -> Result + where + M: MessageToFsp, + { + self.falcon.send_msg(bar, msg.as_bytes())?; + + let response_buf = self.falcon.recv_msg(bar).inspect_err(|e| { + dev_err!(dev, "FSP response error: {:?}\n", e); + })?; + + let (response, _) = FspResponse::from_bytes_prefix(&response_buf[..]).ok_or_else(|| { + dev_err!(dev, "FSP response too small: {}\n", response_buf.len()); + EIO + })?; + + let mctp_header = response.mctp_header; + let nvdm_header = response.nvdm_header; + let command_nvdm_type = response.response.command_nvdm_type; + let error_code = response.response.error_code; + + if !mctp_header.is_single_packet() { + dev_err!( + dev, + "Unexpected MCTP header in FSP reply: {:x?}\n", + mctp_header, + ); + return Err(EIO); + } + + if !nvdm_header.validate(NvdmType::FspResponse) { + dev_err!( + dev, + "Unexpected NVDM header in FSP reply: {:x?}\n", + nvdm_header, + ); + return Err(EIO); + } + + if command_nvdm_type != u8::from(M::NVDM_TYPE).into() { + dev_err!( + dev, + "Expected NVDM type {:?} in reply, got {:#x}\n", + M::NVDM_TYPE, + command_nvdm_type + ); + return Err(EIO); + } + + if error_code != 0 { + dev_err!( + dev, + "NVDM command {:?} failed with error {:#x}\n", + M::NVDM_TYPE, + error_code + ); + return Err(EIO); + } + + Ok(()) + } } From a355d8142f343cffd28ff0f97c251a0334d3c0b3 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 3 Jun 2026 16:30:22 +0900 Subject: [PATCH 126/131] gpu: nova-core: Hopper/Blackwell: select FSP Chain of Trust version The FSP Chain of Trust handshake is versioned: Hopper speaks version 1 and Blackwell speaks version 2. Provide the version through the FSP HAL so the boot message carries the value FSP expects, and so chipsets that do not use FSP need not express a version at all. Signed-off-by: John Hubbard Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260603-b4-blackwell-v13-5-d9f3a06939e0@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/fsp/hal.rs | 8 +++++++- drivers/gpu/nova-core/fsp/hal/gb100.rs | 23 +++++++++++++++++++++++ drivers/gpu/nova-core/fsp/hal/gb202.rs | 4 ++++ drivers/gpu/nova-core/fsp/hal/gh100.rs | 15 ++++++++++++--- 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 drivers/gpu/nova-core/fsp/hal/gb100.rs diff --git a/drivers/gpu/nova-core/fsp/hal.rs b/drivers/gpu/nova-core/fsp/hal.rs index fc5ebb749c59..8aebe1800a64 100644 --- a/drivers/gpu/nova-core/fsp/hal.rs +++ b/drivers/gpu/nova-core/fsp/hal.rs @@ -9,19 +9,25 @@ }, }; +mod gb100; mod gb202; mod gh100; pub(super) trait FspHal { /// Returns the secure boot status from the architecture-specific `NV_THERM_I2CS_SCRATCH` register. fn fsp_boot_status(&self, bar: &Bar0) -> u32; + + /// Returns the FSP Chain of Trust protocol version this chipset advertises. + #[expect(dead_code)] + fn cot_version(&self) -> u16; } /// Returns the FSP HAL, or `None` if the architecture doesn't support FSP. pub(super) fn fsp_hal(chipset: Chipset) -> Option<&'static dyn FspHal> { match chipset.arch() { Architecture::Turing | Architecture::Ampere | Architecture::Ada => None, - Architecture::Hopper | Architecture::BlackwellGB10x => Some(gh100::GH100_HAL), + Architecture::Hopper => Some(gh100::GH100_HAL), + Architecture::BlackwellGB10x => Some(gb100::GB100_HAL), Architecture::BlackwellGB20x => Some(gb202::GB202_HAL), } } diff --git a/drivers/gpu/nova-core/fsp/hal/gb100.rs b/drivers/gpu/nova-core/fsp/hal/gb100.rs new file mode 100644 index 000000000000..d50aaba0a84f --- /dev/null +++ b/drivers/gpu/nova-core/fsp/hal/gb100.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::{ + driver::Bar0, + fsp::hal::FspHal, // +}; + +struct Gb100; + +impl FspHal for Gb100 { + fn fsp_boot_status(&self, bar: &Bar0) -> u32 { + // GB10x shares Hopper's FSP secure boot status register. + super::gh100::fsp_boot_status_gh100(bar) + } + + fn cot_version(&self) -> u16 { + 2 + } +} + +const GB100: Gb100 = Gb100; +pub(super) const GB100_HAL: &dyn FspHal = &GB100; diff --git a/drivers/gpu/nova-core/fsp/hal/gb202.rs b/drivers/gpu/nova-core/fsp/hal/gb202.rs index 2f08b6c9f308..2bca76c8fd64 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb202.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb202.rs @@ -17,6 +17,10 @@ fn fsp_boot_status(&self, bar: &Bar0) -> u32 { .fsp_boot_complete() .into() } + + fn cot_version(&self) -> u16 { + 2 + } } const GB202: Gb202 = Gb202; diff --git a/drivers/gpu/nova-core/fsp/hal/gh100.rs b/drivers/gpu/nova-core/fsp/hal/gh100.rs index 290fb55a81da..c38a7e96eb60 100644 --- a/drivers/gpu/nova-core/fsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gh100.rs @@ -11,11 +11,20 @@ struct Gh100; +/// Reads the FSP secure boot status from the Hopper/GB10x thermal scratch register. +pub(super) fn fsp_boot_status_gh100(bar: &Bar0) -> u32 { + bar.read(regs::gh100::NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE) + .fsp_boot_complete() + .into() +} + impl FspHal for Gh100 { fn fsp_boot_status(&self, bar: &Bar0) -> u32 { - bar.read(regs::gh100::NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE) - .fsp_boot_complete() - .into() + fsp_boot_status_gh100(bar) + } + + fn cot_version(&self) -> u16 { + 1 } } From d317e4585fa39bcee4d075f5c485494b0f238713 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 3 Jun 2026 16:30:23 +0900 Subject: [PATCH 127/131] gpu: nova-core: Hopper/Blackwell: add FSP Chain of Trust boot Build and send the Chain of Trust message to FSP, bundling the DMA-coherent boot parameters that FSP reads at boot time. Signed-off-by: John Hubbard Link: https://patch.msgid.link/20260603-b4-blackwell-v13-6-d9f3a06939e0@nvidia.com [acourbot: rename `frts_offset` to `frts_vidmem_offset`.] [acourbot: add note about frts_sysmem_* CoT members.] Co-developed-by: Alexandre Courbot Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware/fsp.rs | 2 - drivers/gpu/nova-core/fsp.rs | 147 +++++++++++++++++- drivers/gpu/nova-core/fsp/hal.rs | 1 - drivers/gpu/nova-core/gsp.rs | 1 + drivers/gpu/nova-core/gsp/fw.rs | 65 ++++++++ .../gpu/nova-core/gsp/fw/r570_144/bindings.rs | 82 ++++++++++ drivers/gpu/nova-core/gsp/hal/gh100.rs | 23 ++- drivers/gpu/nova-core/mctp.rs | 2 - 8 files changed, 310 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs index 9b211426a75a..6eaf1c684b9d 100644 --- a/drivers/gpu/nova-core/firmware/fsp.rs +++ b/drivers/gpu/nova-core/firmware/fsp.rs @@ -39,10 +39,8 @@ pub(crate) struct FmcSignatures { pub(crate) struct FspFirmware { /// FMC firmware image data (only the "image" ELF section). - #[expect(dead_code)] pub(crate) fmc_image: Coherent<[u8]>, /// FMC firmware signatures. - #[expect(dead_code)] pub(crate) fmc_sigs: KBox, } diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index f1f74832bc40..6eb5c09b3352 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -9,8 +9,14 @@ use kernel::{ device, + dma::Coherent, io::poll::read_poll_timeout, prelude::*, + ptr::{ + Alignable, + Alignment, // + }, + sizes::SZ_2M, time::Delta, transmute::{ AsBytes, @@ -24,13 +30,19 @@ fsp::Fsp as FspEngine, Falcon, // }, - firmware::fsp::FspFirmware, + fb::FbLayout, + firmware::fsp::{ + FmcSignatures, + FspFirmware, // + }, gpu::Chipset, + gsp::GspFmcBootParams, mctp::{ MctpHeader, NvdmHeader, NvdmType, // }, + num, regs, // }; @@ -66,6 +78,116 @@ trait MessageToFsp: AsBytes { const NVDM_TYPE: NvdmType; } +/// NVDM (NVIDIA Data Model) CoT (Chain of Trust) payload, the main +/// message body sent to FSP for Chain of Trust boot. +#[repr(C, packed)] +#[derive(Clone, Copy, Zeroable)] +struct NvdmPayloadCot { + version: u16, + size: u16, + gsp_fmc_sysmem_offset: u64, + frts_sysmem_offset: u64, + frts_sysmem_size: u32, + frts_vidmem_offset: u64, + frts_vidmem_size: u32, + sigs: FmcSignatures, + gsp_boot_args_sysmem_offset: u64, +} + +/// Complete FSP message structure with MCTP and NVDM headers. +#[repr(C)] +#[derive(Clone, Copy)] +struct FspMessage { + mctp_header: MctpHeader, + nvdm_header: NvdmHeader, + cot: NvdmPayloadCot, +} + +impl FspMessage { + /// Returns an in-place initializer for [`FspMessage`]. + fn new<'a>( + fb_layout: &FbLayout, + fsp_fw: &'a FspFirmware, + args: &'a FmcBootArgs, + ) -> Result + 'a> { + // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset + let frts_vidmem_offset = if !args.resume { + let frts_reserved_size = fb_layout.heap.len() + u64::from(fb_layout.pmu_reserved_size); + + frts_reserved_size + .align_up(Alignment::new::()) + .ok_or(EINVAL)? + } else { + 0 + }; + + let frts_size: u32 = if !args.resume { + fb_layout.frts.len().try_into()? + } else { + 0 + }; + + let version = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?.cot_version(); + let size = num::usize_into_u16::<{ core::mem::size_of::() }>(); + + Ok(init!(Self { + mctp_header: MctpHeader::single_packet(), + nvdm_header: NvdmHeader::new(NvdmType::Cot), + // The payload is packed, so we cannot use `init!`. Initialize it member-by-member using + // `chain`. + cot <- pin_init::init_zeroed(), + }) + .chain(move |msg| { + msg.cot.version = version; + msg.cot.size = size; + msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_handle(); + msg.cot.frts_vidmem_offset = frts_vidmem_offset; + msg.cot.frts_vidmem_size = frts_size; + // frts_sysmem_* intentionally left at zero for now, but will be needed for e.g. + // systems without VRAM. + msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_handle(); + msg.cot.sigs = *fsp_fw.fmc_sigs; + + Ok(()) + })) + } +} + +// SAFETY: `FspMessage` is `#[repr(C)]` with no padding, so all of its +// bytes are initialized. +unsafe impl AsBytes for FspMessage {} + +impl MessageToFsp for FspMessage { + const NVDM_TYPE: NvdmType = NvdmType::Cot; +} + +/// Bundled arguments for FMC boot via FSP Chain of Trust. +pub(crate) struct FmcBootArgs { + chipset: Chipset, + fmc_boot_params: Coherent, + resume: bool, +} + +impl FmcBootArgs { + /// Builds FMC boot arguments, allocating the DMA-coherent boot parameter + /// structure that FSP will read. + pub(crate) fn new( + dev: &device::Device, + chipset: Chipset, + wpr_meta_addr: u64, + libos_addr: u64, + resume: bool, + ) -> Result { + let init = GspFmcBootParams::new(wpr_meta_addr, libos_addr); + + Ok(Self { + chipset, + fmc_boot_params: Coherent::::init(dev, GFP_KERNEL, init)?, + resume, + }) + } +} + /// FSP interface for Hopper/Blackwell GPUs. /// /// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot @@ -73,7 +195,6 @@ trait MessageToFsp: AsBytes { /// Chain of Trust boot. pub(crate) struct Fsp { falcon: Falcon, - #[expect(dead_code)] fsp_fw: FspFirmware, } @@ -109,7 +230,6 @@ pub(crate) fn wait_secure_boot( } /// Sends a message to FSP and waits for the response. - #[expect(dead_code)] fn send_sync_fsp(&mut self, dev: &device::Device, bar: &Bar0, msg: &M) -> Result where M: MessageToFsp, @@ -170,4 +290,25 @@ fn send_sync_fsp(&mut self, dev: &device::Device, bar: &Bar0, msg: &M) -> Res Ok(()) } + + /// Boots GSP FMC via FSP Chain of Trust. + /// + /// Builds the CoT message from the pre-configured [`FmcBootArgs`], sends it + /// to FSP, and waits for the response. + pub(crate) fn boot_fmc( + &mut self, + dev: &device::Device, + bar: &Bar0, + fb_layout: &FbLayout, + args: &FmcBootArgs, + ) -> Result { + dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); + + let msg = KBox::init(FspMessage::new(fb_layout, &self.fsp_fw, args)?, GFP_KERNEL)?; + + self.send_sync_fsp(dev, bar, &*msg)?; + + dev_dbg!(dev, "FSP Chain of Trust completed successfully\n"); + Ok(()) + } } diff --git a/drivers/gpu/nova-core/fsp/hal.rs b/drivers/gpu/nova-core/fsp/hal.rs index 8aebe1800a64..86c595d70c8e 100644 --- a/drivers/gpu/nova-core/fsp/hal.rs +++ b/drivers/gpu/nova-core/fsp/hal.rs @@ -18,7 +18,6 @@ pub(super) trait FspHal { fn fsp_boot_status(&self, bar: &Bar0) -> u32; /// Returns the FSP Chain of Trust protocol version this chipset advertises. - #[expect(dead_code)] fn cot_version(&self) -> u16; } diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 1885cfa5cb38..69175ca3315c 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -25,6 +25,7 @@ mod sequencer; pub(crate) use fw::{ + GspFmcBootParams, GspFwWprMeta, LibosParams, // }; diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 0c54e8bf4bb3..4db0cfa4dc4d 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -934,3 +934,68 @@ fn new(cmdq: &Cmdq) -> impl Init + '_ { }) } } + +#[repr(u32)] +pub(crate) enum GspDmaTarget { + #[expect(dead_code)] + LocalFb = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB, + CoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM, + NoncoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM, +} + +type GspAcrBootGspRmParams = bindings::GSP_ACR_BOOT_GSP_RM_PARAMS; + +impl GspAcrBootGspRmParams { + fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init { + #[allow(non_snake_case)] + let params = init!(Self { + target: target as u32, + gspRmDescSize: num::usize_into_u32::<{ size_of::() }>(), + gspRmDescOffset: wpr_meta_addr, + bIsGspRmBoot: 1, + wprCarveoutOffset: 0, + wprCarveoutSize: 0, + __bindgen_padding_0: Default::default(), + }); + + params + } +} + +type GspRmParams = bindings::GSP_RM_PARAMS; + +impl GspRmParams { + fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init { + #[allow(non_snake_case)] + let params = init!(Self { + target: target as u32, + bootArgsOffset: libos_addr, + __bindgen_padding_0: Default::default(), + }); + + params + } +} + +pub(crate) type GspFmcBootParams = bindings::GSP_FMC_BOOT_PARAMS; + +// SAFETY: Padding is explicit and will not contain uninitialized data. +unsafe impl AsBytes for GspFmcBootParams {} +// SAFETY: This struct only contains integer types for which all bit patterns are valid. +unsafe impl FromBytes for GspFmcBootParams {} + +impl GspFmcBootParams { + pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init { + #[allow(non_snake_case)] + let init = init!(Self { + // Blackwell FSP obtains WPR info from other sources, so + // wprCarveoutOffset and wprCarveoutSize are left zero. + bootGspRmParams <- GspAcrBootGspRmParams::new(GspDmaTarget::CoherentSystem, + wpr_meta_addr), + gspRmParams <- GspRmParams::new(GspDmaTarget::NoncoherentSystem, libos_addr), + ..Zeroable::init_zeroed() + }); + + init + } +} diff --git a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs index 1d592bd3f9ed..ea350f9b2cc4 100644 --- a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs +++ b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs @@ -883,6 +883,88 @@ fn default() -> Self { } } } +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB: GSP_DMA_TARGET = 0; +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM: GSP_DMA_TARGET = 1; +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM: GSP_DMA_TARGET = 2; +pub const GSP_DMA_TARGET_GSP_DMA_TARGET_COUNT: GSP_DMA_TARGET = 3; +pub type GSP_DMA_TARGET = ffi::c_uint; +#[repr(C)] +#[derive(Debug, Default, Copy, Clone, MaybeZeroable)] +pub struct GSP_FMC_INIT_PARAMS { + pub regkeys: u32_, +} +#[repr(C)] +#[derive(Debug, Copy, Clone, MaybeZeroable)] +pub struct GSP_ACR_BOOT_GSP_RM_PARAMS { + pub target: GSP_DMA_TARGET, + pub gspRmDescSize: u32_, + pub gspRmDescOffset: u64_, + pub wprCarveoutOffset: u64_, + pub wprCarveoutSize: u32_, + pub bIsGspRmBoot: u8_, + pub __bindgen_padding_0: [u8; 3usize], +} +impl Default for GSP_ACR_BOOT_GSP_RM_PARAMS { + fn default() -> Self { + let mut s = ::core::mem::MaybeUninit::::uninit(); + unsafe { + ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone, MaybeZeroable)] +pub struct GSP_RM_PARAMS { + pub target: GSP_DMA_TARGET, + pub __bindgen_padding_0: [u8; 4usize], + pub bootArgsOffset: u64_, +} +impl Default for GSP_RM_PARAMS { + fn default() -> Self { + let mut s = ::core::mem::MaybeUninit::::uninit(); + unsafe { + ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone, MaybeZeroable)] +pub struct GSP_SPDM_PARAMS { + pub target: GSP_DMA_TARGET, + pub __bindgen_padding_0: [u8; 4usize], + pub payloadBufferOffset: u64_, + pub payloadBufferSize: u32_, + pub __bindgen_padding_1: [u8; 4usize], +} +impl Default for GSP_SPDM_PARAMS { + fn default() -> Self { + let mut s = ::core::mem::MaybeUninit::::uninit(); + unsafe { + ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone, MaybeZeroable)] +pub struct GSP_FMC_BOOT_PARAMS { + pub initParams: GSP_FMC_INIT_PARAMS, + pub __bindgen_padding_0: [u8; 4usize], + pub bootGspRmParams: GSP_ACR_BOOT_GSP_RM_PARAMS, + pub gspRmParams: GSP_RM_PARAMS, + pub gspSpdmParams: GSP_SPDM_PARAMS, +} +impl Default for GSP_FMC_BOOT_PARAMS { + fn default() -> Self { + let mut s = ::core::mem::MaybeUninit::::uninit(); + unsafe { + ::core::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} #[repr(C)] #[derive(Debug, Default, Copy, Clone, MaybeZeroable)] pub struct rpc_unloading_guest_driver_v1F_07 { diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index b25970dd4561..f41f3fea15ff 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -20,7 +20,10 @@ fsp::FspFirmware, FIRMWARE_VERSION, // }, - fsp::Fsp, + fsp::{ + FmcBootArgs, + Fsp, // + }, gpu::Chipset, gsp::{ boot::BootUnloadGuard, @@ -39,17 +42,27 @@ impl GspHal for Gh100 { /// the GSP boot internally - no manual GSP reset/boot is needed. fn boot<'a>( &self, - _gsp: &'a Gsp, + gsp: &'a Gsp, dev: &'a device::Device, bar: &'a Bar0, chipset: Chipset, - _fb_layout: &FbLayout, - _wpr_meta: &Coherent, + fb_layout: &FbLayout, + wpr_meta: &Coherent, _gsp_falcon: &'a Falcon, _sec2_falcon: &'a Falcon, ) -> Result> { let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; - let _fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?; + let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?; + + let args = FmcBootArgs::new( + dev, + chipset, + wpr_meta.dma_handle(), + gsp.libos.dma_handle(), + false, + )?; + + fsp.boot_fmc(dev, bar, fb_layout, &args)?; Err(ENOTSUPP) } diff --git a/drivers/gpu/nova-core/mctp.rs b/drivers/gpu/nova-core/mctp.rs index 90e289d4c3fe..482786e07bc7 100644 --- a/drivers/gpu/nova-core/mctp.rs +++ b/drivers/gpu/nova-core/mctp.rs @@ -7,8 +7,6 @@ //! Data Model) messages between the kernel driver and GPU firmware processors //! such as FSP and GSP. -#![expect(dead_code)] - use kernel::pci::Vendor; /// NVDM message type identifiers carried over MCTP. From a69a9e23dce95a1b7315f73b29200a58e0f54830 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 3 Jun 2026 16:30:24 +0900 Subject: [PATCH 128/131] gpu: nova-core: Hopper/Blackwell: add GSP lockdown release polling On Hopper and Blackwell, FSP boots GSP with hardware lockdown enabled. After FSP Chain of Trust completes, the driver must poll for lockdown release before proceeding with GSP initialization. Add the register bit and helper functions needed for this polling. Signed-off-by: John Hubbard Link: https://patch.msgid.link/20260603-b4-blackwell-v13-7-d9f3a06939e0@nvidia.com [acourbot: fix `lockdown_released` logic and add explanatory comments.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon/gsp.rs | 19 ++++++ drivers/gpu/nova-core/fsp.rs | 6 ++ drivers/gpu/nova-core/gsp/hal/gh100.rs | 90 +++++++++++++++++++++++++- drivers/gpu/nova-core/regs.rs | 2 + 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index df6d5a382c7a..98a1c1dc8465 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -24,6 +24,10 @@ regs, }; +/// Pattern returned by GSP register reads while the PRIV target mask still blocks CPU access. +const GSP_TARGET_MASK_LOCKED_PATTERN: u32 = 0xbadf_4100; +const GSP_TARGET_MASK_LOCKED_MASK: u32 = 0xffff_ff00; + /// Type specifying the `Gsp` falcon engine. Cannot be instantiated. pub(crate) struct Gsp(()); @@ -57,4 +61,19 @@ pub(crate) fn check_reload_completed(&self, bar: &Bar0, timeout: Delta) -> Resul ) .map(|_| true) } + + /// Returns whether the RISC-V branch privilege lockdown bit is set. + pub(crate) fn riscv_branch_privilege_lockdown(&self, bar: &Bar0) -> bool { + bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) + .riscv_br_priv_lockdown() + } + + /// Returns whether GSP registers can be read by the CPU. + pub(crate) fn priv_target_mask_released(&self, bar: &Bar0) -> bool { + let hwcfg2 = bar + .read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) + .into_raw(); + + hwcfg2 != 0 && (hwcfg2 & GSP_TARGET_MASK_LOCKED_MASK) != GSP_TARGET_MASK_LOCKED_PATTERN + } } diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 6eb5c09b3352..2fe5a5e6dd8e 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -186,6 +186,12 @@ pub(crate) fn new( resume, }) } + + /// DMA address of the FMC boot parameters, needed after boot for lockdown + /// release polling. + pub(crate) fn boot_params_dma_handle(&self) -> u64 { + self.fmc_boot_params.dma_handle() + } } /// FSP interface for Hopper/Blackwell GPUs. diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index f41f3fea15ff..57e31ef4819d 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -5,7 +5,9 @@ use kernel::{ device, - dma::Coherent, // + dma::Coherent, + io::poll::read_poll_timeout, + time::Delta, // }; use crate::{ @@ -33,6 +35,88 @@ }, }; +/// GSP falcon mailbox state, used to track lockdown release status. +struct GspMbox { + mbox0: u32, + mbox1: u32, +} + +impl GspMbox { + /// Reads both mailboxes from the GSP falcon. + fn read(gsp_falcon: &Falcon, bar: &Bar0) -> Self { + Self { + mbox0: gsp_falcon.read_mailbox0(bar), + mbox1: gsp_falcon.read_mailbox1(bar), + } + } + + /// Combines mailbox0 and mailbox1 into a 64-bit address. + fn combined_addr(&self) -> u64 { + (u64::from(self.mbox1) << 32) | u64::from(self.mbox0) + } + + /// Returns `true` if GSP lockdown has been released or a GSP-FMC error happened. + /// + /// Returns `true` both on successful lockdown release and on GSP-FMC-reported errors, since + /// either condition should stop the poll loop. + fn lockdown_released_or_error( + &self, + gsp_falcon: &Falcon, + bar: &Bar0, + fmc_boot_params_addr: u64, + ) -> bool { + // GSP-FMC normally clears the boot parameters address from the mailboxes early during + // boot. If the address is still there, keep polling rather than treating it as an error. + // Any other non-zero mailbox0 value is a GSP-FMC error code. + if self.mbox0 != 0 { + return self.combined_addr() != fmc_boot_params_addr; + } + + !gsp_falcon.riscv_branch_privilege_lockdown(bar) + } +} + +/// Waits for GSP lockdown to be released after FSP Chain of Trust. +fn wait_for_gsp_lockdown_release( + dev: &device::Device, + bar: &Bar0, + gsp_falcon: &Falcon, + fmc_boot_params_addr: u64, +) -> Result { + dev_dbg!(dev, "Waiting for GSP lockdown release\n"); + + let mbox = read_poll_timeout( + || { + // While the PRIV target mask is still locked to FSP, GSP register and mailbox reads + // are not meaningful. Wait until HWCFG2 says the CPU can read them. + Ok(match gsp_falcon.priv_target_mask_released(bar) { + false => None, + true => Some(GspMbox::read(gsp_falcon, bar)), + }) + }, + |mbox| match mbox { + None => false, + Some(mbox) => mbox.lockdown_released_or_error(gsp_falcon, bar, fmc_boot_params_addr), + }, + Delta::from_millis(10), + Delta::from_secs(30), + ) + .inspect_err(|_| { + dev_err!(dev, "GSP lockdown release timeout\n"); + })? + .ok_or(EIO)?; + + // If polling stopped with a non-zero mailbox0, it was not the boot parameters address + // anymore and therefore represents a GSP-FMC error code. + if mbox.mbox0 != 0 { + dev_err!(dev, "GSP-FMC boot failed (mbox: {:#x})\n", mbox.mbox0); + return Err(EIO); + } + + dev_dbg!(dev, "GSP lockdown released\n"); + Ok(()) +} + struct Gh100; impl GspHal for Gh100 { @@ -48,7 +132,7 @@ fn boot<'a>( chipset: Chipset, fb_layout: &FbLayout, wpr_meta: &Coherent, - _gsp_falcon: &'a Falcon, + gsp_falcon: &'a Falcon, _sec2_falcon: &'a Falcon, ) -> Result> { let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; @@ -64,6 +148,8 @@ fn boot<'a>( fsp.boot_fmc(dev, bar, fb_layout, &args)?; + wait_for_gsp_lockdown_release(dev, bar, gsp_falcon, args.boot_params_dma_handle())?; + Err(ENOTSUPP) } } diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index ce2392ef2f8b..cc24ab10b922 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -363,6 +363,8 @@ pub(crate) fn vga_workspace_addr(self) -> Option { pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ PFalconBase + 0x000000f4 { /// Signal indicating that reset is completed (GA102+). 31:31 reset_ready => bool; + /// RISC-V branch privilege lockdown bit. + 13:13 riscv_br_priv_lockdown => bool; /// Set to 0 after memory scrubbing is completed. 12:12 mem_scrubbing => bool; 10:10 riscv => bool; From c7fea1f70944c01acdaa3eaeb299770174cef37d Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Wed, 3 Jun 2026 16:30:25 +0900 Subject: [PATCH 129/131] gpu: nova-core: add non-sec2 unload path For non-sec2 it is only required to wait for GSP falcon to halt. This is because GSP does the main work of unloading on GPUs not using sec2. Signed-off-by: Eliot Courtney [ jhubbard: use Result instead of Result<()> in the UnloadBundle impl ] Signed-off-by: John Hubbard Link: https://patch.msgid.link/20260603-b4-blackwell-v13-8-d9f3a06939e0@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/hal/gh100.rs | 38 ++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 57e31ef4819d..63aef7c65ce5 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -29,7 +29,10 @@ gpu::Chipset, gsp::{ boot::BootUnloadGuard, - hal::GspHal, + hal::{ + GspHal, + UnloadBundle, // + }, Gsp, GspFwWprMeta, // }, @@ -117,6 +120,28 @@ fn wait_for_gsp_lockdown_release( Ok(()) } +struct FspUnloadBundle; + +impl UnloadBundle for FspUnloadBundle { + fn run( + &self, + dev: &device::Device, + bar: &Bar0, + gsp_falcon: &Falcon, + _sec2_falcon: &Falcon, + ) -> Result { + // GSP falcon does most of the work of resetting, so just wait for it to finish. + read_poll_timeout( + || Ok(gsp_falcon.is_riscv_active(bar)), + |&active| !active, + Delta::from_millis(10), + Delta::from_secs(5), + ) + .map(|_| ()) + .inspect_err(|_| dev_err!(dev, "GSP falcon failed to halt\n")) + } +} + struct Gh100; impl GspHal for Gh100 { @@ -133,9 +158,18 @@ fn boot<'a>( fb_layout: &FbLayout, wpr_meta: &Coherent, gsp_falcon: &'a Falcon, - _sec2_falcon: &'a Falcon, + sec2_falcon: &'a Falcon, ) -> Result> { let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; + + let unload_bundle = crate::gsp::UnloadBundle( + KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox + ); + + // Wrap the unload bundle into a drop guard so it is automatically run upon failure. + let _unload_guard = + BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, Some(unload_bundle)); + let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?; let args = FmcBootArgs::new( From 723bd79ca9e492cc91850094a2892bde0345c51a Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 3 Jun 2026 16:30:26 +0900 Subject: [PATCH 130/131] gpu: nova-core: gsp: enable FSP boot path Now that all the elements are in place, enable the FSP boot path so Hopper and Blackwell can boot. Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260603-b4-blackwell-v13-9-d9f3a06939e0@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/hal/gh100.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 63aef7c65ce5..9494258b9fc4 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -167,7 +167,7 @@ fn boot<'a>( ); // Wrap the unload bundle into a drop guard so it is automatically run upon failure. - let _unload_guard = + let unload_guard = BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, Some(unload_bundle)); let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?; @@ -184,7 +184,7 @@ fn boot<'a>( wait_for_gsp_lockdown_release(dev, bar, gsp_falcon, args.boot_params_dma_handle())?; - Err(ENOTSUPP) + Ok(unload_guard) } } From 99676aed1fec109d62822e21a06760eb098dc5f4 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 2 Jun 2026 18:04:07 +0100 Subject: [PATCH 131/131] gpu: nova-core: move lifetime to `Bar0` Currently Nova code uses `&'a Bar0` a lot. This is `&'a Mmio`, where `Mmio` represents an owned MMIO region; this type only exists as a target for `Deref` so `Bar` and `IoMem` can share code and should be avoided to be named directly. The upcoming I/O projection series would make `Io` trait much simpler to implement, and thus the owned MMIO type would be removed in favour of direct `Io` implementation on `Bar` and `IoMem`. Add lifetime parameter to `Bar0<'a>` and change it to be alias of `&'a pci::Bar<'a, ..>`. This also prepares Nova core so that when I/O projection series land, this could be changed to using a MMIO view type directly which avoids double indirection. Signed-off-by: Gary Guo Acked-by: Alexandre Courbot Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260602170416.2268531-1-gary@kernel.org [ Rebase onto latest drm-rust-next (Blackwell enablement). - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 2 +- drivers/gpu/nova-core/falcon.rs | 44 +++++++++++-------- drivers/gpu/nova-core/falcon/fsp.rs | 10 ++--- drivers/gpu/nova-core/falcon/gsp.rs | 8 ++-- drivers/gpu/nova-core/falcon/hal.rs | 12 ++--- drivers/gpu/nova-core/falcon/hal/ga102.rs | 18 ++++---- drivers/gpu/nova-core/falcon/hal/tu102.rs | 12 ++--- drivers/gpu/nova-core/fb.rs | 6 +-- drivers/gpu/nova-core/fb/hal.rs | 8 ++-- drivers/gpu/nova-core/fb/hal/ga100.rs | 14 +++--- drivers/gpu/nova-core/fb/hal/ga102.rs | 10 ++--- drivers/gpu/nova-core/fb/hal/gb100.rs | 12 ++--- drivers/gpu/nova-core/fb/hal/gb202.rs | 12 ++--- drivers/gpu/nova-core/fb/hal/gh100.rs | 8 ++-- drivers/gpu/nova-core/fb/hal/tu102.rs | 16 +++---- drivers/gpu/nova-core/firmware/booter.rs | 4 +- drivers/gpu/nova-core/firmware/fwsec.rs | 4 +- .../nova-core/firmware/fwsec/bootloader.rs | 2 +- drivers/gpu/nova-core/fsp.rs | 6 +-- drivers/gpu/nova-core/fsp/hal.rs | 2 +- drivers/gpu/nova-core/fsp/hal/gb100.rs | 2 +- drivers/gpu/nova-core/fsp/hal/gb202.rs | 2 +- drivers/gpu/nova-core/fsp/hal/gh100.rs | 4 +- drivers/gpu/nova-core/gpu.rs | 9 ++-- drivers/gpu/nova-core/gpu/hal.rs | 2 +- drivers/gpu/nova-core/gpu/hal/gh100.rs | 2 +- drivers/gpu/nova-core/gpu/hal/tu102.rs | 2 +- drivers/gpu/nova-core/gsp/boot.rs | 10 ++--- drivers/gpu/nova-core/gsp/cmdq.rs | 10 ++--- drivers/gpu/nova-core/gsp/hal.rs | 6 +-- drivers/gpu/nova-core/gsp/hal/gh100.rs | 10 ++--- drivers/gpu/nova-core/gsp/hal/tu102.rs | 14 +++--- drivers/gpu/nova-core/gsp/sequencer.rs | 4 +- drivers/gpu/nova-core/regs.rs | 2 +- drivers/gpu/nova-core/vbios.rs | 8 ++-- 35 files changed, 152 insertions(+), 145 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index ade73da68be5..5738d4ac521b 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -36,7 +36,7 @@ pub(crate) struct NovaCore<'bound> { const BAR0_SIZE: usize = SZ_16M; -pub(crate) type Bar0 = kernel::io::Mmio; +pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; kernel::pci_device_table!( PCI_TABLE, diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 053ce5bea6cd..94c7696a6493 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -373,7 +373,7 @@ pub(crate) fn new(dev: &device::Device, chipset: Chipset) -> Result { } /// Resets DMA-related registers. - pub(crate) fn dma_reset(&self, bar: &Bar0) { + pub(crate) fn dma_reset(&self, bar: Bar0<'_>) { bar.update(regs::NV_PFALCON_FBIF_CTL::of::(), |v| { v.with_allow_phys_no_ctx(true) }); @@ -385,7 +385,7 @@ pub(crate) fn dma_reset(&self, bar: &Bar0) { } /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete. - pub(crate) fn reset(&self, bar: &Bar0) -> Result { + pub(crate) fn reset(&self, bar: Bar0<'_>) -> Result { self.hal.reset_eng(bar)?; self.hal.select_core(self, bar)?; self.hal.reset_wait_mem_scrubbing(bar)?; @@ -404,7 +404,11 @@ pub(crate) fn reset(&self, bar: &Bar0) -> Result { /// Write a slice to Falcon IMEM memory using programmed I/O (PIO). /// /// Returns `EINVAL` if `img.len()` is not a multiple of 4. - fn pio_wr_imem_slice(&self, bar: &Bar0, load_offsets: FalconPioImemLoadTarget<'_>) -> Result { + fn pio_wr_imem_slice( + &self, + bar: Bar0<'_>, + load_offsets: FalconPioImemLoadTarget<'_>, + ) -> Result { // Rejecting misaligned images here allows us to avoid checking // inside the loops. if load_offsets.data.len() % 4 != 0 { @@ -441,7 +445,11 @@ fn pio_wr_imem_slice(&self, bar: &Bar0, load_offsets: FalconPioImemLoadTarget<'_ /// Write a slice to Falcon DMEM memory using programmed I/O (PIO). /// /// Returns `EINVAL` if `img.len()` is not a multiple of 4. - fn pio_wr_dmem_slice(&self, bar: &Bar0, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result { + fn pio_wr_dmem_slice( + &self, + bar: Bar0<'_>, + load_offsets: FalconPioDmemLoadTarget<'_>, + ) -> Result { // Rejecting misaligned images here allows us to avoid checking // inside the loops. if load_offsets.data.len() % 4 != 0 { @@ -469,7 +477,7 @@ fn pio_wr_dmem_slice(&self, bar: &Bar0, load_offsets: FalconPioDmemLoadTarget<'_ /// Perform a PIO copy into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it. pub(crate) fn pio_load + FalconPioLoadable>( &self, - bar: &Bar0, + bar: Bar0<'_>, fw: &F, ) -> Result { bar.update(regs::NV_PFALCON_FBIF_CTL::of::(), |v| { @@ -505,7 +513,7 @@ pub(crate) fn pio_load + FalconPioLoadable>( /// `sec` is set if the loaded firmware is expected to run in secure mode. fn dma_wr( &self, - bar: &Bar0, + bar: Bar0<'_>, dma_obj: &Coherent<[u8]>, target_mem: FalconMem, load_offsets: FalconDmaLoadTarget, @@ -612,7 +620,7 @@ fn dma_wr( fn dma_load + FalconDmaLoadable>( &self, dev: &Device, - bar: &Bar0, + bar: Bar0<'_>, fw: &F, ) -> Result { // DMA object with firmware content as the source of the DMA engine. @@ -660,7 +668,7 @@ fn dma_load + FalconDmaLoadable>( } /// Wait until the falcon CPU is halted. - pub(crate) fn wait_till_halted(&self, bar: &Bar0) -> Result<()> { + pub(crate) fn wait_till_halted(&self, bar: Bar0<'_>) -> Result<()> { // TIMEOUT: arbitrarily large value, firmwares should complete in less than 2 seconds. read_poll_timeout( || Ok(bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::())), @@ -673,7 +681,7 @@ pub(crate) fn wait_till_halted(&self, bar: &Bar0) -> Result<()> { } /// Start the falcon CPU. - pub(crate) fn start(&self, bar: &Bar0) -> Result<()> { + pub(crate) fn start(&self, bar: Bar0<'_>) -> Result<()> { match bar .read(regs::NV_PFALCON_FALCON_CPUCTL::of::()) .alias_en() @@ -692,7 +700,7 @@ pub(crate) fn start(&self, bar: &Bar0) -> Result<()> { } /// Writes values to the mailbox registers if provided. - pub(crate) fn write_mailboxes(&self, bar: &Bar0, mbox0: Option, mbox1: Option) { + pub(crate) fn write_mailboxes(&self, bar: Bar0<'_>, mbox0: Option, mbox1: Option) { if let Some(mbox0) = mbox0 { bar.write( WithBase::of::(), @@ -709,19 +717,19 @@ pub(crate) fn write_mailboxes(&self, bar: &Bar0, mbox0: Option, mbox1: Opti } /// Reads the value from `mbox0` register. - pub(crate) fn read_mailbox0(&self, bar: &Bar0) -> u32 { + pub(crate) fn read_mailbox0(&self, bar: Bar0<'_>) -> u32 { bar.read(regs::NV_PFALCON_FALCON_MAILBOX0::of::()) .value() } /// Reads the value from `mbox1` register. - pub(crate) fn read_mailbox1(&self, bar: &Bar0) -> u32 { + pub(crate) fn read_mailbox1(&self, bar: Bar0<'_>) -> u32 { bar.read(regs::NV_PFALCON_FALCON_MAILBOX1::of::()) .value() } /// Reads values from both mailbox registers. - pub(crate) fn read_mailboxes(&self, bar: &Bar0) -> (u32, u32) { + pub(crate) fn read_mailboxes(&self, bar: Bar0<'_>) -> (u32, u32) { let mbox0 = self.read_mailbox0(bar); let mbox1 = self.read_mailbox1(bar); @@ -737,7 +745,7 @@ pub(crate) fn read_mailboxes(&self, bar: &Bar0) -> (u32, u32) { /// the `MBOX0` and `MBOX1` registers. pub(crate) fn boot( &self, - bar: &Bar0, + bar: Bar0<'_>, mbox0: Option, mbox1: Option, ) -> Result<(u32, u32)> { @@ -751,7 +759,7 @@ pub(crate) fn boot( /// falcon instance. `engine_id_mask` and `ucode_id` are obtained from the firmware header. pub(crate) fn signature_reg_fuse_version( &self, - bar: &Bar0, + bar: Bar0<'_>, engine_id_mask: u16, ucode_id: u8, ) -> Result { @@ -762,7 +770,7 @@ pub(crate) fn signature_reg_fuse_version( /// Check if the RISC-V core is active. /// /// Returns `true` if the RISC-V core is active, `false` otherwise. - pub(crate) fn is_riscv_active(&self, bar: &Bar0) -> bool { + pub(crate) fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { self.hal.is_riscv_active(bar) } @@ -771,7 +779,7 @@ pub(crate) fn is_riscv_active(&self, bar: &Bar0) -> bool { pub(crate) fn load + FalconDmaLoadable>( &self, dev: &Device, - bar: &Bar0, + bar: Bar0<'_>, fw: &F, ) -> Result { match self.hal.load_method() { @@ -781,7 +789,7 @@ pub(crate) fn load + FalconDmaLoadable>( } /// Write the application version to the OS register. - pub(crate) fn write_os_version(&self, bar: &Bar0, app_version: u32) { + pub(crate) fn write_os_version(&self, bar: Bar0<'_>, app_version: u32) { bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version), diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index d322b81d7345..52cdb84ef0e8 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -53,7 +53,7 @@ impl Falcon { /// /// `data` is interpreted as little-endian 32-bit words. Returns `EINVAL` /// if the `data` length is not 4-byte aligned. - fn write_emem(&mut self, bar: &Bar0, data: &[u8]) -> Result { + fn write_emem(&mut self, bar: Bar0<'_>, data: &[u8]) -> Result { if data.len() % 4 != 0 { return Err(EINVAL); } @@ -81,7 +81,7 @@ fn write_emem(&mut self, bar: &Bar0, data: &[u8]) -> Result { /// /// `data` is stored as little-endian 32-bit words. Returns `EINVAL` if /// the `data` length is not 4-byte aligned. - fn read_emem(&mut self, bar: &Bar0, data: &mut [u8]) -> Result { + fn read_emem(&mut self, bar: Bar0<'_>, data: &mut [u8]) -> Result { if data.len() % 4 != 0 { return Err(EINVAL); } @@ -107,7 +107,7 @@ fn read_emem(&mut self, bar: &Bar0, data: &mut [u8]) -> Result { /// /// The FSP message queue is not circular. Pointers are reset to 0 after each /// message exchange, so `tail >= head` is always true when data is present. - fn poll_msgq(&self, bar: &Bar0) -> u32 { + fn poll_msgq(&self, bar: Bar0<'_>) -> u32 { let head = bar.read(regs::NV_PFSP_MSGQ_HEAD::at(0)).val(); let tail = bar.read(regs::NV_PFSP_MSGQ_TAIL::at(0)).val(); @@ -122,7 +122,7 @@ fn poll_msgq(&self, bar: &Bar0) -> u32 { /// Writes `packet` to FSP EMEM and updates the queue pointers to notify FSP. /// /// Returns `EINVAL` if `packet` is empty or its length is not 4-byte aligned. - pub(crate) fn send_msg(&mut self, bar: &Bar0, packet: &[u8]) -> Result { + pub(crate) fn send_msg(&mut self, bar: Bar0<'_>, packet: &[u8]) -> Result { if packet.is_empty() { return Err(EINVAL); } @@ -148,7 +148,7 @@ pub(crate) fn send_msg(&mut self, bar: &Bar0, packet: &[u8]) -> Result { /// /// Returns `ETIMEDOUT` if no message was available until timeout, or a regular error code if a /// memory allocation error occurred. - pub(crate) fn recv_msg(&mut self, bar: &Bar0) -> Result> { + pub(crate) fn recv_msg(&mut self, bar: Bar0<'_>) -> Result> { let msg_size = read_poll_timeout( || Ok(self.poll_msgq(bar)), |&size| size > 0, diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index 98a1c1dc8465..d1f6f7fcffff 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -44,7 +44,7 @@ impl FalconEngine for Gsp {} impl Falcon { /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to /// allow GSP to signal CPU for processing new messages in message queue. - pub(crate) fn clear_swgen0_intr(&self, bar: &Bar0) { + pub(crate) fn clear_swgen0_intr(&self, bar: Bar0<'_>) { bar.write( WithBase::of::(), regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true), @@ -52,7 +52,7 @@ pub(crate) fn clear_swgen0_intr(&self, bar: &Bar0) { } /// Checks if GSP reload/resume has completed during the boot process. - pub(crate) fn check_reload_completed(&self, bar: &Bar0, timeout: Delta) -> Result { + pub(crate) fn check_reload_completed(&self, bar: Bar0<'_>, timeout: Delta) -> Result { read_poll_timeout( || Ok(bar.read(regs::NV_PGC6_BSI_SECURE_SCRATCH_14)), |val| val.boot_stage_3_handoff(), @@ -63,13 +63,13 @@ pub(crate) fn check_reload_completed(&self, bar: &Bar0, timeout: Delta) -> Resul } /// Returns whether the RISC-V branch privilege lockdown bit is set. - pub(crate) fn riscv_branch_privilege_lockdown(&self, bar: &Bar0) -> bool { + pub(crate) fn riscv_branch_privilege_lockdown(&self, bar: Bar0<'_>) -> bool { bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) .riscv_br_priv_lockdown() } /// Returns whether GSP registers can be read by the CPU. - pub(crate) fn priv_target_mask_released(&self, bar: &Bar0) -> bool { + pub(crate) fn priv_target_mask_released(&self, bar: Bar0<'_>) -> bool { let hwcfg2 = bar .read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) .into_raw(); diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index a524c8096b67..89b56823906b 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -34,7 +34,7 @@ pub(crate) enum LoadMethod { /// registers. pub(crate) trait FalconHal: Send + Sync { /// Activates the Falcon core if the engine is a risvc/falcon dual engine. - fn select_core(&self, _falcon: &Falcon, _bar: &Bar0) -> Result { + fn select_core(&self, _falcon: &Falcon, _bar: Bar0<'_>) -> Result { Ok(()) } @@ -43,23 +43,23 @@ fn select_core(&self, _falcon: &Falcon, _bar: &Bar0) -> Result { fn signature_reg_fuse_version( &self, falcon: &Falcon, - bar: &Bar0, + bar: Bar0<'_>, engine_id_mask: u16, ucode_id: u8, ) -> Result; /// Program the boot ROM registers prior to starting a secure firmware. - fn program_brom(&self, falcon: &Falcon, bar: &Bar0, params: &FalconBromParams); + fn program_brom(&self, falcon: &Falcon, bar: Bar0<'_>, params: &FalconBromParams); /// Check if the RISC-V core is active. /// Returns `true` if the RISC-V core is active, `false` otherwise. - fn is_riscv_active(&self, bar: &Bar0) -> bool; + fn is_riscv_active(&self, bar: Bar0<'_>) -> bool; /// Wait for memory scrubbing to complete. - fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result; + fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result; /// Reset the falcon engine. - fn reset_eng(&self, bar: &Bar0) -> Result; + fn reset_eng(&self, bar: Bar0<'_>) -> Result; /// Returns the method used to load data into the falcon's memory. /// diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs index 3df1ffa159b8..cf6ce47e6b25 100644 --- a/drivers/gpu/nova-core/falcon/hal/ga102.rs +++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs @@ -31,7 +31,7 @@ use super::FalconHal; -fn select_core_ga102(bar: &Bar0) -> Result { +fn select_core_ga102(bar: Bar0<'_>) -> Result { let bcr_ctrl = bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::()); if bcr_ctrl.core_select() != PeregrineCoreSelect::Falcon { bar.write( @@ -53,7 +53,7 @@ fn select_core_ga102(bar: &Bar0) -> Result { fn signature_reg_fuse_version_ga102( dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, engine_id_mask: u16, ucode_id: u8, ) -> Result { @@ -86,7 +86,7 @@ fn signature_reg_fuse_version_ga102( Ok(u16::BITS - reg_fuse_version.leading_zeros()) } -fn program_brom_ga102(bar: &Bar0, params: &FalconBromParams) { +fn program_brom_ga102(bar: Bar0<'_>, params: &FalconBromParams) { bar.write( WithBase::of::().at(0), regs::NV_PFALCON2_FALCON_BROM_PARAADDR::zeroed().with_value(params.pkc_data_offset), @@ -115,30 +115,30 @@ pub(super) fn new() -> Self { } impl FalconHal for Ga102 { - fn select_core(&self, _falcon: &Falcon, bar: &Bar0) -> Result { + fn select_core(&self, _falcon: &Falcon, bar: Bar0<'_>) -> Result { select_core_ga102::(bar) } fn signature_reg_fuse_version( &self, falcon: &Falcon, - bar: &Bar0, + bar: Bar0<'_>, engine_id_mask: u16, ucode_id: u8, ) -> Result { signature_reg_fuse_version_ga102(&falcon.dev, bar, engine_id_mask, ucode_id) } - fn program_brom(&self, _falcon: &Falcon, bar: &Bar0, params: &FalconBromParams) { + fn program_brom(&self, _falcon: &Falcon, bar: Bar0<'_>, params: &FalconBromParams) { program_brom_ga102::(bar, params); } - fn is_riscv_active(&self, bar: &Bar0) -> bool { + fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { bar.read(regs::NV_PRISCV_RISCV_CPUCTL::of::()) .active_stat() } - fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result { + fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 20ms. read_poll_timeout( || Ok(bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::())), @@ -149,7 +149,7 @@ fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result { .map(|_| ()) } - fn reset_eng(&self, bar: &Bar0) -> Result { + fn reset_eng(&self, bar: Bar0<'_>) -> Result { let _ = bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::()); // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index d8f5d271811b..3aaee3869312 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -34,28 +34,28 @@ pub(super) fn new() -> Self { } impl FalconHal for Tu102 { - fn select_core(&self, _falcon: &Falcon, _bar: &Bar0) -> Result { + fn select_core(&self, _falcon: &Falcon, _bar: Bar0<'_>) -> Result { Ok(()) } fn signature_reg_fuse_version( &self, _falcon: &Falcon, - _bar: &Bar0, + _bar: Bar0<'_>, _engine_id_mask: u16, _ucode_id: u8, ) -> Result { Ok(0) } - fn program_brom(&self, _falcon: &Falcon, _bar: &Bar0, _params: &FalconBromParams) {} + fn program_brom(&self, _falcon: &Falcon, _bar: Bar0<'_>, _params: &FalconBromParams) {} - fn is_riscv_active(&self, bar: &Bar0) -> bool { + fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { bar.read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::()) .active_stat() } - fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result { + fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 10ms. read_poll_timeout( || Ok(bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::())), @@ -66,7 +66,7 @@ fn reset_wait_mem_scrubbing(&self, bar: &Bar0) -> Result { .map(|_| ()) } - fn reset_eng(&self, bar: &Bar0) -> Result { + fn reset_eng(&self, bar: Bar0<'_>) -> Result { regs::NV_PFALCON_FALCON_ENGINE::reset_engine::(bar); self.reset_wait_mem_scrubbing(bar)?; diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 0aaee718c2c3..725e428154cf 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -46,7 +46,7 @@ pub(crate) struct SysmemFlush<'sys> { /// Chipset we are operating on. chipset: Chipset, device: &'sys device::Device, - bar: &'sys Bar0, + bar: Bar0<'sys>, /// Keep the page alive as long as we need it. page: CoherentHandle, } @@ -55,7 +55,7 @@ impl<'sys> SysmemFlush<'sys> { /// Allocate a memory page and register it as the sysmem flush page. pub(crate) fn register( dev: &'sys device::Device, - bar: &'sys Bar0, + bar: Bar0<'sys>, chipset: Chipset, ) -> Result { let page = CoherentHandle::alloc(dev, kernel::page::PAGE_SIZE, GFP_KERNEL)?; @@ -171,7 +171,7 @@ pub(crate) struct FbLayout { impl FbLayout { /// Computes the FB layout for `chipset` required to run the `gsp_fw` GSP firmware. - pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result { + pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, gsp_fw: &GspFirmware) -> Result { let hal = hal::fb_hal(chipset); let fb = { diff --git a/drivers/gpu/nova-core/fb/hal.rs b/drivers/gpu/nova-core/fb/hal.rs index be9e75f990f0..714f0b51cd8f 100644 --- a/drivers/gpu/nova-core/fb/hal.rs +++ b/drivers/gpu/nova-core/fb/hal.rs @@ -20,18 +20,18 @@ pub(crate) trait FbHal { /// Returns the address of the currently-registered sysmem flush page. - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64; + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64; /// Register `addr` as the address of the sysmem flush page. /// /// This might fail if the address is too large for the receiving register. - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result; + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result; /// Returns `true` is display is supported. - fn supports_display(&self, bar: &Bar0) -> bool; + fn supports_display(&self, bar: Bar0<'_>) -> bool; /// Returns the VRAM size, in bytes. - fn vidmem_size(&self, bar: &Bar0) -> u64; + fn vidmem_size(&self, bar: Bar0<'_>) -> u64; /// Returns the amount of VRAM to reserve for the PMU. fn pmu_reserved_size(&self) -> u32; diff --git a/drivers/gpu/nova-core/fb/hal/ga100.rs b/drivers/gpu/nova-core/fb/hal/ga100.rs index af95f1bdd273..3cc1caf361c7 100644 --- a/drivers/gpu/nova-core/fb/hal/ga100.rs +++ b/drivers/gpu/nova-core/fb/hal/ga100.rs @@ -17,13 +17,13 @@ struct Ga100; -pub(super) fn read_sysmem_flush_page_ga100(bar: &Bar0) -> u64 { +pub(super) fn read_sysmem_flush_page_ga100(bar: Bar0<'_>) -> u64 { u64::from(bar.read(regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR).adr_39_08()) << FLUSH_SYSMEM_ADDR_SHIFT | u64::from(bar.read(regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI).adr_63_40()) << FLUSH_SYSMEM_ADDR_SHIFT_HI } -pub(super) fn write_sysmem_flush_page_ga100(bar: &Bar0, addr: u64) { +pub(super) fn write_sysmem_flush_page_ga100(bar: Bar0<'_>, addr: u64) { bar.write_reg( regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr_63_40( Bounded::::from(addr) @@ -40,7 +40,7 @@ pub(super) fn write_sysmem_flush_page_ga100(bar: &Bar0, addr: u64) { ); } -pub(super) fn display_enabled_ga100(bar: &Bar0) -> bool { +pub(super) fn display_enabled_ga100(bar: Bar0<'_>) -> bool { !bar.read(regs::ga100::NV_FUSE_STATUS_OPT_DISPLAY) .display_disabled() } @@ -50,21 +50,21 @@ pub(super) fn display_enabled_ga100(bar: &Bar0) -> bool { const FLUSH_SYSMEM_ADDR_SHIFT_HI: u32 = 40; impl FbHal for Ga100 { - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { read_sysmem_flush_page_ga100(bar) } - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { write_sysmem_flush_page_ga100(bar, addr); Ok(()) } - fn supports_display(&self, bar: &Bar0) -> bool { + fn supports_display(&self, bar: Bar0<'_>) -> bool { display_enabled_ga100(bar) } - fn vidmem_size(&self, bar: &Bar0) -> u64 { + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { super::tu102::vidmem_size_gp102(bar) } diff --git a/drivers/gpu/nova-core/fb/hal/ga102.rs b/drivers/gpu/nova-core/fb/hal/ga102.rs index e06dbb08349e..44a2cf8a00f1 100644 --- a/drivers/gpu/nova-core/fb/hal/ga102.rs +++ b/drivers/gpu/nova-core/fb/hal/ga102.rs @@ -12,28 +12,28 @@ regs, // }; -pub(super) fn vidmem_size_ga102(bar: &Bar0) -> u64 { +pub(super) fn vidmem_size_ga102(bar: Bar0<'_>) -> u64 { bar.read(regs::NV_USABLE_FB_SIZE_IN_MB).usable_fb_size() } struct Ga102; impl FbHal for Ga102 { - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { super::ga100::read_sysmem_flush_page_ga100(bar) } - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { super::ga100::write_sysmem_flush_page_ga100(bar, addr); Ok(()) } - fn supports_display(&self, bar: &Bar0) -> bool { + fn supports_display(&self, bar: Bar0<'_>) -> bool { super::ga100::display_enabled_ga100(bar) } - fn vidmem_size(&self, bar: &Bar0) -> u64 { + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { vidmem_size_ga102(bar) } diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index ecea4ff446ff..6e0eba101ca1 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -33,7 +33,7 @@ impl RegisterBase for Gb100 { const BASE: usize = 0x0087_0000; } -fn read_sysmem_flush_page_gb100(bar: &Bar0) -> u64 { +fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 { let lo = u64::from( bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::()) .adr(), @@ -50,7 +50,7 @@ fn read_sysmem_flush_page_gb100(bar: &Bar0) -> u64 { /// /// Both the primary and EG (egress) register pairs must be programmed to the same address, /// as required by hardware. -fn write_sysmem_flush_page_gb100(bar: &Bar0, addr: Bounded) { +fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded) { // CAST: lower 32 bits. Hardware ignores bits 7:0. let addr_lo = *addr as u32; let addr_hi = addr.shr::<32, 20>().cast::(); @@ -84,11 +84,11 @@ pub(super) const fn pmu_reserved_size_gb100() -> u32 { } impl FbHal for Gb100 { - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { read_sysmem_flush_page_gb100(bar) } - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { let addr = Bounded::::try_new(addr).ok_or(EINVAL)?; write_sysmem_flush_page_gb100(bar, addr); @@ -96,11 +96,11 @@ fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { Ok(()) } - fn supports_display(&self, bar: &Bar0) -> bool { + fn supports_display(&self, bar: Bar0<'_>) -> bool { super::ga100::display_enabled_ga100(bar) } - fn vidmem_size(&self, bar: &Bar0) -> u64 { + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { super::ga102::vidmem_size_ga102(bar) } diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs index fa5c3f7f2b2e..038d1278c634 100644 --- a/drivers/gpu/nova-core/fb/hal/gb202.rs +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -28,7 +28,7 @@ impl RegisterBase for Gb202 { const BASE: usize = 0x008a_0000; } -fn read_sysmem_flush_page_gb202(bar: &Bar0) -> u64 { +fn read_sysmem_flush_page_gb202(bar: Bar0<'_>) -> u64 { let lo = u64::from( bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::()) .adr(), @@ -42,7 +42,7 @@ fn read_sysmem_flush_page_gb202(bar: &Bar0) -> u64 { } /// Write the sysmem flush page address through the GB20x FBHUB0 registers. -fn write_sysmem_flush_page_gb202(bar: &Bar0, addr: Bounded) { +fn write_sysmem_flush_page_gb202(bar: Bar0<'_>, addr: Bounded) { // Write HI first. The hardware will trigger the flush on the LO write. bar.write( regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::(), @@ -57,11 +57,11 @@ fn write_sysmem_flush_page_gb202(bar: &Bar0, addr: Bounded) { } impl FbHal for Gb202 { - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { read_sysmem_flush_page_gb202(bar) } - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { let addr = Bounded::::try_new(addr).ok_or(EINVAL)?; write_sysmem_flush_page_gb202(bar, addr); @@ -69,11 +69,11 @@ fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { Ok(()) } - fn supports_display(&self, bar: &Bar0) -> bool { + fn supports_display(&self, bar: Bar0<'_>) -> bool { super::ga100::display_enabled_ga100(bar) } - fn vidmem_size(&self, bar: &Bar0) -> u64 { + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { super::ga102::vidmem_size_ga102(bar) } diff --git a/drivers/gpu/nova-core/fb/hal/gh100.rs b/drivers/gpu/nova-core/fb/hal/gh100.rs index 8f79c72b1823..5450c7254dad 100644 --- a/drivers/gpu/nova-core/fb/hal/gh100.rs +++ b/drivers/gpu/nova-core/fb/hal/gh100.rs @@ -14,21 +14,21 @@ struct Gh100; impl FbHal for Gh100 { - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { super::ga100::read_sysmem_flush_page_ga100(bar) } - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { super::ga100::write_sysmem_flush_page_ga100(bar, addr); Ok(()) } - fn supports_display(&self, bar: &Bar0) -> bool { + fn supports_display(&self, bar: Bar0<'_>) -> bool { super::ga100::display_enabled_ga100(bar) } - fn vidmem_size(&self, bar: &Bar0) -> u64 { + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { super::ga102::vidmem_size_ga102(bar) } diff --git a/drivers/gpu/nova-core/fb/hal/tu102.rs b/drivers/gpu/nova-core/fb/hal/tu102.rs index 62d9357987f7..f629e8e9d5d5 100644 --- a/drivers/gpu/nova-core/fb/hal/tu102.rs +++ b/drivers/gpu/nova-core/fb/hal/tu102.rs @@ -17,11 +17,11 @@ /// to be used by HALs. pub(super) const FLUSH_SYSMEM_ADDR_SHIFT: u32 = 8; -pub(super) fn read_sysmem_flush_page_gm107(bar: &Bar0) -> u64 { +pub(super) fn read_sysmem_flush_page_gm107(bar: Bar0<'_>) -> u64 { u64::from(bar.read(regs::NV_PFB_NISO_FLUSH_SYSMEM_ADDR).adr_39_08()) << FLUSH_SYSMEM_ADDR_SHIFT } -pub(super) fn write_sysmem_flush_page_gm107(bar: &Bar0, addr: u64) -> Result { +pub(super) fn write_sysmem_flush_page_gm107(bar: Bar0<'_>, addr: u64) -> Result { // Check that the address doesn't overflow the receiving 32-bit register. u32::try_from(addr >> FLUSH_SYSMEM_ADDR_SHIFT) .map_err(|_| EINVAL) @@ -30,12 +30,12 @@ pub(super) fn write_sysmem_flush_page_gm107(bar: &Bar0, addr: u64) -> Result { }) } -pub(super) fn display_enabled_gm107(bar: &Bar0) -> bool { +pub(super) fn display_enabled_gm107(bar: Bar0<'_>) -> bool { !bar.read(regs::gm107::NV_FUSE_STATUS_OPT_DISPLAY) .display_disabled() } -pub(super) fn vidmem_size_gp102(bar: &Bar0) -> u64 { +pub(super) fn vidmem_size_gp102(bar: Bar0<'_>) -> u64 { bar.read(regs::NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE) .usable_fb_size() } @@ -55,19 +55,19 @@ pub(super) const fn frts_size_tu102() -> u64 { struct Tu102; impl FbHal for Tu102 { - fn read_sysmem_flush_page(&self, bar: &Bar0) -> u64 { + fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { read_sysmem_flush_page_gm107(bar) } - fn write_sysmem_flush_page(&self, bar: &Bar0, addr: u64) -> Result { + fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { write_sysmem_flush_page_gm107(bar, addr) } - fn supports_display(&self, bar: &Bar0) -> bool { + fn supports_display(&self, bar: Bar0<'_>) -> bool { display_enabled_gm107(bar) } - fn vidmem_size(&self, bar: &Bar0) -> u64 { + fn vidmem_size(&self, bar: Bar0<'_>) -> u64 { vidmem_size_gp102(bar) } diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index c5e17605e1a3..d9313ac361af 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -294,7 +294,7 @@ pub(crate) fn new( chipset: Chipset, ver: &str, falcon: &Falcon<::Target>, - bar: &Bar0, + bar: Bar0<'_>, ) -> Result { let fw_name = match kind { BooterKind::Loader => "booter_load", @@ -405,7 +405,7 @@ pub(crate) fn new( pub(crate) fn run( &self, dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, sec2_falcon: &Falcon, wpr_meta: &Coherent, ) -> Result { diff --git a/drivers/gpu/nova-core/firmware/fwsec.rs b/drivers/gpu/nova-core/firmware/fwsec.rs index 4108f28cd338..199ae2adb664 100644 --- a/drivers/gpu/nova-core/firmware/fwsec.rs +++ b/drivers/gpu/nova-core/firmware/fwsec.rs @@ -321,7 +321,7 @@ impl FwsecFirmware { pub(crate) fn new( dev: &Device, falcon: &Falcon, - bar: &Bar0, + bar: Bar0<'_>, bios: &Vbios, cmd: FwsecCommand, ) -> Result { @@ -394,7 +394,7 @@ pub(crate) fn run( &self, dev: &Device, falcon: &Falcon, - bar: &Bar0, + bar: Bar0<'_>, ) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index bcb713a868e2..039920dc340b 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -280,7 +280,7 @@ pub(crate) fn run( &self, dev: &Device, falcon: &Falcon, - bar: &Bar0, + bar: Bar0<'_>, ) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 2fe5a5e6dd8e..8fc243c66e35 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -212,7 +212,7 @@ impl Fsp { /// interface is not used before secure boot has completed. pub(crate) fn wait_secure_boot( dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, chipset: Chipset, fsp_fw: FspFirmware, ) -> Result { @@ -236,7 +236,7 @@ pub(crate) fn wait_secure_boot( } /// Sends a message to FSP and waits for the response. - fn send_sync_fsp(&mut self, dev: &device::Device, bar: &Bar0, msg: &M) -> Result + fn send_sync_fsp(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) -> Result where M: MessageToFsp, { @@ -304,7 +304,7 @@ fn send_sync_fsp(&mut self, dev: &device::Device, bar: &Bar0, msg: &M) -> Res pub(crate) fn boot_fmc( &mut self, dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, fb_layout: &FbLayout, args: &FmcBootArgs, ) -> Result { diff --git a/drivers/gpu/nova-core/fsp/hal.rs b/drivers/gpu/nova-core/fsp/hal.rs index 86c595d70c8e..b6f2624bb13d 100644 --- a/drivers/gpu/nova-core/fsp/hal.rs +++ b/drivers/gpu/nova-core/fsp/hal.rs @@ -15,7 +15,7 @@ pub(super) trait FspHal { /// Returns the secure boot status from the architecture-specific `NV_THERM_I2CS_SCRATCH` register. - fn fsp_boot_status(&self, bar: &Bar0) -> u32; + fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32; /// Returns the FSP Chain of Trust protocol version this chipset advertises. fn cot_version(&self) -> u16; diff --git a/drivers/gpu/nova-core/fsp/hal/gb100.rs b/drivers/gpu/nova-core/fsp/hal/gb100.rs index d50aaba0a84f..42f5ecfc6400 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb100.rs @@ -9,7 +9,7 @@ struct Gb100; impl FspHal for Gb100 { - fn fsp_boot_status(&self, bar: &Bar0) -> u32 { + fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32 { // GB10x shares Hopper's FSP secure boot status register. super::gh100::fsp_boot_status_gh100(bar) } diff --git a/drivers/gpu/nova-core/fsp/hal/gb202.rs b/drivers/gpu/nova-core/fsp/hal/gb202.rs index 2bca76c8fd64..1091b169a645 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb202.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb202.rs @@ -12,7 +12,7 @@ struct Gb202; impl FspHal for Gb202 { - fn fsp_boot_status(&self, bar: &Bar0) -> u32 { + fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32 { bar.read(regs::gb202::NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE) .fsp_boot_complete() .into() diff --git a/drivers/gpu/nova-core/fsp/hal/gh100.rs b/drivers/gpu/nova-core/fsp/hal/gh100.rs index c38a7e96eb60..291acaf2845a 100644 --- a/drivers/gpu/nova-core/fsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gh100.rs @@ -12,14 +12,14 @@ struct Gh100; /// Reads the FSP secure boot status from the Hopper/GB10x thermal scratch register. -pub(super) fn fsp_boot_status_gh100(bar: &Bar0) -> u32 { +pub(super) fn fsp_boot_status_gh100(bar: Bar0<'_>) -> u32 { bar.read(regs::gh100::NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE) .fsp_boot_complete() .into() } impl FspHal for Gh100 { - fn fsp_boot_status(&self, bar: &Bar0) -> u32 { + fn fsp_boot_status(&self, bar: Bar0<'_>) -> u32 { fsp_boot_status_gh100(bar) } diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index b7341bde04be..b3c91731db45 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -208,7 +208,7 @@ pub(crate) struct Spec { } impl Spec { - fn new(dev: &device::Device, bar: &Bar0) -> Result { + fn new(dev: &device::Device, bar: Bar0<'_>) -> Result { // Some brief notes about boot0 and boot42, in chronological order: // // NV04 through NV50: @@ -269,7 +269,7 @@ pub(crate) struct Gpu<'gpu> { device: &'gpu device::Device, spec: Spec, /// MMIO mapping of PCI BAR 0. - bar: &'gpu Bar0, + bar: Bar0<'gpu>, /// System memory page required for flushing all pending GPU-side memory writes done through /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). sysmem_flush: SysmemFlush<'gpu>, @@ -287,7 +287,7 @@ pub(crate) struct Gpu<'gpu> { impl<'gpu> Gpu<'gpu> { pub(crate) fn new( pdev: &'gpu pci::Device>, - bar: &'gpu Bar0, + bar: Bar0<'gpu>, ) -> impl PinInit + 'gpu { try_pin_init!(Self { device: pdev.as_ref(), @@ -308,8 +308,6 @@ pub(crate) fn new( .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?; }, - bar, - sysmem_flush: SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?, gsp_falcon: Falcon::new( @@ -326,6 +324,7 @@ pub(crate) fn new( // outside of the constructed `Gpu`, ensuring that the unload sequence is properly run // in case of failure. unload_bundle: gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)?, + bar, }) } } diff --git a/drivers/gpu/nova-core/gpu/hal.rs b/drivers/gpu/nova-core/gpu/hal.rs index cd833bd49b9b..3f25882d0e56 100644 --- a/drivers/gpu/nova-core/gpu/hal.rs +++ b/drivers/gpu/nova-core/gpu/hal.rs @@ -20,7 +20,7 @@ pub(crate) trait GpuHal { /// Waits for GFW_BOOT completion if required by this hardware family. - fn wait_gfw_boot_completion(&self, bar: &Bar0) -> Result; + fn wait_gfw_boot_completion(&self, bar: Bar0<'_>) -> Result; /// Returns the DMA mask for the current architecture. fn dma_mask(&self) -> DmaMask; diff --git a/drivers/gpu/nova-core/gpu/hal/gh100.rs b/drivers/gpu/nova-core/gpu/hal/gh100.rs index 17778a618900..e3f8ba0fab33 100644 --- a/drivers/gpu/nova-core/gpu/hal/gh100.rs +++ b/drivers/gpu/nova-core/gpu/hal/gh100.rs @@ -14,7 +14,7 @@ struct Gh100; impl GpuHal for Gh100 { - fn wait_gfw_boot_completion(&self, _bar: &Bar0) -> Result { + fn wait_gfw_boot_completion(&self, _bar: Bar0<'_>) -> Result { Ok(()) } diff --git a/drivers/gpu/nova-core/gpu/hal/tu102.rs b/drivers/gpu/nova-core/gpu/hal/tu102.rs index 125478bfe07a..b0732e53edea 100644 --- a/drivers/gpu/nova-core/gpu/hal/tu102.rs +++ b/drivers/gpu/nova-core/gpu/hal/tu102.rs @@ -55,7 +55,7 @@ impl GpuHal for Tu102 { /// This function waits for a signal indicating that core initialization is complete. Before /// this signal is received, little can be done with the GPU. This signal is set by the FWSEC /// running on the GSP in Heavy-secured mode. - fn wait_gfw_boot_completion(&self, bar: &Bar0) -> Result { + fn wait_gfw_boot_completion(&self, bar: Bar0<'_>) -> Result { // Before accessing the completion status in `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05`, we must // first check `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK`. This is because // `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05` becomes accessible only after the secure firmware diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 8c316fa2e585..8afb62d689cb 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -38,7 +38,7 @@ pub(super) struct BootUnloadArgs<'a> { gsp: &'a super::Gsp, dev: &'a device::Device, - bar: &'a Bar0, + bar: Bar0<'a>, gsp_falcon: &'a Falcon, sec2_falcon: &'a Falcon, unload_bundle: Option, @@ -57,7 +57,7 @@ impl<'a> BootUnloadGuard<'a> { pub(super) fn new( gsp: &'a super::Gsp, dev: &'a device::Device, - bar: &'a Bar0, + bar: Bar0<'a>, gsp_falcon: &'a Falcon, sec2_falcon: &'a Falcon, unload_bundle: Option, @@ -104,7 +104,7 @@ impl super::Gsp { pub(crate) fn boot( self: Pin<&mut Self>, pdev: &pci::Device, - bar: &Bar0, + bar: Bar0<'_>, chipset: Chipset, gsp_falcon: &Falcon, sec2_falcon: &Falcon, @@ -166,7 +166,7 @@ pub(crate) fn boot( /// Shut down the GSP and wait until it is offline. fn shutdown_gsp( cmdq: &Cmdq, - bar: &Bar0, + bar: Bar0<'_>, gsp_falcon: &Falcon, mode: commands::PowerStateLevel, ) -> Result { @@ -190,7 +190,7 @@ fn shutdown_gsp( pub(crate) fn unload( &self, dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, gsp_falcon: &Falcon, sec2_falcon: &Falcon, unload_bundle: Option, diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 275da9b1ee0e..0bc5a95a9cd7 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -532,7 +532,7 @@ fn calculate_checksum>(it: T) -> u32 { } /// Notifies the GSP that we have updated the command queue pointers. - fn notify_gsp(bar: &Bar0) { + fn notify_gsp(bar: Bar0<'_>) { bar.write_reg(regs::NV_PGSP_QUEUE_HEAD::zeroed().with_address(0u32)); } @@ -552,7 +552,7 @@ fn notify_gsp(bar: &Bar0) { /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command and reply initializers are propagated as-is. - pub(crate) fn send_command(&self, bar: &Bar0, command: M) -> Result + pub(crate) fn send_command(&self, bar: Bar0<'_>, command: M) -> Result where M: CommandToGsp, M::Reply: MessageFromGsp, @@ -580,7 +580,7 @@ pub(crate) fn send_command(&self, bar: &Bar0, command: M) -> Result /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command initializers are propagated as-is. - pub(crate) fn send_command_no_wait(&self, bar: &Bar0, command: M) -> Result + pub(crate) fn send_command_no_wait(&self, bar: Bar0<'_>, command: M) -> Result where M: CommandToGsp, Error: From, @@ -624,7 +624,7 @@ impl CmdqInner { /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command initializers are propagated as-is. - fn send_single_command(&mut self, bar: &Bar0, command: M) -> Result + fn send_single_command(&mut self, bar: Bar0<'_>, command: M) -> Result where M: CommandToGsp, // This allows all error types, including `Infallible`, to be used for `M::InitError`. @@ -694,7 +694,7 @@ fn send_single_command(&mut self, bar: &Bar0, command: M) -> Result /// written to by its [`CommandToGsp::init_variable_payload`] method. /// /// Error codes returned by the command initializers are propagated as-is. - fn send_command(&mut self, bar: &Bar0, command: M) -> Result + fn send_command(&mut self, bar: Bar0<'_>, command: M) -> Result where M: CommandToGsp, Error: From, diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 88fc3e791114..04f004856c60 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -41,7 +41,7 @@ pub(super) trait UnloadBundle: Send { fn run( &self, dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, gsp_falcon: &Falcon, sec2_falcon: &Falcon, ) -> Result; @@ -58,7 +58,7 @@ fn boot<'a>( &self, gsp: &'a Gsp, dev: &'a device::Device, - bar: &'a Bar0, + bar: Bar0<'a>, chipset: Chipset, fb_layout: &FbLayout, wpr_meta: &Coherent, @@ -74,7 +74,7 @@ fn post_boot( &self, _gsp: &Gsp, _dev: &device::Device, - _bar: &Bar0, + _bar: Bar0<'_>, _gsp_fw: &GspFirmware, _gsp_falcon: &Falcon, _sec2_falcon: &Falcon, diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 9494258b9fc4..98f5ce197d13 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -46,7 +46,7 @@ struct GspMbox { impl GspMbox { /// Reads both mailboxes from the GSP falcon. - fn read(gsp_falcon: &Falcon, bar: &Bar0) -> Self { + fn read(gsp_falcon: &Falcon, bar: Bar0<'_>) -> Self { Self { mbox0: gsp_falcon.read_mailbox0(bar), mbox1: gsp_falcon.read_mailbox1(bar), @@ -65,7 +65,7 @@ fn combined_addr(&self) -> u64 { fn lockdown_released_or_error( &self, gsp_falcon: &Falcon, - bar: &Bar0, + bar: Bar0<'_>, fmc_boot_params_addr: u64, ) -> bool { // GSP-FMC normally clears the boot parameters address from the mailboxes early during @@ -82,7 +82,7 @@ fn lockdown_released_or_error( /// Waits for GSP lockdown to be released after FSP Chain of Trust. fn wait_for_gsp_lockdown_release( dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, gsp_falcon: &Falcon, fmc_boot_params_addr: u64, ) -> Result { @@ -126,7 +126,7 @@ impl UnloadBundle for FspUnloadBundle { fn run( &self, dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, gsp_falcon: &Falcon, _sec2_falcon: &Falcon, ) -> Result { @@ -153,7 +153,7 @@ fn boot<'a>( &self, gsp: &'a Gsp, dev: &'a device::Device, - bar: &'a Bar0, + bar: Bar0<'a>, chipset: Chipset, fb_layout: &FbLayout, wpr_meta: &Coherent, diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index a033bc892066..2f6301af7113 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -61,7 +61,7 @@ impl FwsecUnloadFirmware { /// Loads the FWSEC SB firmware, as well as its bootloader if `chipset` requires it. fn new( dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, chipset: Chipset, bios: &Vbios, gsp_falcon: &Falcon, @@ -79,7 +79,7 @@ fn new( fn run( &self, dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, gsp_falcon: &Falcon, ) -> Result { match self { @@ -100,7 +100,7 @@ impl Sec2UnloadBundle { /// Load and prepare the resources required to properly reset the GSP after it has been stopped. fn build( dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, chipset: Chipset, bios: &Vbios, gsp_falcon: &Falcon, @@ -129,7 +129,7 @@ impl UnloadBundle for Sec2UnloadBundle { fn run( &self, dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, gsp_falcon: &Falcon, sec2_falcon: &Falcon, ) -> Result { @@ -172,7 +172,7 @@ fn run_fwsec_frts( dev: &device::Device, chipset: Chipset, falcon: &Falcon, - bar: &Bar0, + bar: Bar0<'_>, bios: &Vbios, fb_layout: &FbLayout, ) -> Result { @@ -259,7 +259,7 @@ fn boot<'a>( &self, gsp: &'a Gsp, dev: &'a device::Device, - bar: &'a Bar0, + bar: Bar0<'a>, chipset: Chipset, fb_layout: &FbLayout, wpr_meta: &Coherent, @@ -325,7 +325,7 @@ fn post_boot( &self, gsp: &Gsp, dev: &device::Device, - bar: &Bar0, + bar: Bar0<'_>, gsp_fw: &GspFirmware, gsp_falcon: &Falcon, sec2_falcon: &Falcon, diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index b3015483ed17..e0850d21adca 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -131,7 +131,7 @@ pub(crate) struct GspSequencer<'a> { /// Sequencer information with command data. seq_info: GspSequence, /// `Bar0` for register access. - bar: &'a Bar0, + bar: Bar0<'a>, /// SEC2 falcon for core operations. sec2_falcon: &'a Falcon, /// GSP falcon for core operations. @@ -351,7 +351,7 @@ pub(crate) struct GspSequencerParams<'a> { /// Device for logging. pub(crate) dev: &'a device::Device, /// BAR0 for register access. - pub(crate) bar: &'a Bar0, + pub(crate) bar: Bar0<'a>, } impl<'a> GspSequencer<'a> { diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index cc24ab10b922..0f49c1ab83ad 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -518,7 +518,7 @@ pub(crate) fn with_falcon_mem(self, mem: FalconMem) -> Self { impl NV_PFALCON_FALCON_ENGINE { /// Resets the falcon - pub(crate) fn reset_engine(bar: &Bar0) { + pub(crate) fn reset_engine(bar: Bar0<'_>) { bar.update(Self::of::(), |r| r.with_reset(true)); // TIMEOUT: falcon engine should not take more than 10us to reset. diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index c0bc1008ed75..fd168c5da78c 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -58,7 +58,7 @@ fn try_from(code: u8) -> Result { /// Vbios Reader for constructing the VBIOS data. struct VbiosIterator<'a> { dev: &'a device::Device, - bar0: &'a Bar0, + bar0: Bar0<'a>, /// VBIOS data vector: As BIOS images are scanned, they are added to this vector for reference /// or copying into other data structures. It is the entire scanned contents of the VBIOS which /// progressively extends. It is used so that we do not re-read any contents that are already @@ -90,7 +90,7 @@ impl<'a> VbiosIterator<'a> { /// so that PROM reads transparently skip the header. On GA100, for some reason, the IFR offset /// is not applied to PROM reads. Therefore, the search for the PCI expansion must skip the IFR /// header, if found. - fn rom_offset(dev: &device::Device, bar0: &Bar0) -> Result { + fn rom_offset(dev: &device::Device, bar0: Bar0<'_>) -> Result { // IFR Header in VBIOS. register! { NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 { @@ -158,7 +158,7 @@ fn rom_offset(dev: &device::Device, bar0: &Bar0) -> Result { } } - fn new(dev: &'a device::Device, bar0: &'a Bar0) -> Result { + fn new(dev: &'a device::Device, bar0: Bar0<'a>) -> Result { Ok(Self { dev, bar0, @@ -297,7 +297,7 @@ impl Vbios { /// Probe for VBIOS extraction. /// /// Once the VBIOS object is built, `bar0` is not read for [`Vbios`] purposes anymore. - pub(crate) fn new(dev: &device::Device, bar0: &Bar0) -> Result { + pub(crate) fn new(dev: &device::Device, bar0: Bar0<'_>) -> Result { // Images to extract from iteration let mut pci_at_image: Option = None; let mut fwsec_section: Option> = None;