From b2bce0e551e89af37cfcb1b1158d80f369eeea3f Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Mon, 10 Nov 2025 22:14:58 +0000 Subject: [PATCH 001/187] drm/xe/pat: Add helper to query compression enable status Add xe_pat_index_get_comp_en() helper function to check whether compression is enabled for a given PAT index by extracting the XE2_COMP_EN bit from the PAT table entry. There are no current users, however there are multiple in-flight series which will all use this helper. CC: Nitin Gote CC: Sanjay Yadav CC: Matt Roper Suggested-by: Matthew Auld Signed-off-by: Xin Wang Reviewed-by: Matt Roper Reviewed-by: Nitin Gote Reviewed-by: Matthew Auld Reviewed-by: Sanjay Yadav Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20251110221458.1864507-2-x.wang@intel.com --- drivers/gpu/drm/xe/xe_pat.c | 6 ++++++ drivers/gpu/drm/xe/xe_pat.h | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c index 68171cceea18..1b4d5d3def0f 100644 --- a/drivers/gpu/drm/xe/xe_pat.c +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -196,6 +196,12 @@ u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) return xe->pat.table[pat_index].coh_mode; } +bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index) +{ + WARN_ON(pat_index >= xe->pat.n_entries); + return !!(xe->pat.table[pat_index].value & XE2_COMP_EN); +} + static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], int n_entries) { diff --git a/drivers/gpu/drm/xe/xe_pat.h b/drivers/gpu/drm/xe/xe_pat.h index 05dae03a5f54..b8559120989e 100644 --- a/drivers/gpu/drm/xe/xe_pat.h +++ b/drivers/gpu/drm/xe/xe_pat.h @@ -58,4 +58,14 @@ int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p); */ u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index); +/** + * xe_pat_index_get_comp_en - Extract the compression enable flag for + * the given pat_index. + * @xe: xe device + * @pat_index: The pat_index to query + * + * Return: true if compression is enabled for this pat_index, false otherwise. + */ +bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index); + #endif From 8f565bdd14eec5611cc041dba4650e42ccdf71d9 Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Wed, 12 Nov 2025 18:10:06 +0000 Subject: [PATCH 002/187] drm/xe: Prevent BIT() overflow when handling invalid prefetch region If user provides a large value (such as 0x80) for parameter prefetch_mem_region_instance in vm_bind ioctl, it will cause BIT(prefetch_region) overflow as below: " ------------[ cut here ]------------ UBSAN: shift-out-of-bounds in drivers/gpu/drm/xe/xe_vm.c:3414:7 shift exponent 128 is too large for 64-bit type 'long unsigned int' CPU: 8 UID: 0 PID: 53120 Comm: xe_exec_system_ Tainted: G W 6.18.0-rc1-lgci-xe-kernel+ #200 PREEMPT(voluntary) Tainted: [W]=WARN Hardware name: ASUS System Product Name/PRIME Z790-P WIFI, BIOS 0812 02/24/2023 Call Trace: dump_stack_lvl+0xa0/0xc0 dump_stack+0x10/0x20 ubsan_epilogue+0x9/0x40 __ubsan_handle_shift_out_of_bounds+0x10e/0x170 ? mutex_unlock+0x12/0x20 xe_vm_bind_ioctl.cold+0x20/0x3c [xe] ... " Fix it by validating prefetch_region before the BIT() usage. v2: Add Closes and Cc stable kernels. (Matt) Reported-by: Koen Koning Reported-by: Peter Senna Tschudin Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/6478 Cc: # v6.8+ Reviewed-by: Matthew Auld Signed-off-by: Shuicheng Lin Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20251112181005.2120521-2-shuicheng.lin@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 8fb5cc6a69ec..7cac646bdf1c 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -3411,8 +3411,10 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, op == DRM_XE_VM_BIND_OP_PREFETCH) || XE_IOCTL_DBG(xe, prefetch_region && op != DRM_XE_VM_BIND_OP_PREFETCH) || - XE_IOCTL_DBG(xe, (prefetch_region != DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC && - !(BIT(prefetch_region) & xe->info.mem_region_mask))) || + XE_IOCTL_DBG(xe, (prefetch_region != DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC && + /* Guard against undefined shift in BIT(prefetch_region) */ + (prefetch_region >= (sizeof(xe->info.mem_region_mask) * 8) || + !(BIT(prefetch_region) & xe->info.mem_region_mask)))) || XE_IOCTL_DBG(xe, obj && op == DRM_XE_VM_BIND_OP_UNMAP) || XE_IOCTL_DBG(xe, (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET) && From 562b0f254d8b1515a1c8d2a650f940d4f719300e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Winiarski?= Date: Fri, 14 Nov 2025 14:40:30 +0100 Subject: [PATCH 003/187] drm/xe/pf: Fix kernel-doc warning in migration_save_consume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel-doc for xe_sriov_pf_migration_save_consume() contained multiple "Return:" sections, causing a warning. Fix it by removing the extra line. Fixes: 67df4a5cbc583 ("drm/xe/pf: Add data structures and handlers for migration rings") Signed-off-by: Michał Winiarski Reviewed-by: Michal Wajdeczko Link: https://patch.msgid.link/20251114134030.1795947-1-michal.winiarski@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_sriov_pf_migration.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c index 21b06ce6830a..de06cc690fc8 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c @@ -148,8 +148,6 @@ pf_migration_consume(struct xe_device *xe, unsigned int vfid) * Return: Pointer to &xe_sriov_packet on success, * NULL if ring is empty and no more migration data is expected, * ERR_PTR value in case of error. - * - * Return: 0 on success or a negative error code on failure. */ struct xe_sriov_packet * xe_sriov_pf_migration_save_consume(struct xe_device *xe, unsigned int vfid) From 9be4f0f687048ba77428ceca11994676736507b7 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Thu, 13 Nov 2025 15:40:39 -0800 Subject: [PATCH 004/187] drm/xe/kunit: Fix forcewake assertion in mocs test The MOCS kunit test calls KUNIT_ASSERT_TRUE_MSG() with a condition of 'true;' this prevents the assertion from ever failing. Replace KUNIT_ASSERT_TRUE_MSG with KUNIT_FAIL_AND_ABORT to get the intended failure behavior in cases where forcewake was not acquired successfully. Fixes: 51c0ee84e4dc ("drm/xe/tests/mocs: Hold XE_FORCEWAKE_ALL for LNCF regs") Cc: Tejas Upadhyay Cc: Gustavo Sousa Reviewed-by: Lucas De Marchi Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251113234038.2256106-2-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/tests/xe_mocs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/tests/xe_mocs.c b/drivers/gpu/drm/xe/tests/xe_mocs.c index 0e502feaca81..6bb278167aaf 100644 --- a/drivers/gpu/drm/xe/tests/xe_mocs.c +++ b/drivers/gpu/drm/xe/tests/xe_mocs.c @@ -49,7 +49,7 @@ static void read_l3cc_table(struct xe_gt *gt, fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) { xe_force_wake_put(gt_to_fw(gt), fw_ref); - KUNIT_ASSERT_TRUE_MSG(test, true, "Forcewake Failed.\n"); + KUNIT_FAIL_AND_ABORT(test, "Forcewake Failed.\n"); } for (i = 0; i < info->num_mocs_regs; i++) { From b1aa02acd03bfef3ed39c511d33c4a4303d2f9b1 Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Mon, 10 Nov 2025 23:26:58 +0000 Subject: [PATCH 005/187] drm/xe: Remove duplicate DRM_EXEC selection from Kconfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are 2 identical "select DRM_EXEC" lines for DRM_XE. Remove one to clean up the configuration. Fixes: d490ecf57790 ("drm/xe: Rework xe_exec and the VM rebind worker to use the drm_exec helper") Cc: Thomas Hellström Cc: Lucas De Marchi Signed-off-by: Shuicheng Lin Reviewed-by: Nitin Gote Reviewed-by: Lucas De Marchi Link: https://patch.msgid.link/20251110232657.1807998-2-shuicheng.lin@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/xe/Kconfig b/drivers/gpu/drm/xe/Kconfig index 7219f6b884b6..4b288eb3f5b0 100644 --- a/drivers/gpu/drm/xe/Kconfig +++ b/drivers/gpu/drm/xe/Kconfig @@ -13,7 +13,6 @@ config DRM_XE select TMPFS select DRM_BUDDY select DRM_CLIENT_SELECTION - select DRM_EXEC select DRM_KMS_HELPER select DRM_KUNIT_TEST_HELPERS if DRM_XE_KUNIT_TEST != n select DRM_PANEL From dab751b4240f0f0eadea81f93ff0b439379bc6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Winiarski?= Date: Fri, 14 Nov 2025 11:07:13 +0100 Subject: [PATCH 006/187] drm/xe/pf: Drop the VF VRAM BO reference on successful restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reference is only dropped on error. Fix it by adding the missing xe_bo_put(). Fixes: 49cf1b9b609fe ("drm/xe/pf: Handle VRAM migration data as part of PF control") Reported-by: Adam Miszczak Reviewed-by: Matthew Auld Link: https://patch.msgid.link/20251114100713.1776073-1-michal.winiarski@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c index 35a12d48dcc1..7540a3854dc7 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c @@ -661,6 +661,8 @@ static int pf_restore_vf_vram_mig_data(struct xe_gt *gt, unsigned int vfid, goto err; } + xe_bo_put(vram); + return 0; err: xe_bo_put(vram); From 78ff838a8ab78b3cd438e382ff5204b93db3237e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Winiarski?= Date: Fri, 14 Nov 2025 13:23:39 +0100 Subject: [PATCH 007/187] drm/xe/pf: Check for fence error on VRAM save/restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code incorrectly assumes that the VRAM save/restore fence is valid. Fix it by checking for error. Fixes: 49cf1b9b609fe ("drm/xe/pf: Handle VRAM migration data as part of PF control") Suggested-by: Matthew Auld Reviewed-by: Matthew Auld Reviewed-by: Michal Wajdeczko Link: https://patch.msgid.link/20251114122339.1791026-1-michal.winiarski@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c index 7540a3854dc7..d5d918ddce4f 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c @@ -571,6 +571,10 @@ static int pf_save_vram_chunk(struct xe_gt *gt, unsigned int vfid, fence = __pf_save_restore_vram(gt, vfid, src_vram, src_vram_offset, data->bo, 0, size, true); + if (IS_ERR(fence)) { + ret = PTR_ERR(fence); + goto fail; + } ret = dma_fence_wait_timeout(fence, false, PF_VRAM_SAVE_RESTORE_TIMEOUT); dma_fence_put(fence); @@ -654,6 +658,11 @@ static int pf_restore_vf_vram_mig_data(struct xe_gt *gt, unsigned int vfid, fence = __pf_save_restore_vram(gt, vfid, vram, data->hdr.offset, data->bo, 0, data->hdr.size, false); + if (IS_ERR(fence)) { + ret = PTR_ERR(fence); + goto err; + } + ret = dma_fence_wait_timeout(fence, false, PF_VRAM_SAVE_RESTORE_TIMEOUT); dma_fence_put(fence); if (!ret) { From c34a14bce7090862ebe5a64abe8d85df75e62737 Mon Sep 17 00:00:00 2001 From: Venkata Ramana Nayana Date: Fri, 7 Nov 2025 14:01:41 +0530 Subject: [PATCH 008/187] drm/xe/irq: Handle msix vector0 interrupt Current gu2host handler registered as MSI-X vector 0 and as per bspec for a msix vector 0 interrupt, the driver must check the legacy registers 190008(TILE_INT_REG), 190060h (GT INTR Identity Reg 0) and other registers mentioned in "Interrupt Service Routine Pseudocode" otherwise it will block the next interrupts. To overcome this issue replacing guc2host handler with legacy xe_irq_handler. Fixes: da889070be7b2 ("drm/xe/irq: Separate MSI and MSI-X flows") Bspec: 62357 Signed-off-by: Venkata Ramana Nayana Reviewed-by: Balasubramani Vivekanandan Link: https://patch.msgid.link/20251107083141.2080189-1-venkata.ramana.nayana@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_irq.c | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c index e5ed0242f7b1..024e13e606ec 100644 --- a/drivers/gpu/drm/xe/xe_irq.c +++ b/drivers/gpu/drm/xe/xe_irq.c @@ -897,22 +897,6 @@ static int xe_irq_msix_init(struct xe_device *xe) return 0; } -static irqreturn_t guc2host_irq_handler(int irq, void *arg) -{ - struct xe_device *xe = arg; - struct xe_tile *tile; - u8 id; - - if (!atomic_read(&xe->irq.enabled)) - return IRQ_NONE; - - for_each_tile(tile, xe, id) - xe_guc_irq_handler(&tile->primary_gt->uc.guc, - GUC_INTR_GUC2HOST); - - return IRQ_HANDLED; -} - static irqreturn_t xe_irq_msix_default_hwe_handler(int irq, void *arg) { unsigned int tile_id, gt_id; @@ -1029,7 +1013,7 @@ int xe_irq_msix_request_irqs(struct xe_device *xe) u16 msix; msix = GUC2HOST_MSIX; - err = xe_irq_msix_request_irq(xe, guc2host_irq_handler, xe, + err = xe_irq_msix_request_irq(xe, xe_irq_handler(xe), xe, DRIVER_NAME "-guc2host", false, &msix); if (err) return err; From 1f2cf5295cdba71818288c9e495b4ef5097565ed Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Tue, 18 Nov 2025 12:07:44 +0000 Subject: [PATCH 009/187] drm/xe/sa: Shadow buffer support in the sub-allocator pool The existing sub-allocator is limited to managing a single buffer object. This enhancement introduces shadow buffer functionality to support scenarios requiring dual buffer management. The changes include added shadow buffer object creation capability, Management for both primary and shadow buffers, and appropriate locking mechanisms for thread-safe operations. This enables more flexible buffer allocation strategies in scenarios where shadow buffering is required. Signed-off-by: Satyanarayana K V P Suggested-by: Matthew Brost Cc: Michal Wajdeczko Cc: Matthew Auld Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251118120745.3460172-2-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_guc_buf.c | 2 +- drivers/gpu/drm/xe/xe_sa.c | 67 +++++++++++++++++++++++++++- drivers/gpu/drm/xe/xe_sa.h | 20 ++++++++- drivers/gpu/drm/xe/xe_sa_types.h | 3 ++ drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 3 ++ 5 files changed, 91 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_buf.c b/drivers/gpu/drm/xe/xe_guc_buf.c index 3ce442500130..c36fc31e0438 100644 --- a/drivers/gpu/drm/xe/xe_guc_buf.c +++ b/drivers/gpu/drm/xe/xe_guc_buf.c @@ -30,7 +30,7 @@ static int guc_buf_cache_init(struct xe_guc_buf_cache *cache, u32 size) struct xe_gt *gt = cache_to_gt(cache); struct xe_sa_manager *sam; - sam = __xe_sa_bo_manager_init(gt_to_tile(gt), size, 0, sizeof(u32)); + sam = __xe_sa_bo_manager_init(gt_to_tile(gt), size, 0, sizeof(u32), 0); if (IS_ERR(sam)) return PTR_ERR(sam); cache->sam = sam; diff --git a/drivers/gpu/drm/xe/xe_sa.c b/drivers/gpu/drm/xe/xe_sa.c index 63a5263dcf1b..a87c1436c7c1 100644 --- a/drivers/gpu/drm/xe/xe_sa.c +++ b/drivers/gpu/drm/xe/xe_sa.c @@ -29,6 +29,7 @@ static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg) kvfree(sa_manager->cpu_ptr); sa_manager->bo = NULL; + sa_manager->shadow = NULL; } /** @@ -37,12 +38,14 @@ static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg) * @size: number of bytes to allocate * @guard: number of bytes to exclude from suballocations * @align: alignment for each suballocated chunk + * @flags: flags for suballocator * * Prepares the suballocation manager for suballocations. * * Return: a pointer to the &xe_sa_manager or an ERR_PTR on failure. */ -struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 guard, u32 align) +struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, + u32 guard, u32 align, u32 flags) { struct xe_device *xe = tile_to_xe(tile); struct xe_sa_manager *sa_manager; @@ -79,6 +82,26 @@ struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u3 memset(sa_manager->cpu_ptr, 0, bo->ttm.base.size); } + if (flags & XE_SA_BO_MANAGER_FLAG_SHADOW) { + struct xe_bo *shadow; + + ret = drmm_mutex_init(&xe->drm, &sa_manager->swap_guard); + if (ret) + return ERR_PTR(ret); + + shadow = xe_managed_bo_create_pin_map(xe, tile, size, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(shadow)) { + drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n", + size / SZ_1K, shadow); + return ERR_CAST(shadow); + } + sa_manager->shadow = shadow; + } + drm_suballoc_manager_init(&sa_manager->base, managed_size, align); ret = drmm_add_action_or_reset(&xe->drm, xe_sa_bo_manager_fini, sa_manager); @@ -88,6 +111,48 @@ struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u3 return sa_manager; } +/** + * xe_sa_bo_swap_shadow() - Swap the SA BO with shadow BO. + * @sa_manager: the XE sub allocator manager + * + * Swaps the sub-allocator primary buffer object with shadow buffer object. + * + * Return: None. + */ +void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager) +{ + struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); + + xe_assert(xe, sa_manager->shadow); + lockdep_assert_held(&sa_manager->swap_guard); + + swap(sa_manager->bo, sa_manager->shadow); + if (!sa_manager->bo->vmap.is_iomem) + sa_manager->cpu_ptr = sa_manager->bo->vmap.vaddr; +} + +/** + * xe_sa_bo_sync_shadow() - Sync the SA Shadow BO with primary BO. + * @sa_bo: the sub-allocator buffer object. + * + * Synchronize sub-allocator shadow buffer object with primary buffer object. + * + * Return: None. + */ +void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo) +{ + struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager); + struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); + + xe_assert(xe, sa_manager->shadow); + lockdep_assert_held(&sa_manager->swap_guard); + + xe_map_memcpy_to(xe, &sa_manager->shadow->vmap, + drm_suballoc_soffset(sa_bo), + xe_sa_bo_cpu_addr(sa_bo), + drm_suballoc_size(sa_bo)); +} + /** * __xe_sa_bo_new() - Make a suballocation but use custom gfp flags. * @sa_manager: the &xe_sa_manager diff --git a/drivers/gpu/drm/xe/xe_sa.h b/drivers/gpu/drm/xe/xe_sa.h index 1be744350836..05e9a4e00e78 100644 --- a/drivers/gpu/drm/xe/xe_sa.h +++ b/drivers/gpu/drm/xe/xe_sa.h @@ -14,12 +14,14 @@ struct dma_fence; struct xe_tile; -struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 guard, u32 align); +#define XE_SA_BO_MANAGER_FLAG_SHADOW BIT(0) +struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, + u32 guard, u32 align, u32 flags); struct drm_suballoc *__xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size, gfp_t gfp); static inline struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 align) { - return __xe_sa_bo_manager_init(tile, size, SZ_4K, align); + return __xe_sa_bo_manager_init(tile, size, SZ_4K, align, 0); } /** @@ -69,4 +71,18 @@ static inline void *xe_sa_bo_cpu_addr(struct drm_suballoc *sa) drm_suballoc_soffset(sa); } +void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager); +void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo); + +/** + * xe_sa_bo_swap_guard() - Retrieve the SA BO swap guard within sub-allocator. + * @sa_manager: the &xe_sa_manager + * + * Return: Sub alloctor swap guard mutex. + */ +static inline struct mutex *xe_sa_bo_swap_guard(struct xe_sa_manager *sa_manager) +{ + return &sa_manager->swap_guard; +} + #endif diff --git a/drivers/gpu/drm/xe/xe_sa_types.h b/drivers/gpu/drm/xe/xe_sa_types.h index cb7238799dcb..1085c9c37d6b 100644 --- a/drivers/gpu/drm/xe/xe_sa_types.h +++ b/drivers/gpu/drm/xe/xe_sa_types.h @@ -12,6 +12,9 @@ struct xe_bo; struct xe_sa_manager { struct drm_suballoc_manager base; struct xe_bo *bo; + struct xe_bo *shadow; + /** @swap_guard: Timeline guard updating @bo and @shadow */ + struct mutex swap_guard; void *cpu_ptr; bool is_iomem; }; diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index 797a4b866226..9959d619addc 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -162,9 +162,12 @@ static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx) offset = 0; xe_map_memset(xe, &sa_manager->bo->vmap, offset, MI_NOOP, bb_pool_size); + xe_map_memset(xe, &sa_manager->shadow->vmap, offset, MI_NOOP, + bb_pool_size); offset = bb_pool_size - sizeof(u32); xe_map_wr(xe, &sa_manager->bo->vmap, offset, u32, MI_BATCH_BUFFER_END); + xe_map_wr(xe, &sa_manager->shadow->vmap, offset, u32, MI_BATCH_BUFFER_END); ctx->mem.ccs_bb_pool = sa_manager; From fa18290bf0723b02bfa8d30d2e14722f0d096c2c Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Tue, 18 Nov 2025 12:07:45 +0000 Subject: [PATCH 010/187] drm/xe/vf: Shadow buffer management for CCS read/write operations CCS copy command consist of 5-dword sequence. If vCPU halts during save/restore operations while these sequences are being programmed, incomplete writes can cause page faults during IGPU CCS metadata saving. Use shadow buffer management to prevent partial write issues during CCS operations. Signed-off-by: Satyanarayana K V P Suggested-by: Matthew Brost Cc: Michal Wajdeczko Cc: Matthew Auld Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251118120745.3460172-3-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_migrate.c | 57 ++++++++++++++++++++++++++-- drivers/gpu/drm/xe/xe_migrate.h | 3 ++ drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 19 ++++++++-- drivers/gpu/drm/xe/xe_sriov_vf_ccs.h | 1 + 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 2184af413b91..f3b66b55acfb 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -34,6 +34,7 @@ #include "xe_res_cursor.h" #include "xe_sa.h" #include "xe_sched_job.h" +#include "xe_sriov_vf_ccs.h" #include "xe_sync.h" #include "xe_trace_bo.h" #include "xe_validation.h" @@ -1103,12 +1104,16 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, u32 batch_size, batch_size_allocated; struct xe_device *xe = gt_to_xe(gt); struct xe_res_cursor src_it, ccs_it; + struct xe_sriov_vf_ccs_ctx *ctx; + struct xe_sa_manager *bb_pool; u64 size = xe_bo_size(src_bo); struct xe_bb *bb = NULL; u64 src_L0, src_L0_ofs; u32 src_L0_pt; int err; + ctx = &xe->sriov.vf.ccs.contexts[read_write]; + xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), @@ -1141,11 +1146,15 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, size -= src_L0; } + bb_pool = ctx->mem.ccs_bb_pool; + guard(mutex) (xe_sa_bo_swap_guard(bb_pool)); + xe_sa_bo_swap_shadow(bb_pool); + bb = xe_bb_ccs_new(gt, batch_size, read_write); if (IS_ERR(bb)) { drm_err(&xe->drm, "BB allocation failed.\n"); err = PTR_ERR(bb); - goto err_ret; + return err; } batch_size_allocated = batch_size; @@ -1194,10 +1203,52 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, xe_assert(xe, (batch_size_allocated == bb->len)); src_bo->bb_ccs[read_write] = bb; + xe_sriov_vf_ccs_rw_update_bb_addr(ctx); + xe_sa_bo_sync_shadow(bb->bo); return 0; +} -err_ret: - return err; +/** + * xe_migrate_ccs_rw_copy_clear() - Clear the CCS read/write batch buffer + * content. + * @src_bo: The buffer object @src is currently bound to. + * @read_write : Creates BB commands for CCS read/write. + * + * Directly clearing the BB lacks atomicity and can lead to undefined + * behavior if the vCPU is halted mid-operation during the clearing + * process. To avoid this issue, we use a shadow buffer object approach. + * + * First swap the SA BO address with the shadow BO, perform the clearing + * operation on the BB, update the shadow BO in the ring buffer, then + * sync the shadow and the actual buffer to maintain consistency. + * + * Returns: None. + */ +void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo, + enum xe_sriov_vf_ccs_rw_ctxs read_write) +{ + struct xe_bb *bb = src_bo->bb_ccs[read_write]; + struct xe_device *xe = xe_bo_device(src_bo); + struct xe_sriov_vf_ccs_ctx *ctx; + struct xe_sa_manager *bb_pool; + u32 *cs; + + xe_assert(xe, IS_SRIOV_VF(xe)); + + ctx = &xe->sriov.vf.ccs.contexts[read_write]; + bb_pool = ctx->mem.ccs_bb_pool; + + guard(mutex) (xe_sa_bo_swap_guard(bb_pool)); + xe_sa_bo_swap_shadow(bb_pool); + + cs = xe_sa_bo_cpu_addr(bb->bo); + memset(cs, MI_NOOP, bb->len * sizeof(u32)); + xe_sriov_vf_ccs_rw_update_bb_addr(ctx); + + xe_sa_bo_sync_shadow(bb->bo); + + xe_bb_free(bb, NULL); + src_bo->bb_ccs[read_write] = NULL; } /** diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index 260e298e5dd7..464c05dde1ba 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -134,6 +134,9 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, struct xe_bo *src_bo, enum xe_sriov_vf_ccs_rw_ctxs read_write); +void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo, + enum xe_sriov_vf_ccs_rw_ctxs read_write); + struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate); struct xe_exec_queue *xe_migrate_exec_queue(struct xe_migrate *migrate); struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_offset, diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index 9959d619addc..33f4238604e1 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -150,7 +150,8 @@ static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx) xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n", ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M); - sa_manager = xe_sa_bo_manager_init(tile, bb_pool_size, SZ_16); + sa_manager = __xe_sa_bo_manager_init(tile, bb_pool_size, SZ_4K, SZ_16, + XE_SA_BO_MANAGER_FLAG_SHADOW); if (IS_ERR(sa_manager)) { xe_sriov_err(xe, "Suballocator init failed with error: %pe\n", @@ -384,6 +385,18 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe) return err; } +#define XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET (2 * sizeof(u32)) +void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx) +{ + u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool); + struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); + struct xe_device *xe = gt_to_xe(ctx->mig_q->gt); + + xe_device_wmb(xe); + xe_map_wr(xe, &lrc->bo->vmap, XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET, u32, addr); + xe_device_wmb(xe); +} + /** * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO. * @bo: the &buffer object to which batch buffer commands will be added. @@ -444,9 +457,7 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo) if (!bb) continue; - memset(bb->cs, MI_NOOP, bb->len * sizeof(u32)); - xe_bb_free(bb, NULL); - bo->bb_ccs[ctx_id] = NULL; + xe_migrate_ccs_rw_copy_clear(bo, ctx_id); } return 0; } diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h index f8ca6efce9ee..00e58b36c510 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h @@ -20,6 +20,7 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo); int xe_sriov_vf_ccs_register_context(struct xe_device *xe); void xe_sriov_vf_ccs_rebase(struct xe_device *xe); void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p); +void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx); static inline bool xe_sriov_vf_ccs_ready(struct xe_device *xe) { From 0a4c2ddc711a9a0d3d9566379b217f2ae42d6152 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 12:26:05 -0800 Subject: [PATCH 011/187] drm/xe/vm: Use for_each_tlb_inval() to calculate invalidation fences ops_execute() calculates the size of a fence array based on XE_MAX_GT_PER_TILE, while the code that actually fills in the fence array uses a for_each_tlb_inval() iterator. This works out okay today since both approaches come up with the same number of invalidation fences (2: primary GT invalidation + media GT invalidation), but could be problematic in the future if there isn't a 1:1 relationship between TLBs needing invalidation and potential GTs on the tile. Adjust the allocation code to use the same for_each_tlb_inval() counting logic as the code that fills the array to future-proof the code. Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251118202604.3715782-2-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_vm.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 7cac646bdf1c..f9989a7a710c 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -3104,19 +3104,19 @@ static struct dma_fence *ops_execute(struct xe_vm *vm, struct dma_fence *fence = NULL; struct dma_fence **fences = NULL; struct dma_fence_array *cf = NULL; - int number_tiles = 0, current_fence = 0, n_fence = 0, err; + int number_tiles = 0, current_fence = 0, n_fence = 0, err, i; u8 id; number_tiles = vm_ops_setup_tile_args(vm, vops); if (number_tiles == 0) return ERR_PTR(-ENODATA); - if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT) { - for_each_tile(tile, vm->xe, id) - ++n_fence; - } else { - for_each_tile(tile, vm->xe, id) - n_fence += (1 + XE_MAX_GT_PER_TILE); + for_each_tile(tile, vm->xe, id) { + ++n_fence; + + if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)) + for_each_tlb_inval(i) + ++n_fence; } fences = kmalloc_array(n_fence, sizeof(*fences), GFP_KERNEL); @@ -3146,7 +3146,6 @@ static struct dma_fence *ops_execute(struct xe_vm *vm, for_each_tile(tile, vm->xe, id) { struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q; - int i; fence = NULL; if (!vops->pt_update_ops[id].num_ops) From 074edfbdfba25493324807f749fbc86bf0af3a2d Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:40 -0800 Subject: [PATCH 012/187] drm/xe/forcewake: Add scope-based cleanup for forcewake Since forcewake uses a reference counting get/put model, there are many places where we need to be careful to drop the forcewake reference when bailing out of a function early on an error path. Add scope-based cleanup options that can be used in place of explicit get/put to help prevent mistakes in this area. Examples: CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); Obtain forcewake on the XE_FW_GT domain and hold it until the end of the current block. The wakeref will be dropped automatically when the current scope is exited by any means (return, break, reaching the end of the block, etc.). xe_with_force_wake(fw_ref, gt_to_fw(ss->gt), XE_FORCEWAKE_ALL) { ... } Hold all forcewake domains for the following block. As with the CLASS usage, forcewake will be dropped automatically when the block is exited by any means. Use of these cleanup helpers should allow us to remove some ugly goto-based error handling and help avoid mistakes in functions with lots of early error exits. An 'xe_force_wake_release_only' class is also added for cases where a forcewake reference is passed in from another function and the current function is responsible for releasing it in every flow and error path. v2: - Create a separate constructor that just wraps xe_force_wake_get for use in the class. This eliminates the need to update the signature of xe_force_wake_get(). (Michal) v3: - Wrap xe_with_force_wake's 'done' marker in __UNIQUE_ID. (Gustavo) - Add a note to xe_force_wake_get()'s kerneldoc explaining that scope-based cleanup is preferred when possible. (Gustavo) - Add an xe_force_wake_release_only class. (Gustavo) v4: - Add NULL check on fw in release_only variant. (Gustavo) Cc: Michal Wajdeczko Cc: Gustavo Sousa Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-30-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_force_wake.c | 7 ++++++ drivers/gpu/drm/xe/xe_force_wake.h | 40 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_force_wake.c b/drivers/gpu/drm/xe/xe_force_wake.c index c59a9b330697..76e054f314ee 100644 --- a/drivers/gpu/drm/xe/xe_force_wake.c +++ b/drivers/gpu/drm/xe/xe_force_wake.c @@ -166,6 +166,13 @@ static int domain_sleep_wait(struct xe_gt *gt, * xe_force_wake_ref_has_domain() function. Caller must call * xe_force_wake_put() function to decrease incremented refcounts. * + * When possible, scope-based forcewake (through CLASS(xe_force_wake, ...) or + * xe_with_force_wake()) should be used instead of direct calls to this + * function. Direct usage of get/put should only be used when the function + * has goto-based flows that can interfere with scope-based cleanup, or when + * the lifetime of the forcewake reference does not match a specific scope + * (e.g., forcewake obtained in one function and released in a different one). + * * Return: opaque reference to woken domains or zero if none of requested * domains were awake. */ diff --git a/drivers/gpu/drm/xe/xe_force_wake.h b/drivers/gpu/drm/xe/xe_force_wake.h index 0e3e84bfa51c..1e2198f6a007 100644 --- a/drivers/gpu/drm/xe/xe_force_wake.h +++ b/drivers/gpu/drm/xe/xe_force_wake.h @@ -61,4 +61,44 @@ xe_force_wake_ref_has_domain(unsigned int fw_ref, enum xe_force_wake_domains dom return fw_ref & domain; } +struct xe_force_wake_ref { + struct xe_force_wake *fw; + unsigned int domains; +}; + +static struct xe_force_wake_ref +xe_force_wake_constructor(struct xe_force_wake *fw, unsigned int domains) +{ + struct xe_force_wake_ref fw_ref = { .fw = fw }; + + fw_ref.domains = xe_force_wake_get(fw, domains); + + return fw_ref; +} + +DEFINE_CLASS(xe_force_wake, struct xe_force_wake_ref, + xe_force_wake_put(_T.fw, _T.domains), + xe_force_wake_constructor(fw, domains), + struct xe_force_wake *fw, unsigned int domains); + +/* + * Scoped helper for the forcewake class, using the same trick as scoped_guard() + * to bind the lifetime to the next statement/block. + */ +#define __xe_with_force_wake(ref, fw, domains, done) \ + for (CLASS(xe_force_wake, ref)(fw, domains), *(done) = NULL; \ + !(done); (done) = (void *)1) + +#define xe_with_force_wake(ref, fw, domains) \ + __xe_with_force_wake(ref, fw, domains, __UNIQUE_ID(done)) + +/* + * Used when xe_force_wake_constructor() has already been called by another + * function and the current function is responsible for releasing the forcewake + * reference in all possible cases and error paths. + */ +DEFINE_CLASS(xe_force_wake_release_only, struct xe_force_wake_ref, + if (_T.fw) xe_force_wake_put(_T.fw, _T.domains), fw_ref, + struct xe_force_wake_ref fw_ref); + #endif From 59e7528dbfd52efbed05e0f11b2143217a12bc74 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:41 -0800 Subject: [PATCH 013/187] drm/xe/pm: Add scope-based cleanup helper for runtime PM Add a scope-based helpers for runtime PM that may be used to simplify cleanup logic and potentially avoid goto-based cleanup. For example, using guard(xe_pm_runtime)(xe); will get runtime PM and cause a corresponding put to occur automatically when the current scope is exited. 'xe_pm_runtime_noresume' can be used as a guard replacement for the corresponding 'noresume' variant. There's also an xe_pm_runtime_ioctl conditional guard that can be used as a replacement for xe_runtime_ioctl(): ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); if ((ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm)) < 0) /* failed */ In a few rare cases (such as gt_reset_worker()) we need to ensure that runtime PM is dropped when the function is exited by any means (including error paths), but the function does not need to acquire runtime PM because that has already been done earlier by a different function. For these special cases, an 'xe_pm_runtime_release_only' guard can be used to handle the release without doing an acquisition. These guards will be used in future patches to eliminate some of our goto-based cleanup. v2: - Specify success condition for xe_pm runtime_ioctl as _RET >= 0 so that positive values will be properly identified as success and trigger destructor cleanup properly. v3: - Add comments to the kerneldoc for the existing 'get' functions indicating that scope-based handling should be preferred where possible. (Gustavo) Cc: Gustavo Sousa Reviewed-by: Michal Wajdeczko Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-31-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_pm.c | 21 +++++++++++++++++++++ drivers/gpu/drm/xe/xe_pm.h | 17 +++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index 44924512830f..766922530265 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -726,6 +726,13 @@ static void xe_pm_runtime_lockdep_prime(void) /** * xe_pm_runtime_get - Get a runtime_pm reference and resume synchronously * @xe: xe device instance + * + * When possible, scope-based runtime PM (through guard(xe_pm_runtime)) is + * be preferred over direct usage of this function. Manual get/put handling + * should only be used when the function contains goto-based logic which + * can break scope-based handling, or when the lifetime of the runtime PM + * reference does not match a specific scope (e.g., runtime PM obtained in one + * function and released in a different one). */ void xe_pm_runtime_get(struct xe_device *xe) { @@ -758,6 +765,13 @@ void xe_pm_runtime_put(struct xe_device *xe) * xe_pm_runtime_get_ioctl - Get a runtime_pm reference before ioctl * @xe: xe device instance * + * When possible, scope-based runtime PM (through + * ACQUIRE(xe_pm_runtime_ioctl, ...)) is be preferred over direct usage of this + * function. Manual get/put handling should only be used when the function + * contains goto-based logic which can break scope-based handling, or when the + * lifetime of the runtime PM reference does not match a specific scope (e.g., + * runtime PM obtained in one function and released in a different one). + * * Returns: Any number greater than or equal to 0 for success, negative error * code otherwise. */ @@ -827,6 +841,13 @@ static bool xe_pm_suspending_or_resuming(struct xe_device *xe) * It will warn if not protected. * The reference should be put back after this function regardless, since it * will always bump the usage counter, regardless. + * + * When possible, scope-based runtime PM (through guard(xe_pm_runtime_noresume)) + * is be preferred over direct usage of this function. Manual get/put handling + * should only be used when the function contains goto-based logic which can + * break scope-based handling, or when the lifetime of the runtime PM reference + * does not match a specific scope (e.g., runtime PM obtained in one function + * and released in a different one). */ void xe_pm_runtime_get_noresume(struct xe_device *xe) { diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h index f7f89a18b6fc..6b27039e7b2d 100644 --- a/drivers/gpu/drm/xe/xe_pm.h +++ b/drivers/gpu/drm/xe/xe_pm.h @@ -6,6 +6,7 @@ #ifndef _XE_PM_H_ #define _XE_PM_H_ +#include #include #define DEFAULT_VRAM_THRESHOLD 300 /* in MB */ @@ -37,4 +38,20 @@ int xe_pm_block_on_suspend(struct xe_device *xe); void xe_pm_might_block_on_suspend(void); int xe_pm_module_init(void); +static inline void __xe_pm_runtime_noop(struct xe_device *xe) {} + +DEFINE_GUARD(xe_pm_runtime, struct xe_device *, + xe_pm_runtime_get(_T), xe_pm_runtime_put(_T)) +DEFINE_GUARD(xe_pm_runtime_noresume, struct xe_device *, + xe_pm_runtime_get_noresume(_T), xe_pm_runtime_put(_T)) +DEFINE_GUARD_COND(xe_pm_runtime, _ioctl, xe_pm_runtime_get_ioctl(_T), _RET >= 0) + +/* + * Used when a function needs to release runtime PM in all possible cases + * and error paths, but the wakeref was already acquired by a different + * function (i.e., get() has already happened so only a put() is needed). + */ +DEFINE_GUARD(xe_pm_runtime_release_only, struct xe_device *, + __xe_pm_runtime_noop(_T), xe_pm_runtime_put(_T)); + #endif From 83d2ea17d53f538724f0222f1cb5f60061efb06a Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:42 -0800 Subject: [PATCH 014/187] drm/xe/gt: Use scope-based cleanup Using scope-based cleanup for forcewake and runtime PM allows us to reduce or eliminate some of the goto-based error handling and simplify several functions. v2: - Drop changes to do_gt_restart(). This function still has goto-based logic, making scope-based cleanup unsafe for now. (Gustavo) Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-32-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_gt.c | 130 ++++++++++++------------------------- 1 file changed, 41 insertions(+), 89 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 6d479948bf21..514ed50e6d83 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -103,14 +103,13 @@ void xe_gt_sanitize(struct xe_gt *gt) static void xe_gt_enable_host_l2_vram(struct xe_gt *gt) { - unsigned int fw_ref; u32 reg; if (!XE_GT_WA(gt, 16023588340)) return; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return; if (xe_gt_is_main_type(gt)) { @@ -120,12 +119,10 @@ static void xe_gt_enable_host_l2_vram(struct xe_gt *gt) } xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0xF); - xe_force_wake_put(gt_to_fw(gt), fw_ref); } static void xe_gt_disable_host_l2_vram(struct xe_gt *gt) { - unsigned int fw_ref; u32 reg; if (!XE_GT_WA(gt, 16023588340)) @@ -134,15 +131,13 @@ static void xe_gt_disable_host_l2_vram(struct xe_gt *gt) if (xe_gt_is_media_type(gt)) return; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return; reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL); reg &= ~CG_DIS_CNTLBUS; xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } static void gt_reset_worker(struct work_struct *w); @@ -389,7 +384,6 @@ int xe_gt_record_default_lrcs(struct xe_gt *gt) int xe_gt_init_early(struct xe_gt *gt) { - unsigned int fw_ref; int err; if (IS_SRIOV_PF(gt_to_xe(gt))) { @@ -436,13 +430,12 @@ int xe_gt_init_early(struct xe_gt *gt) if (err) return err; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; xe_gt_mcr_init_early(gt); xe_pat_init(gt); - xe_force_wake_put(gt_to_fw(gt), fw_ref); return 0; } @@ -460,16 +453,15 @@ static void dump_pat_on_error(struct xe_gt *gt) static int gt_init_with_gt_forcewake(struct xe_gt *gt) { - unsigned int fw_ref; int err; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; err = xe_uc_init(>->uc); if (err) - goto err_force_wake; + return err; xe_gt_topology_init(gt); xe_gt_mcr_init(gt); @@ -478,7 +470,7 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt) if (xe_gt_is_main_type(gt)) { err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt); if (err) - goto err_force_wake; + return err; if (IS_SRIOV_PF(gt_to_xe(gt))) xe_lmtt_init(>_to_tile(gt)->sriov.pf.lmtt); } @@ -492,17 +484,17 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt) err = xe_hw_engines_init_early(gt); if (err) { dump_pat_on_error(gt); - goto err_force_wake; + return err; } err = xe_hw_engine_class_sysfs_init(gt); if (err) - goto err_force_wake; + return err; /* Initialize CCS mode sysfs after early initialization of HW engines */ err = xe_gt_ccs_mode_sysfs_init(gt); if (err) - goto err_force_wake; + return err; /* * Stash hardware-reported version. Since this register does not exist @@ -510,25 +502,16 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt) */ gt->info.gmdid = xe_mmio_read32(>->mmio, GMD_ID); - xe_force_wake_put(gt_to_fw(gt), fw_ref); return 0; - -err_force_wake: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - - return err; } static int gt_init_with_all_forcewake(struct xe_gt *gt) { - unsigned int fw_ref; int err; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) { - err = -ETIMEDOUT; - goto err_force_wake; - } + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) + return -ETIMEDOUT; xe_gt_mcr_set_implicit_defaults(gt); xe_wa_process_gt(gt); @@ -537,20 +520,20 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt) err = xe_gt_clock_init(gt); if (err) - goto err_force_wake; + return err; xe_mocs_init(gt); err = xe_execlist_init(gt); if (err) - goto err_force_wake; + return err; err = xe_hw_engines_init(gt); if (err) - goto err_force_wake; + return err; err = xe_uc_init_post_hwconfig(>->uc); if (err) - goto err_force_wake; + return err; if (xe_gt_is_main_type(gt)) { /* @@ -561,10 +544,8 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt) gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt), IS_DGFX(xe) ? SZ_1M : SZ_512K, 16); - if (IS_ERR(gt->usm.bb_pool)) { - err = PTR_ERR(gt->usm.bb_pool); - goto err_force_wake; - } + if (IS_ERR(gt->usm.bb_pool)) + return PTR_ERR(gt->usm.bb_pool); } } @@ -573,12 +554,12 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt) err = xe_migrate_init(tile->migrate); if (err) - goto err_force_wake; + return err; } err = xe_uc_load_hw(>->uc); if (err) - goto err_force_wake; + return err; /* Configure default CCS mode of 1 engine with all resources */ if (xe_gt_ccs_mode_enabled(gt)) { @@ -592,14 +573,7 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt) if (IS_SRIOV_PF(gt_to_xe(gt))) xe_gt_sriov_pf_init_hw(gt); - xe_force_wake_put(gt_to_fw(gt), fw_ref); - return 0; - -err_force_wake: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - - return err; } static void xe_gt_fini(void *arg) @@ -902,56 +876,42 @@ void xe_gt_reset_async(struct xe_gt *gt) void xe_gt_suspend_prepare(struct xe_gt *gt) { - unsigned int fw_ref; - - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); xe_uc_suspend_prepare(>->uc); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } int xe_gt_suspend(struct xe_gt *gt) { - unsigned int fw_ref; int err; xe_gt_dbg(gt, "suspending\n"); xe_gt_sanitize(gt); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) - goto err_msg; + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(-ETIMEDOUT)); + return -ETIMEDOUT; + } err = xe_uc_suspend(>->uc); - if (err) - goto err_force_wake; + if (err) { + xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err)); + return err; + } xe_gt_idle_disable_pg(gt); xe_gt_disable_host_l2_vram(gt); - xe_force_wake_put(gt_to_fw(gt), fw_ref); xe_gt_dbg(gt, "suspended\n"); return 0; - -err_msg: - err = -ETIMEDOUT; -err_force_wake: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err)); - - return err; } void xe_gt_shutdown(struct xe_gt *gt) { - unsigned int fw_ref; - - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); do_gt_reset(gt); - xe_force_wake_put(gt_to_fw(gt), fw_ref); } /** @@ -976,32 +936,24 @@ int xe_gt_sanitize_freq(struct xe_gt *gt) int xe_gt_resume(struct xe_gt *gt) { - unsigned int fw_ref; int err; xe_gt_dbg(gt, "resuming\n"); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) - goto err_msg; + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(-ETIMEDOUT)); + return -ETIMEDOUT; + } err = do_gt_restart(gt); if (err) - goto err_force_wake; + return err; xe_gt_idle_enable_pg(gt); - xe_force_wake_put(gt_to_fw(gt), fw_ref); xe_gt_dbg(gt, "resumed\n"); return 0; - -err_msg: - err = -ETIMEDOUT; -err_force_wake: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err)); - - return err; } struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, From 917714f2787db1f42cc41705e02cb381734ad663 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:43 -0800 Subject: [PATCH 015/187] drm/xe/gt_idle: Use scope-based cleanup Use scope-based cleanup for runtime PM and forcewake in the GT idle code. v2: - Use scoped_guard() over guard() in idle_status_show() and idle_residency_ms_show(). (Gustavo) - Eliminate unnecessary 'ret' local variable in name_show(). Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-33-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_gt_idle.c | 41 ++++++++++----------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c index bdc9d9877ec4..3ca7bd7c9bcd 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.c +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -103,7 +103,6 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt) struct xe_gt_idle *gtidle = >->gtidle; struct xe_mmio *mmio = >->mmio; u32 vcs_mask, vecs_mask; - unsigned int fw_ref; int i, j; if (IS_SRIOV_VF(xe)) @@ -135,7 +134,7 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt) } } - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); if (xe->info.skip_guc_pc) { /* * GuC sets the hysteresis value when GuC PC is enabled @@ -146,13 +145,11 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt) } xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable); - xe_force_wake_put(gt_to_fw(gt), fw_ref); } void xe_gt_idle_disable_pg(struct xe_gt *gt) { struct xe_gt_idle *gtidle = >->gtidle; - unsigned int fw_ref; if (IS_SRIOV_VF(gt_to_xe(gt))) return; @@ -160,9 +157,8 @@ void xe_gt_idle_disable_pg(struct xe_gt *gt) xe_device_assert_mem_access(gt_to_xe(gt)); gtidle->powergate_enable = 0; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); xe_mmio_write32(>->mmio, POWERGATE_ENABLE, gtidle->powergate_enable); - xe_force_wake_put(gt_to_fw(gt), fw_ref); } /** @@ -181,7 +177,6 @@ int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p) enum xe_gt_idle_state state; u32 pg_enabled, pg_status = 0; u32 vcs_mask, vecs_mask; - unsigned int fw_ref; int n; /* * Media Slices @@ -218,14 +213,12 @@ int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p) /* Do not wake the GT to read powergating status */ if (state != GT_IDLE_C6) { - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; pg_enabled = xe_mmio_read32(>->mmio, POWERGATE_ENABLE); pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) { @@ -263,13 +256,9 @@ static ssize_t name_show(struct kobject *kobj, struct device *dev = kobj_to_dev(kobj); struct xe_gt_idle *gtidle = dev_to_gtidle(dev); struct xe_guc_pc *pc = gtidle_to_pc(gtidle); - ssize_t ret; - xe_pm_runtime_get(pc_to_xe(pc)); - ret = sysfs_emit(buff, "%s\n", gtidle->name); - xe_pm_runtime_put(pc_to_xe(pc)); - - return ret; + guard(xe_pm_runtime)(pc_to_xe(pc)); + return sysfs_emit(buff, "%s\n", gtidle->name); } static struct kobj_attribute name_attr = __ATTR_RO(name); @@ -281,9 +270,8 @@ static ssize_t idle_status_show(struct kobject *kobj, struct xe_guc_pc *pc = gtidle_to_pc(gtidle); enum xe_gt_idle_state state; - xe_pm_runtime_get(pc_to_xe(pc)); - state = gtidle->idle_status(pc); - xe_pm_runtime_put(pc_to_xe(pc)); + scoped_guard(xe_pm_runtime, pc_to_xe(pc)) + state = gtidle->idle_status(pc); return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state)); } @@ -311,9 +299,8 @@ static ssize_t idle_residency_ms_show(struct kobject *kobj, struct xe_guc_pc *pc = gtidle_to_pc(gtidle); u64 residency; - xe_pm_runtime_get(pc_to_xe(pc)); - residency = xe_gt_idle_residency_msec(gtidle); - xe_pm_runtime_put(pc_to_xe(pc)); + scoped_guard(xe_pm_runtime, pc_to_xe(pc)) + residency = xe_gt_idle_residency_msec(gtidle); return sysfs_emit(buff, "%llu\n", residency); } @@ -396,21 +383,17 @@ void xe_gt_idle_enable_c6(struct xe_gt *gt) int xe_gt_idle_disable_c6(struct xe_gt *gt) { - unsigned int fw_ref; - xe_device_assert_mem_access(gt_to_xe(gt)); if (IS_SRIOV_VF(gt_to_xe(gt))) return 0; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; xe_mmio_write32(>->mmio, RC_CONTROL, 0); xe_mmio_write32(>->mmio, RC_STATE, 0); - xe_force_wake_put(gt_to_fw(gt), fw_ref); - return 0; } From 3947e482b5ebb95be49803fdd584a38f005f5042 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:44 -0800 Subject: [PATCH 016/187] drm/xe/guc: Use scope-based cleanup Use scope-based cleanup for forcewake and runtime PM. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-34-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_guc.c | 13 ++++--------- drivers/gpu/drm/xe/xe_guc_log.c | 10 ++++------ drivers/gpu/drm/xe/xe_guc_submit.c | 11 +++-------- drivers/gpu/drm/xe/xe_guc_tlb_inval.c | 4 +--- 4 files changed, 12 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index a686b04879d6..cf92de1c88a7 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -660,11 +660,9 @@ static void guc_fini_hw(void *arg) { struct xe_guc *guc = arg; struct xe_gt *gt = guc_to_gt(guc); - unsigned int fw_ref; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - xe_uc_sanitize_reset(&guc_to_gt(guc)->uc); - xe_force_wake_put(gt_to_fw(gt), fw_ref); + xe_with_force_wake(fw_ref, gt_to_fw(gt), XE_FORCEWAKE_ALL) + xe_uc_sanitize_reset(&guc_to_gt(guc)->uc); guc_g2g_fini(guc); } @@ -1621,15 +1619,14 @@ int xe_guc_start(struct xe_guc *guc) void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p) { struct xe_gt *gt = guc_to_gt(guc); - unsigned int fw_ref; u32 status; int i; xe_uc_fw_print(&guc->fw, p); if (!IS_SRIOV_VF(gt_to_xe(gt))) { - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return; status = xe_mmio_read32(>->mmio, GUC_STATUS); @@ -1649,8 +1646,6 @@ void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p) drm_printf(p, "\t%2d: \t0x%x\n", i, xe_mmio_read32(>->mmio, SOFT_SCRATCH(i))); } - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } drm_puts(p, "\n"); diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c index c01ccb35dc75..0c704a11078a 100644 --- a/drivers/gpu/drm/xe/xe_guc_log.c +++ b/drivers/gpu/drm/xe/xe_guc_log.c @@ -145,7 +145,6 @@ struct xe_guc_log_snapshot *xe_guc_log_snapshot_capture(struct xe_guc_log *log, struct xe_device *xe = log_to_xe(log); struct xe_guc *guc = log_to_guc(log); struct xe_gt *gt = log_to_gt(log); - unsigned int fw_ref; size_t remain; int i; @@ -165,13 +164,12 @@ struct xe_guc_log_snapshot *xe_guc_log_snapshot_capture(struct xe_guc_log *log, remain -= size; } - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) { + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) snapshot->stamp = ~0ULL; - } else { + else snapshot->stamp = xe_mmio_read64_2x32(>->mmio, GUC_PMTIMESTAMP_LO); - xe_force_wake_put(gt_to_fw(gt), fw_ref); - } + snapshot->ktime = ktime_get_boottime_ns(); snapshot->level = log->level; snapshot->ver_found = guc->fw.versions.found[XE_UC_FW_VER_RELEASE]; diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index d4ffdb71ef3d..7e0882074a99 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -1225,7 +1225,6 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) struct xe_guc *guc = exec_queue_to_guc(q); const char *process_name = "no process"; struct xe_device *xe = guc_to_xe(guc); - unsigned int fw_ref; int err = -ETIME; pid_t pid = -1; int i = 0; @@ -1258,13 +1257,11 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) if (!exec_queue_killed(q) && !xe->devcoredump.captured && !xe_guc_capture_get_matching_and_lock(q)) { /* take force wake before engine register manual capture */ - fw_ref = xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n"); xe_engine_snapshot_capture_for_queue(q); - - xe_force_wake_put(gt_to_fw(q->gt), fw_ref); } /* @@ -1455,7 +1452,7 @@ static void __guc_exec_queue_destroy_async(struct work_struct *w) struct xe_exec_queue *q = ge->q; struct xe_guc *guc = exec_queue_to_guc(q); - xe_pm_runtime_get(guc_to_xe(guc)); + guard(xe_pm_runtime)(guc_to_xe(guc)); trace_xe_exec_queue_destroy(q); if (xe_exec_queue_is_lr(q)) @@ -1464,8 +1461,6 @@ static void __guc_exec_queue_destroy_async(struct work_struct *w) cancel_delayed_work_sync(&ge->sched.base.work_tdr); xe_exec_queue_fini(q); - - xe_pm_runtime_put(guc_to_xe(guc)); } static void guc_exec_queue_destroy_async(struct xe_exec_queue *q) diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c index a80175c7c478..848d3493df10 100644 --- a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c @@ -71,12 +71,11 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) return send_tlb_inval(guc, action, ARRAY_SIZE(action)); } else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) { struct xe_mmio *mmio = >->mmio; - unsigned int fw_ref; if (IS_SRIOV_VF(xe)) return -ECANCELED; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) { xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1, PVC_GUC_TLB_INV_DESC1_INVALIDATE); @@ -86,7 +85,6 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) xe_mmio_write32(mmio, GUC_TLB_INV_CR, GUC_TLB_INV_CR_INVALIDATE); } - xe_force_wake_put(gt_to_fw(gt), fw_ref); } return -ECANCELED; From e9bc4162bf39154c1eb8d5edb11a00fc30771c39 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:45 -0800 Subject: [PATCH 017/187] drm/xe/guc_pc: Use scope-based cleanup Use scope-based cleanup for forcewake and runtime PM in the GuC PC code. This allows us to eliminate to goto-based cleanup and simplifies some other functions. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-35-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_guc_pc.c | 62 ++++++++++------------------------ 1 file changed, 17 insertions(+), 45 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index 951a49fb1d3e..e2e6edb851ae 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -499,21 +499,17 @@ u32 xe_guc_pc_get_cur_freq_fw(struct xe_guc_pc *pc) int xe_guc_pc_get_cur_freq(struct xe_guc_pc *pc, u32 *freq) { struct xe_gt *gt = pc_to_gt(pc); - unsigned int fw_ref; /* * GuC SLPC plays with cur freq request when GuCRC is enabled * Block RC6 for a more reliable read. */ - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) { - xe_force_wake_put(gt_to_fw(gt), fw_ref); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) return -ETIMEDOUT; - } *freq = get_cur_freq(gt); - xe_force_wake_put(gt_to_fw(gt), fw_ref); return 0; } @@ -1087,13 +1083,8 @@ int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc) */ int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mode) { - int ret; - - xe_pm_runtime_get(pc_to_xe(pc)); - ret = pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode); - xe_pm_runtime_put(pc_to_xe(pc)); - - return ret; + guard(xe_pm_runtime)(pc_to_xe(pc)); + return pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode); } /** @@ -1104,13 +1095,8 @@ int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mod */ int xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc *pc) { - int ret; - - xe_pm_runtime_get(pc_to_xe(pc)); - ret = pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE); - xe_pm_runtime_put(pc_to_xe(pc)); - - return ret; + guard(xe_pm_runtime)(pc_to_xe(pc)); + return pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE); } static void pc_init_pcode_freq(struct xe_guc_pc *pc) @@ -1198,7 +1184,7 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc *pc, const char *buf) return -EINVAL; guard(mutex)(&pc->freq_lock); - xe_pm_runtime_get_noresume(pc_to_xe(pc)); + guard(xe_pm_runtime_noresume)(pc_to_xe(pc)); ret = pc_action_set_param(pc, SLPC_PARAM_POWER_PROFILE, @@ -1209,8 +1195,6 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc *pc, const char *buf) else pc->power_profile = val; - xe_pm_runtime_put(pc_to_xe(pc)); - return ret; } @@ -1223,17 +1207,14 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) struct xe_device *xe = pc_to_xe(pc); struct xe_gt *gt = pc_to_gt(pc); u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data)); - unsigned int fw_ref; ktime_t earlier; int ret; xe_gt_assert(gt, xe_device_uc_enabled(xe)); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) { - xe_force_wake_put(gt_to_fw(gt), fw_ref); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) return -ETIMEDOUT; - } if (xe->info.skip_guc_pc) { if (xe->info.platform != XE_PVC) @@ -1241,9 +1222,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) /* Request max possible since dynamic freq mgmt is not enabled */ pc_set_cur_freq(pc, UINT_MAX); - - ret = 0; - goto out; + return 0; } xe_map_memset(xe, &pc->bo->vmap, 0, 0, size); @@ -1252,7 +1231,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) earlier = ktime_get(); ret = pc_action_reset(pc); if (ret) - goto out; + return ret; if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING, SLPC_RESET_TIMEOUT_MS)) { @@ -1263,8 +1242,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING, SLPC_RESET_EXTENDED_TIMEOUT_MS)) { xe_gt_err(gt, "GuC PC Start failed: Dynamic GT frequency control and GT sleep states are now disabled.\n"); - ret = -EIO; - goto out; + return -EIO; } xe_gt_warn(gt, "GuC PC excessive start time: %lldms", @@ -1273,21 +1251,20 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) ret = pc_init_freqs(pc); if (ret) - goto out; + return ret; ret = pc_set_mert_freq_cap(pc); if (ret) - goto out; + return ret; if (xe->info.platform == XE_PVC) { xe_guc_pc_gucrc_disable(pc); - ret = 0; - goto out; + return 0; } ret = pc_action_setup_gucrc(pc, GUCRC_FIRMWARE_CONTROL); if (ret) - goto out; + return ret; /* Enable SLPC Optimized Strategy for compute */ ret = pc_action_set_strategy(pc, SLPC_OPTIMIZED_STRATEGY_COMPUTE); @@ -1297,8 +1274,6 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) if (unlikely(ret)) xe_gt_err(gt, "Failed to set SLPC power profile: %pe\n", ERR_PTR(ret)); -out: - xe_force_wake_put(gt_to_fw(gt), fw_ref); return ret; } @@ -1330,19 +1305,16 @@ static void xe_guc_pc_fini_hw(void *arg) { struct xe_guc_pc *pc = arg; struct xe_device *xe = pc_to_xe(pc); - unsigned int fw_ref; if (xe_device_wedged(xe)) return; - fw_ref = xe_force_wake_get(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL); xe_guc_pc_gucrc_disable(pc); XE_WARN_ON(xe_guc_pc_stop(pc)); /* Bind requested freq to mert_freq_cap before unload */ pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), xe_guc_pc_get_rpe_freq(pc))); - - xe_force_wake_put(gt_to_fw(pc_to_gt(pc)), fw_ref); } /** From 008db7d4e1a465bf65c5cf7a4cc7ab041b2dc00e Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:46 -0800 Subject: [PATCH 018/187] drm/xe/mocs: Use scope-based cleanup Using scope-based cleanup for runtime PM and forcewake in the MOCS code allows us to eliminate some goto-based error handling and simplify some other functions. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-36-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/tests/xe_mocs.c | 17 ++++++----------- drivers/gpu/drm/xe/xe_mocs.c | 18 ++++++------------ 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/xe/tests/xe_mocs.c b/drivers/gpu/drm/xe/tests/xe_mocs.c index 6bb278167aaf..28374330b894 100644 --- a/drivers/gpu/drm/xe/tests/xe_mocs.c +++ b/drivers/gpu/drm/xe/tests/xe_mocs.c @@ -43,14 +43,12 @@ static void read_l3cc_table(struct xe_gt *gt, { struct kunit *test = kunit_get_current_test(); u32 l3cc, l3cc_expected; - unsigned int fw_ref, i; + unsigned int i; u32 reg_val; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) { - xe_force_wake_put(gt_to_fw(gt), fw_ref); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) KUNIT_FAIL_AND_ABORT(test, "Forcewake Failed.\n"); - } for (i = 0; i < info->num_mocs_regs; i++) { if (!(i & 1)) { @@ -74,7 +72,6 @@ static void read_l3cc_table(struct xe_gt *gt, KUNIT_EXPECT_EQ_MSG(test, l3cc_expected, l3cc, "l3cc idx=%u has incorrect val.\n", i); } - xe_force_wake_put(gt_to_fw(gt), fw_ref); } static void read_mocs_table(struct xe_gt *gt, @@ -82,14 +79,14 @@ static void read_mocs_table(struct xe_gt *gt, { struct kunit *test = kunit_get_current_test(); u32 mocs, mocs_expected; - unsigned int fw_ref, i; + unsigned int i; u32 reg_val; KUNIT_EXPECT_TRUE_MSG(test, info->unused_entries_index, "Unused entries index should have been defined\n"); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - KUNIT_ASSERT_NE_MSG(test, fw_ref, 0, "Forcewake Failed.\n"); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + KUNIT_ASSERT_NE_MSG(test, fw_ref.domains, 0, "Forcewake Failed.\n"); for (i = 0; i < info->num_mocs_regs; i++) { if (regs_are_mcr(gt)) @@ -106,8 +103,6 @@ static void read_mocs_table(struct xe_gt *gt, KUNIT_EXPECT_EQ_MSG(test, mocs_expected, mocs, "mocs reg 0x%x has incorrect val.\n", i); } - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } static int mocs_kernel_test_run_device(struct xe_device *xe) diff --git a/drivers/gpu/drm/xe/xe_mocs.c b/drivers/gpu/drm/xe/xe_mocs.c index 6613d3b48a84..0b7225bd77e0 100644 --- a/drivers/gpu/drm/xe/xe_mocs.c +++ b/drivers/gpu/drm/xe/xe_mocs.c @@ -811,26 +811,20 @@ int xe_mocs_dump(struct xe_gt *gt, struct drm_printer *p) struct xe_device *xe = gt_to_xe(gt); enum xe_force_wake_domains domain; struct xe_mocs_info table; - unsigned int fw_ref, flags; - int err = 0; + unsigned int flags; flags = get_mocs_settings(xe, &table); domain = flags & HAS_LNCF_MOCS ? XE_FORCEWAKE_ALL : XE_FW_GT; - xe_pm_runtime_get_noresume(xe); - fw_ref = xe_force_wake_get(gt_to_fw(gt), domain); - if (!xe_force_wake_ref_has_domain(fw_ref, domain)) { - err = -ETIMEDOUT; - goto err_fw; - } + guard(xe_pm_runtime_noresume)(xe); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), domain); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, domain)) + return -ETIMEDOUT; table.ops->dump(&table, flags, gt, p); -err_fw: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - xe_pm_runtime_put(xe); - return err; + return 0; } #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) From ba2562a391c66bff5bcc35583ce8c0beda862f03 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:47 -0800 Subject: [PATCH 019/187] drm/xe/pat: Use scope-based forcewake Use scope-based cleanup for forcewake in the PAT code to slightly simplify the code. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-37-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_pat.c | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c index 1b4d5d3def0f..717425dd0475 100644 --- a/drivers/gpu/drm/xe/xe_pat.c +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -239,11 +239,10 @@ static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry ta static int xelp_dump(struct xe_gt *gt, struct drm_printer *p) { struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref; int i; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; drm_printf(p, "PAT table:\n"); @@ -256,7 +255,6 @@ static int xelp_dump(struct xe_gt *gt, struct drm_printer *p) XELP_MEM_TYPE_STR_MAP[mem_type], pat); } - xe_force_wake_put(gt_to_fw(gt), fw_ref); return 0; } @@ -268,11 +266,10 @@ static const struct xe_pat_ops xelp_pat_ops = { static int xehp_dump(struct xe_gt *gt, struct drm_printer *p) { struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref; int i; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; drm_printf(p, "PAT table:\n"); @@ -287,7 +284,6 @@ static int xehp_dump(struct xe_gt *gt, struct drm_printer *p) XELP_MEM_TYPE_STR_MAP[mem_type], pat); } - xe_force_wake_put(gt_to_fw(gt), fw_ref); return 0; } @@ -299,11 +295,10 @@ static const struct xe_pat_ops xehp_pat_ops = { static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p) { struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref; int i; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; drm_printf(p, "PAT table:\n"); @@ -316,7 +311,6 @@ static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p) REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat); } - xe_force_wake_put(gt_to_fw(gt), fw_ref); return 0; } @@ -328,11 +322,10 @@ static const struct xe_pat_ops xehpc_pat_ops = { static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p) { struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref; int i; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; drm_printf(p, "PAT table:\n"); @@ -350,7 +343,6 @@ static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p) REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat); } - xe_force_wake_put(gt_to_fw(gt), fw_ref); return 0; } @@ -367,12 +359,11 @@ static const struct xe_pat_ops xelpg_pat_ops = { static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) { struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref; u32 pat; int i; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; drm_printf(p, "PAT table: (* = reserved entry)\n"); @@ -412,7 +403,6 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) REG_FIELD_GET(XE2_COH_MODE, pat), pat); - xe_force_wake_put(gt_to_fw(gt), fw_ref); return 0; } @@ -425,12 +415,11 @@ static const struct xe_pat_ops xe2_pat_ops = { static int xe3p_xpc_dump(struct xe_gt *gt, struct drm_printer *p) { struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref; u32 pat; int i; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; drm_printf(p, "PAT table: (* = reserved entry)\n"); @@ -462,7 +451,6 @@ static int xe3p_xpc_dump(struct xe_gt *gt, struct drm_printer *p) REG_FIELD_GET(XE2_COH_MODE, pat), pat); - xe_force_wake_put(gt_to_fw(gt), fw_ref); return 0; } From 62a35753f547c7744c062750703c2ae9883b33c5 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:48 -0800 Subject: [PATCH 020/187] drm/xe/pxp: Use scope-based cleanup Use scope-based cleanup for forcewake and runtime pm. This allows us to eliminate some goto-based error handling and simplify other functions. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-38-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_pxp.c | 55 ++++++++++++------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c index bdbdbbf6a678..508f4c128a48 100644 --- a/drivers/gpu/drm/xe/xe_pxp.c +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -58,10 +58,9 @@ bool xe_pxp_is_enabled(const struct xe_pxp *pxp) static bool pxp_prerequisites_done(const struct xe_pxp *pxp) { struct xe_gt *gt = pxp->gt; - unsigned int fw_ref; bool ready; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); /* * If force_wake fails we could falsely report the prerequisites as not @@ -71,14 +70,12 @@ static bool pxp_prerequisites_done(const struct xe_pxp *pxp) * PXP. Therefore, we can just log the force_wake error and not escalate * it. */ - XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)); + XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)); /* PXP requires both HuC authentication via GSC and GSC proxy initialized */ ready = xe_huc_is_authenticated(>->uc.huc, XE_HUC_AUTH_VIA_GSC) && xe_gsc_proxy_init_done(>->uc.gsc); - xe_force_wake_put(gt_to_fw(gt), fw_ref); - return ready; } @@ -104,13 +101,12 @@ int xe_pxp_get_readiness_status(struct xe_pxp *pxp) xe_uc_fw_status_to_error(pxp->gt->uc.gsc.fw.status)) return -EIO; - xe_pm_runtime_get(pxp->xe); + guard(xe_pm_runtime)(pxp->xe); /* PXP requires both HuC loaded and GSC proxy initialized */ if (pxp_prerequisites_done(pxp)) ret = 1; - xe_pm_runtime_put(pxp->xe); return ret; } @@ -135,35 +131,28 @@ static void pxp_invalidate_queues(struct xe_pxp *pxp); static int pxp_terminate_hw(struct xe_pxp *pxp) { struct xe_gt *gt = pxp->gt; - unsigned int fw_ref; int ret = 0; drm_dbg(&pxp->xe->drm, "Terminating PXP\n"); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) { - ret = -EIO; - goto out; - } + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) + return -EIO; /* terminate the hw session */ ret = xe_pxp_submit_session_termination(pxp, ARB_SESSION); if (ret) - goto out; + return ret; ret = pxp_wait_for_session_state(pxp, ARB_SESSION, false); if (ret) - goto out; + return ret; /* Trigger full HW cleanup */ xe_mmio_write32(>->mmio, KCR_GLOBAL_TERMINATE, 1); /* now we can tell the GSC to clean up its own state */ - ret = xe_pxp_submit_session_invalidation(&pxp->gsc_res, ARB_SESSION); - -out: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - return ret; + return xe_pxp_submit_session_invalidation(&pxp->gsc_res, ARB_SESSION); } static void mark_termination_in_progress(struct xe_pxp *pxp) @@ -326,14 +315,12 @@ static int kcr_pxp_set_status(const struct xe_pxp *pxp, bool enable) { u32 val = enable ? _MASKED_BIT_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) : _MASKED_BIT_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES); - unsigned int fw_ref; - fw_ref = xe_force_wake_get(gt_to_fw(pxp->gt), XE_FW_GT); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) return -EIO; xe_mmio_write32(&pxp->gt->mmio, KCR_INIT, val); - xe_force_wake_put(gt_to_fw(pxp->gt), fw_ref); return 0; } @@ -453,34 +440,28 @@ int xe_pxp_init(struct xe_device *xe) static int __pxp_start_arb_session(struct xe_pxp *pxp) { int ret; - unsigned int fw_ref; - fw_ref = xe_force_wake_get(gt_to_fw(pxp->gt), XE_FW_GT); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) return -EIO; - if (pxp_session_is_in_play(pxp, ARB_SESSION)) { - ret = -EEXIST; - goto out_force_wake; - } + if (pxp_session_is_in_play(pxp, ARB_SESSION)) + return -EEXIST; ret = xe_pxp_submit_session_init(&pxp->gsc_res, ARB_SESSION); if (ret) { drm_err(&pxp->xe->drm, "Failed to init PXP arb session: %pe\n", ERR_PTR(ret)); - goto out_force_wake; + return ret; } ret = pxp_wait_for_session_state(pxp, ARB_SESSION, true); if (ret) { drm_err(&pxp->xe->drm, "PXP ARB session failed to go in play%pe\n", ERR_PTR(ret)); - goto out_force_wake; + return ret; } drm_dbg(&pxp->xe->drm, "PXP ARB session is active\n"); - -out_force_wake: - xe_force_wake_put(gt_to_fw(pxp->gt), fw_ref); - return ret; + return 0; } /** From be675564cca58fd509b3666bbb1b7f61c6cc03f9 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:49 -0800 Subject: [PATCH 021/187] drm/xe/gsc: Use scope-based cleanup Use scope-based cleanup for forcewake and runtime PM to eliminate some goto-based error handling and simplify other functions. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-39-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_gsc.c | 21 ++++++--------------- drivers/gpu/drm/xe/xe_gsc_proxy.c | 17 +++++++---------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c index dd69cb834f8e..a3157b0fe791 100644 --- a/drivers/gpu/drm/xe/xe_gsc.c +++ b/drivers/gpu/drm/xe/xe_gsc.c @@ -352,7 +352,6 @@ static void gsc_work(struct work_struct *work) struct xe_gsc *gsc = container_of(work, typeof(*gsc), work); struct xe_gt *gt = gsc_to_gt(gsc); struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref; u32 actions; int ret; @@ -361,13 +360,12 @@ static void gsc_work(struct work_struct *work) gsc->work_actions = 0; spin_unlock_irq(&gsc->lock); - xe_pm_runtime_get(xe); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC); + guard(xe_pm_runtime)(xe); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC); if (actions & GSC_ACTION_ER_COMPLETE) { - ret = gsc_er_complete(gt); - if (ret) - goto out; + if (gsc_er_complete(gt)) + return; } if (actions & GSC_ACTION_FW_LOAD) { @@ -380,10 +378,6 @@ static void gsc_work(struct work_struct *work) if (actions & GSC_ACTION_SW_PROXY) xe_gsc_proxy_request_handler(gsc); - -out: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - xe_pm_runtime_put(xe); } void xe_gsc_hwe_irq_handler(struct xe_hw_engine *hwe, u16 intr_vec) @@ -615,7 +609,6 @@ void xe_gsc_print_info(struct xe_gsc *gsc, struct drm_printer *p) { struct xe_gt *gt = gsc_to_gt(gsc); struct xe_mmio *mmio = >->mmio; - unsigned int fw_ref; xe_uc_fw_print(&gsc->fw, p); @@ -624,8 +617,8 @@ void xe_gsc_print_info(struct xe_gsc *gsc, struct drm_printer *p) if (!xe_uc_fw_is_enabled(&gsc->fw)) return; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC); + if (!fw_ref.domains) return; drm_printf(p, "\nHECI1 FWSTS: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", @@ -635,6 +628,4 @@ void xe_gsc_print_info(struct xe_gsc *gsc, struct drm_printer *p) xe_mmio_read32(mmio, HECI_FWSTS4(MTL_GSC_HECI1_BASE)), xe_mmio_read32(mmio, HECI_FWSTS5(MTL_GSC_HECI1_BASE)), xe_mmio_read32(mmio, HECI_FWSTS6(MTL_GSC_HECI1_BASE))); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } diff --git a/drivers/gpu/drm/xe/xe_gsc_proxy.c b/drivers/gpu/drm/xe/xe_gsc_proxy.c index 464282a89eef..e7573a0c5e5d 100644 --- a/drivers/gpu/drm/xe/xe_gsc_proxy.c +++ b/drivers/gpu/drm/xe/xe_gsc_proxy.c @@ -440,22 +440,19 @@ static void xe_gsc_proxy_remove(void *arg) struct xe_gsc *gsc = arg; struct xe_gt *gt = gsc_to_gt(gsc); struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref = 0; if (!gsc->proxy.component_added) return; /* disable HECI2 IRQs */ - xe_pm_runtime_get(xe); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC); - if (!fw_ref) - xe_gt_err(gt, "failed to get forcewake to disable GSC interrupts\n"); + scoped_guard(xe_pm_runtime, xe) { + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC); + if (!fw_ref.domains) + xe_gt_err(gt, "failed to get forcewake to disable GSC interrupts\n"); - /* try do disable irq even if forcewake failed */ - gsc_proxy_irq_toggle(gsc, false); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); - xe_pm_runtime_put(xe); + /* try do disable irq even if forcewake failed */ + gsc_proxy_irq_toggle(gsc, false); + } xe_gsc_wait_for_worker_completion(gsc); From 89bba8fe925a9dc30b8fdcdd2316eb5b322c296d Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:50 -0800 Subject: [PATCH 022/187] drm/xe/device: Use scope-based cleanup Convert device code to use scope-based forcewake and runtime PM. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-40-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_device.c | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 9f2f19dc1fd3..92f883dd8877 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -166,7 +166,7 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file) struct xe_exec_queue *q; unsigned long idx; - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); /* * No need for exec_queue.lock here as there is no contention for it @@ -184,8 +184,6 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file) xe_vm_close_and_put(vm); xe_file_put(xef); - - xe_pm_runtime_put(xe); } static const struct drm_ioctl_desc xe_ioctls[] = { @@ -220,10 +218,10 @@ static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) if (xe_device_wedged(xe)) return -ECANCELED; - ret = xe_pm_runtime_get_ioctl(xe); + ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); + ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); if (ret >= 0) ret = drm_ioctl(file, cmd, arg); - xe_pm_runtime_put(xe); return ret; } @@ -238,10 +236,10 @@ static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned lo if (xe_device_wedged(xe)) return -ECANCELED; - ret = xe_pm_runtime_get_ioctl(xe); + ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); + ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); if (ret >= 0) ret = drm_compat_ioctl(file, cmd, arg); - xe_pm_runtime_put(xe); return ret; } @@ -775,7 +773,6 @@ ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ static int probe_has_flat_ccs(struct xe_device *xe) { struct xe_gt *gt; - unsigned int fw_ref; u32 reg; /* Always enabled/disabled, no runtime check to do */ @@ -786,8 +783,8 @@ static int probe_has_flat_ccs(struct xe_device *xe) if (!gt) return 0; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); @@ -797,8 +794,6 @@ static int probe_has_flat_ccs(struct xe_device *xe) drm_dbg(&xe->drm, "Flat CCS has been disabled in bios, May lead to performance impact"); - xe_force_wake_put(gt_to_fw(gt), fw_ref); - return 0; } @@ -1034,7 +1029,6 @@ void xe_device_wmb(struct xe_device *xe) */ static void tdf_request_sync(struct xe_device *xe) { - unsigned int fw_ref; struct xe_gt *gt; u8 id; @@ -1042,8 +1036,8 @@ static void tdf_request_sync(struct xe_device *xe) if (xe_gt_is_media_type(gt)) continue; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return; xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST); @@ -1058,15 +1052,12 @@ static void tdf_request_sync(struct xe_device *xe) if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, 150, NULL, false)) xe_gt_err_once(gt, "TD flush timeout\n"); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } } void xe_device_l2_flush(struct xe_device *xe) { struct xe_gt *gt; - unsigned int fw_ref; gt = xe_root_mmio_gt(xe); if (!gt) @@ -1075,8 +1066,8 @@ void xe_device_l2_flush(struct xe_device *xe) if (!XE_GT_WA(gt, 16023588340)) return; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return; spin_lock(>->global_invl_lock); @@ -1086,8 +1077,6 @@ void xe_device_l2_flush(struct xe_device *xe) xe_gt_err_once(gt, "Global invalidation timeout\n"); spin_unlock(>->global_invl_lock); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } /** From 7fc616a309e854f75bcc46e82b948065886ebddf Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:51 -0800 Subject: [PATCH 023/187] drm/xe/devcoredump: Use scope-based cleanup Use scope-based cleanup for forcewake and runtime PM in the devcoredump code. This eliminates some goto-based error handling and slightly simplifies other functions. v2: - Move the forcewake acquisition slightly higher in devcoredump_snapshot() so that we maintain an easy-to-understand LIFO cleanup order. (Gustavo) Cc: Gustavo Sousa Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-41-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_devcoredump.c | 30 ++++++++++++----------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c index 203e3038cc81..bf347714b5e0 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump.c +++ b/drivers/gpu/drm/xe/xe_devcoredump.c @@ -276,7 +276,6 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work) struct xe_devcoredump_snapshot *ss = container_of(work, typeof(*ss), work); struct xe_devcoredump *coredump = container_of(ss, typeof(*coredump), snapshot); struct xe_device *xe = coredump_to_xe(coredump); - unsigned int fw_ref; /* * NB: Despite passing a GFP_ flags parameter here, more allocations are done @@ -287,15 +286,15 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work) xe_devcoredump_read, xe_devcoredump_free, XE_COREDUMP_TIMEOUT_JIFFIES); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); /* keep going if fw fails as we still want to save the memory and SW data */ - fw_ref = xe_force_wake_get(gt_to_fw(ss->gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) - xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n"); - xe_vm_snapshot_capture_delayed(ss->vm); - xe_guc_exec_queue_snapshot_capture_delayed(ss->ge); - xe_force_wake_put(gt_to_fw(ss->gt), fw_ref); + xe_with_force_wake(fw_ref, gt_to_fw(ss->gt), XE_FORCEWAKE_ALL) { + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) + xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n"); + xe_vm_snapshot_capture_delayed(ss->vm); + xe_guc_exec_queue_snapshot_capture_delayed(ss->ge); + } ss->read.chunk_position = 0; @@ -306,7 +305,7 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work) ss->read.buffer = kvmalloc(XE_DEVCOREDUMP_CHUNK_MAX, GFP_USER); if (!ss->read.buffer) - goto put_pm; + return; __xe_devcoredump_read(ss->read.buffer, XE_DEVCOREDUMP_CHUNK_MAX, @@ -314,15 +313,12 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work) } else { ss->read.buffer = kvmalloc(ss->read.size, GFP_USER); if (!ss->read.buffer) - goto put_pm; + return; __xe_devcoredump_read(ss->read.buffer, ss->read.size, 0, coredump); xe_devcoredump_snapshot_free(ss); } - -put_pm: - xe_pm_runtime_put(xe); } static void devcoredump_snapshot(struct xe_devcoredump *coredump, @@ -332,7 +328,6 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump, struct xe_devcoredump_snapshot *ss = &coredump->snapshot; struct xe_guc *guc = exec_queue_to_guc(q); const char *process_name = "no process"; - unsigned int fw_ref; bool cookie; ss->snapshot_time = ktime_get_real(); @@ -348,10 +343,10 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump, ss->gt = q->gt; INIT_WORK(&ss->work, xe_devcoredump_deferred_snap_work); - cookie = dma_fence_begin_signalling(); - /* keep going if fw fails as we still want to save the memory and SW data */ - fw_ref = xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); + + cookie = dma_fence_begin_signalling(); ss->guc.log = xe_guc_log_snapshot_capture(&guc->log, true); ss->guc.ct = xe_guc_ct_snapshot_capture(&guc->ct); @@ -364,7 +359,6 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump, queue_work(system_unbound_wq, &ss->work); - xe_force_wake_put(gt_to_fw(q->gt), fw_ref); dma_fence_end_signalling(cookie); } From b11f88699b13d7b6e314b9d19107aebdc79570f0 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:52 -0800 Subject: [PATCH 024/187] drm/xe/display: Use scoped-cleanup Eliminate some goto-based cleanup by utilizing scoped cleanup helpers. v2: - Eliminate unnecessary 'ret' variable in intel_hdcp_gsc_check_status() (Gustavo) Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-42-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/display/xe_fb_pin.c | 23 +++++++----------- drivers/gpu/drm/xe/display/xe_hdcp_gsc.c | 31 +++++++----------------- 2 files changed, 18 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index 1fd4a815e784..6a935a75f2a4 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -210,10 +210,11 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, /* TODO: Consider sharing framebuffer mapping? * embed i915_vma inside intel_framebuffer */ - xe_pm_runtime_get_noresume(xe); - ret = mutex_lock_interruptible(&ggtt->lock); + guard(xe_pm_runtime_noresume)(xe); + ACQUIRE(mutex_intr, lock)(&ggtt->lock); + ret = ACQUIRE_ERR(mutex_intr, &lock); if (ret) - goto out; + return ret; align = XE_PAGE_SIZE; if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K) @@ -223,15 +224,13 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, vma->node = bo->ggtt_node[tile0->id]; } else if (view->type == I915_GTT_VIEW_NORMAL) { vma->node = xe_ggtt_node_init(ggtt); - if (IS_ERR(vma->node)) { - ret = PTR_ERR(vma->node); - goto out_unlock; - } + if (IS_ERR(vma->node)) + return PTR_ERR(vma->node); ret = xe_ggtt_node_insert_locked(vma->node, xe_bo_size(bo), align, 0); if (ret) { xe_ggtt_node_fini(vma->node); - goto out_unlock; + return ret; } xe_ggtt_map_bo(ggtt, vma->node, bo, xe->pat.idx[XE_CACHE_NONE]); @@ -245,13 +244,13 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, vma->node = xe_ggtt_node_init(ggtt); if (IS_ERR(vma->node)) { ret = PTR_ERR(vma->node); - goto out_unlock; + return ret; } ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0); if (ret) { xe_ggtt_node_fini(vma->node); - goto out_unlock; + return ret; } ggtt_ofs = vma->node->base.start; @@ -265,10 +264,6 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, rot_info->plane[i].dst_stride); } -out_unlock: - mutex_unlock(&ggtt->lock); -out: - xe_pm_runtime_put(xe); return ret; } diff --git a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c index 4ae847b628e2..71d21fde1736 100644 --- a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c +++ b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c @@ -36,8 +36,6 @@ bool intel_hdcp_gsc_check_status(struct drm_device *drm) struct xe_tile *tile = xe_device_get_root_tile(xe); struct xe_gt *gt = tile->media_gt; struct xe_gsc *gsc = >->uc.gsc; - bool ret = true; - unsigned int fw_ref; if (!gsc || !xe_uc_fw_is_enabled(&gsc->fw)) { drm_dbg_kms(&xe->drm, @@ -45,22 +43,15 @@ bool intel_hdcp_gsc_check_status(struct drm_device *drm) return false; } - xe_pm_runtime_get(xe); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC); - if (!fw_ref) { + guard(xe_pm_runtime)(xe); + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC); + if (!fw_ref.domains) { drm_dbg_kms(&xe->drm, "failed to get forcewake to check proxy status\n"); - ret = false; - goto out; + return false; } - if (!xe_gsc_proxy_init_done(gsc)) - ret = false; - - xe_force_wake_put(gt_to_fw(gt), fw_ref); -out: - xe_pm_runtime_put(xe); - return ret; + return xe_gsc_proxy_init_done(gsc); } /*This function helps allocate memory for the command that we will send to gsc cs */ @@ -166,17 +157,15 @@ ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_context, u32 addr_out_off, addr_in_wr_off = 0; int ret, tries = 0; - if (msg_in_len > max_msg_size || msg_out_len > max_msg_size) { - ret = -ENOSPC; - goto out; - } + if (msg_in_len > max_msg_size || msg_out_len > max_msg_size) + return -ENOSPC; msg_size_in = msg_in_len + HDCP_GSC_HEADER_SIZE; msg_size_out = msg_out_len + HDCP_GSC_HEADER_SIZE; addr_out_off = PAGE_SIZE; host_session_id = xe_gsc_create_host_session_id(); - xe_pm_runtime_get_noresume(xe); + guard(xe_pm_runtime_noresume)(xe); addr_in_wr_off = xe_gsc_emit_header(xe, &gsc_context->hdcp_bo->vmap, addr_in_wr_off, HECI_MEADDRESS_HDCP, host_session_id, msg_in_len); @@ -201,13 +190,11 @@ ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_context, } while (++tries < 20); if (ret) - goto out; + return ret; xe_map_memcpy_from(xe, msg_out, &gsc_context->hdcp_bo->vmap, addr_out_off + HDCP_GSC_HEADER_SIZE, msg_out_len); -out: - xe_pm_runtime_put(xe); return ret; } From 1fe7ea3287ba35b92db7f8fecf773a6bc9273f51 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:53 -0800 Subject: [PATCH 025/187] drm/xe: Return forcewake reference type from force_wake_get_any_engine() Adjust the signature of force_wake_get_any_engine() such that it returns a 'struct xe_force_wake_ref' rather than a boolean success/failure. Failure cases are now recognized by inspecting the hardware engine returned by reference; a NULL hwe indicates that no engine's forcewake could be obtained. These changes will make it cleaner and easier to incorporate scope-based cleanup in force_wake_get_any_engine()'s caller in a future patch. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-43-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_drm_client.c | 38 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c index f931ff9b1ec0..78551832723b 100644 --- a/drivers/gpu/drm/xe/xe_drm_client.c +++ b/drivers/gpu/drm/xe/xe_drm_client.c @@ -285,32 +285,31 @@ static struct xe_hw_engine *any_engine(struct xe_device *xe) return NULL; } -static bool force_wake_get_any_engine(struct xe_device *xe, - struct xe_hw_engine **phwe, - unsigned int *pfw_ref) +/* + * Pick any engine and grab its forcewake. On error phwe will be NULL and + * the returned forcewake reference will be invalid. Callers should check + * phwe against NULL. + */ +static struct xe_force_wake_ref force_wake_get_any_engine(struct xe_device *xe, + struct xe_hw_engine **phwe) { enum xe_force_wake_domains domain; - unsigned int fw_ref; + struct xe_force_wake_ref fw_ref = {}; struct xe_hw_engine *hwe; - struct xe_force_wake *fw; + + *phwe = NULL; hwe = any_engine(xe); if (!hwe) - return false; + return fw_ref; /* will be invalid */ domain = xe_hw_engine_to_fw_domain(hwe); - fw = gt_to_fw(hwe->gt); - fw_ref = xe_force_wake_get(fw, domain); - if (!xe_force_wake_ref_has_domain(fw_ref, domain)) { - xe_force_wake_put(fw, fw_ref); - return false; - } + fw_ref = xe_force_wake_constructor(gt_to_fw(hwe->gt), domain); + if (xe_force_wake_ref_has_domain(fw_ref.domains, domain)) + *phwe = hwe; /* valid forcewake */ - *phwe = hwe; - *pfw_ref = fw_ref; - - return true; + return fw_ref; } static void show_run_ticks(struct drm_printer *p, struct drm_file *file) @@ -322,7 +321,7 @@ static void show_run_ticks(struct drm_printer *p, struct drm_file *file) struct xe_hw_engine *hwe; struct xe_exec_queue *q; u64 gpu_timestamp; - unsigned int fw_ref; + struct xe_force_wake_ref fw_ref; /* * RING_TIMESTAMP registers are inaccessible in VF mode. @@ -340,7 +339,8 @@ static void show_run_ticks(struct drm_printer *p, struct drm_file *file) !atomic_read(&xef->exec_queue.pending_removal)); xe_pm_runtime_get(xe); - if (!force_wake_get_any_engine(xe, &hwe, &fw_ref)) { + fw_ref = force_wake_get_any_engine(xe, &hwe); + if (!hwe) { xe_pm_runtime_put(xe); return; } @@ -360,7 +360,7 @@ static void show_run_ticks(struct drm_printer *p, struct drm_file *file) gpu_timestamp = xe_hw_engine_read_timestamp(hwe); - xe_force_wake_put(gt_to_fw(hwe->gt), fw_ref); + xe_force_wake_put(gt_to_fw(hwe->gt), fw_ref.domains); xe_pm_runtime_put(xe); for (class = 0; class < XE_ENGINE_CLASS_MAX; class++) { From 062a6b83d56066f86c1bed170265ecfa98b83175 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:54 -0800 Subject: [PATCH 026/187] drm/xe/drm_client: Use scope-based cleanup Use scope-based cleanup for forcewake and runtime PM. v2: - Use xe_force_wake_release_only rather than a custom one-off class for "any engine" forcewake. (Gustavo) Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-44-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_drm_client.c | 37 +++++++++++++----------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c index 78551832723b..2787bbb36141 100644 --- a/drivers/gpu/drm/xe/xe_drm_client.c +++ b/drivers/gpu/drm/xe/xe_drm_client.c @@ -321,7 +321,6 @@ static void show_run_ticks(struct drm_printer *p, struct drm_file *file) struct xe_hw_engine *hwe; struct xe_exec_queue *q; u64 gpu_timestamp; - struct xe_force_wake_ref fw_ref; /* * RING_TIMESTAMP registers are inaccessible in VF mode. @@ -338,30 +337,26 @@ static void show_run_ticks(struct drm_printer *p, struct drm_file *file) wait_var_event(&xef->exec_queue.pending_removal, !atomic_read(&xef->exec_queue.pending_removal)); - xe_pm_runtime_get(xe); - fw_ref = force_wake_get_any_engine(xe, &hwe); - if (!hwe) { - xe_pm_runtime_put(xe); - return; - } + scoped_guard(xe_pm_runtime, xe) { + CLASS(xe_force_wake_release_only, fw_ref)(force_wake_get_any_engine(xe, &hwe)); + if (!hwe) + return; - /* Accumulate all the exec queues from this client */ - mutex_lock(&xef->exec_queue.lock); - xa_for_each(&xef->exec_queue.xa, i, q) { - xe_exec_queue_get(q); + /* Accumulate all the exec queues from this client */ + mutex_lock(&xef->exec_queue.lock); + xa_for_each(&xef->exec_queue.xa, i, q) { + xe_exec_queue_get(q); + mutex_unlock(&xef->exec_queue.lock); + + xe_exec_queue_update_run_ticks(q); + + mutex_lock(&xef->exec_queue.lock); + xe_exec_queue_put(q); + } mutex_unlock(&xef->exec_queue.lock); - xe_exec_queue_update_run_ticks(q); - - mutex_lock(&xef->exec_queue.lock); - xe_exec_queue_put(q); + gpu_timestamp = xe_hw_engine_read_timestamp(hwe); } - mutex_unlock(&xef->exec_queue.lock); - - gpu_timestamp = xe_hw_engine_read_timestamp(hwe); - - xe_force_wake_put(gt_to_fw(hwe->gt), fw_ref.domains); - xe_pm_runtime_put(xe); for (class = 0; class < XE_ENGINE_CLASS_MAX; class++) { const char *class_name; From bedad003e8e76ef37d069816ae418e7917c5f4d8 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:55 -0800 Subject: [PATCH 027/187] drm/xe/gt_debugfs: Use scope-based cleanup Use scope-based cleanup for forcewake and runtime PM to simplify the debugfs code slightly. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-45-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_gt_debugfs.c | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c index e4fd632f43cf..7c3de6539044 100644 --- a/drivers/gpu/drm/xe/xe_gt_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -105,35 +105,24 @@ int xe_gt_debugfs_show_with_rpm(struct seq_file *m, void *data) struct drm_info_node *node = m->private; struct xe_gt *gt = node_to_gt(node); struct xe_device *xe = gt_to_xe(gt); - int ret; - xe_pm_runtime_get(xe); - ret = xe_gt_debugfs_simple_show(m, data); - xe_pm_runtime_put(xe); - - return ret; + guard(xe_pm_runtime)(xe); + return xe_gt_debugfs_simple_show(m, data); } static int hw_engines(struct xe_gt *gt, struct drm_printer *p) { struct xe_hw_engine *hwe; enum xe_hw_engine_id id; - unsigned int fw_ref; - int ret = 0; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) { - ret = -ETIMEDOUT; - goto fw_put; - } + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) + return -ETIMEDOUT; for_each_hw_engine(hwe, gt, id) xe_hw_engine_print(hwe, p); -fw_put: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - - return ret; + return 0; } static int steering(struct xe_gt *gt, struct drm_printer *p) @@ -269,9 +258,8 @@ static void force_reset(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); xe_gt_reset_async(gt); - xe_pm_runtime_put(xe); } static ssize_t force_reset_write(struct file *file, @@ -297,9 +285,8 @@ static void force_reset_sync(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); xe_gt_reset(gt); - xe_pm_runtime_put(xe); } static ssize_t force_reset_sync_write(struct file *file, From 175b9aaba3e2b32935204b46ff1ea6356b775801 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:56 -0800 Subject: [PATCH 028/187] drm/xe/huc: Use scope-based forcewake Use scope-based forcewake in the HuC code for a small simplification and consistency with other parts of the driver. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-46-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_huc.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_huc.c b/drivers/gpu/drm/xe/xe_huc.c index 0a70c8924582..4212162913af 100644 --- a/drivers/gpu/drm/xe/xe_huc.c +++ b/drivers/gpu/drm/xe/xe_huc.c @@ -300,19 +300,16 @@ void xe_huc_sanitize(struct xe_huc *huc) void xe_huc_print_info(struct xe_huc *huc, struct drm_printer *p) { struct xe_gt *gt = huc_to_gt(huc); - unsigned int fw_ref; xe_uc_fw_print(&huc->fw, p); if (!xe_uc_fw_is_enabled(&huc->fw)) return; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return; drm_printf(p, "\nHuC status: 0x%08x\n", xe_mmio_read32(>->mmio, HUC_KERNEL_LOAD_INFO)); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } From 667fc27e81bc4bd0aae008b6c3f16df9e707707e Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:57 -0800 Subject: [PATCH 029/187] drm/xe/query: Use scope-based forcewake Use scope-based forcewake handling for consistency with other parts of the driver. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-47-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_query.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 1c0915e2cc16..a7bf1fd6dd6a 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -122,7 +122,6 @@ query_engine_cycles(struct xe_device *xe, __ktime_func_t cpu_clock; struct xe_hw_engine *hwe; struct xe_gt *gt; - unsigned int fw_ref; if (IS_SRIOV_VF(xe)) return -EOPNOTSUPP; @@ -158,17 +157,14 @@ query_engine_cycles(struct xe_device *xe, if (!hwe) return -EINVAL; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) { - xe_force_wake_put(gt_to_fw(gt), fw_ref); - return -EIO; + xe_with_force_wake(fw_ref, gt_to_fw(gt), XE_FORCEWAKE_ALL) { + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) + return -EIO; + + hwe_read_timestamp(hwe, &resp.engine_cycles, &resp.cpu_timestamp, + &resp.cpu_delta, cpu_clock); } - hwe_read_timestamp(hwe, &resp.engine_cycles, &resp.cpu_timestamp, - &resp.cpu_delta, cpu_clock); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); - if (GRAPHICS_VER(xe) >= 20) resp.width = 64; else From f875dead3ec75ba37edf986603a69c89e4a771e6 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:58 -0800 Subject: [PATCH 030/187] drm/xe/reg_sr: Use scope-based forcewake Use scope-based forcewake to slightly simplify the reg_sr code. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-48-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_reg_sr.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c index fc8447a838c4..1a465385f909 100644 --- a/drivers/gpu/drm/xe/xe_reg_sr.c +++ b/drivers/gpu/drm/xe/xe_reg_sr.c @@ -168,7 +168,6 @@ void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) { struct xe_reg_sr_entry *entry; unsigned long reg; - unsigned int fw_ref; if (xa_empty(&sr->xa)) return; @@ -178,20 +177,14 @@ void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) - goto err_force_wake; + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n"); + return; + } xa_for_each(&sr->xa, reg, entry) apply_one_mmio(gt, entry); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); - - return; - -err_force_wake: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n"); } /** From 60a4661d12ca58c794337d09d26f3f57e235cd2d Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:43:59 -0800 Subject: [PATCH 031/187] drm/xe/vram: Use scope-based forcewake Switch VRAM code to use scope-based forcewake for consistency with other parts of the driver. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-49-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_vram.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index 0e10da790cc5..0a645e76e5fa 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -186,12 +186,11 @@ static int determine_lmem_bar_size(struct xe_device *xe, struct xe_vram_region * static int get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size, u64 *poffset) { struct xe_device *xe = gt_to_xe(gt); - unsigned int fw_ref; u64 offset; u32 reg; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; if (GRAPHICS_VER(xe) >= 20) { @@ -223,7 +222,6 @@ static int get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size, u64 *poffset) offset = (u64)REG_FIELD_GET(XEHP_FLAT_CCS_PTR, reg) * SZ_64K; } - xe_force_wake_put(gt_to_fw(gt), fw_ref); *poffset = offset; return 0; From faa4b73bf75003ad89ed4d1afd3e9e59a48a63fd Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:44:00 -0800 Subject: [PATCH 032/187] drm/xe/bo: Use scope-based runtime PM Use scope-based runtime power management in the BO code for consistency with other parts of the driver. v2: - Drop unnecessary 'ret' variable. (Gustavo) Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-50-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_bo.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index bbda20d7c089..465cf9fc7ce9 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -2032,13 +2032,9 @@ static int xe_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, struct ttm_buffer_object *ttm_bo = vma->vm_private_data; struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); struct xe_device *xe = xe_bo_device(bo); - int ret; - xe_pm_runtime_get(xe); - ret = ttm_bo_vm_access(vma, addr, buf, len, write); - xe_pm_runtime_put(xe); - - return ret; + guard(xe_pm_runtime)(xe); + return ttm_bo_vm_access(vma, addr, buf, len, write); } /** From 8a579f4b2476fd1df07e2bca9fedc82a39a56a65 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:44:01 -0800 Subject: [PATCH 033/187] drm/xe/ggtt: Use scope-based runtime pm Switch the GGTT code to scope-based runtime PM for consistency with other parts of the driver. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-51-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_ggtt.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index ef481b334af4..48ab8b43fcd0 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -396,9 +396,8 @@ static void ggtt_node_remove_work_func(struct work_struct *work) delayed_removal_work); struct xe_device *xe = tile_to_xe(node->ggtt->tile); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ggtt_node_remove(node); - xe_pm_runtime_put(xe); } /** From 008f3fcf31b7a7a6b13b774b76871679805c2661 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:44:02 -0800 Subject: [PATCH 034/187] drm/xe/hwmon: Use scope-based runtime PM Use scope-based runtime power management in the hwmon code for consistency with other parts of the driver. v2: - Drop unnecessary 'ret' variables. (Gustavo) Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-52-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_hwmon.c | 52 ++++++++++------------------------- 1 file changed, 14 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c index 97879daeefc1..ff2aea52ef75 100644 --- a/drivers/gpu/drm/xe/xe_hwmon.c +++ b/drivers/gpu/drm/xe/xe_hwmon.c @@ -502,7 +502,7 @@ xe_hwmon_power_max_interval_show(struct device *dev, struct device_attribute *at int ret = 0; - xe_pm_runtime_get(hwmon->xe); + guard(xe_pm_runtime)(hwmon->xe); mutex_lock(&hwmon->hwmon_lock); @@ -521,8 +521,6 @@ xe_hwmon_power_max_interval_show(struct device *dev, struct device_attribute *at mutex_unlock(&hwmon->hwmon_lock); - xe_pm_runtime_put(hwmon->xe); - x = REG_FIELD_GET(PWR_LIM_TIME_X, reg_val); y = REG_FIELD_GET(PWR_LIM_TIME_Y, reg_val); @@ -604,7 +602,7 @@ xe_hwmon_power_max_interval_store(struct device *dev, struct device_attribute *a rxy = REG_FIELD_PREP(PWR_LIM_TIME_X, x) | REG_FIELD_PREP(PWR_LIM_TIME_Y, y); - xe_pm_runtime_get(hwmon->xe); + guard(xe_pm_runtime)(hwmon->xe); mutex_lock(&hwmon->hwmon_lock); @@ -616,8 +614,6 @@ xe_hwmon_power_max_interval_store(struct device *dev, struct device_attribute *a mutex_unlock(&hwmon->hwmon_lock); - xe_pm_runtime_put(hwmon->xe); - return count; } @@ -1124,37 +1120,25 @@ xe_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val) { struct xe_hwmon *hwmon = dev_get_drvdata(dev); - int ret; - xe_pm_runtime_get(hwmon->xe); + guard(xe_pm_runtime)(hwmon->xe); switch (type) { case hwmon_temp: - ret = xe_hwmon_temp_read(hwmon, attr, channel, val); - break; + return xe_hwmon_temp_read(hwmon, attr, channel, val); case hwmon_power: - ret = xe_hwmon_power_read(hwmon, attr, channel, val); - break; + return xe_hwmon_power_read(hwmon, attr, channel, val); case hwmon_curr: - ret = xe_hwmon_curr_read(hwmon, attr, channel, val); - break; + return xe_hwmon_curr_read(hwmon, attr, channel, val); case hwmon_in: - ret = xe_hwmon_in_read(hwmon, attr, channel, val); - break; + return xe_hwmon_in_read(hwmon, attr, channel, val); case hwmon_energy: - ret = xe_hwmon_energy_read(hwmon, attr, channel, val); - break; + return xe_hwmon_energy_read(hwmon, attr, channel, val); case hwmon_fan: - ret = xe_hwmon_fan_read(hwmon, attr, channel, val); - break; + return xe_hwmon_fan_read(hwmon, attr, channel, val); default: - ret = -EOPNOTSUPP; - break; + return -EOPNOTSUPP; } - - xe_pm_runtime_put(hwmon->xe); - - return ret; } static int @@ -1162,25 +1146,17 @@ xe_hwmon_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long val) { struct xe_hwmon *hwmon = dev_get_drvdata(dev); - int ret; - xe_pm_runtime_get(hwmon->xe); + guard(xe_pm_runtime)(hwmon->xe); switch (type) { case hwmon_power: - ret = xe_hwmon_power_write(hwmon, attr, channel, val); - break; + return xe_hwmon_power_write(hwmon, attr, channel, val); case hwmon_curr: - ret = xe_hwmon_curr_write(hwmon, attr, channel, val); - break; + return xe_hwmon_curr_write(hwmon, attr, channel, val); default: - ret = -EOPNOTSUPP; - break; + return -EOPNOTSUPP; } - - xe_pm_runtime_put(hwmon->xe); - - return ret; } static int xe_hwmon_read_label(struct device *dev, From fcee6854e6fb9ddff0597296728a4f7fd67bccc9 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:44:03 -0800 Subject: [PATCH 035/187] drm/xe/sriov: Use scope-based runtime PM Use scope-based runtime power management in the SRIOV code for consistency with other parts of the driver. v2: - Drop unnecessary 'ret' variables. (Gustavo) Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-53-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_pci_sriov.c | 10 +++------- drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c | 6 ++---- drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c | 16 ++++------------ drivers/gpu/drm/xe/xe_sriov_vf_ccs.c | 5 +---- drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c | 3 +-- 5 files changed, 11 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c index 9ff69c4843b0..3fd22034f03e 100644 --- a/drivers/gpu/drm/xe/xe_pci_sriov.c +++ b/drivers/gpu/drm/xe/xe_pci_sriov.c @@ -219,7 +219,6 @@ static int pf_disable_vfs(struct xe_device *xe) int xe_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) { struct xe_device *xe = pdev_to_xe_device(pdev); - int ret; if (!IS_SRIOV_PF(xe)) return -ENODEV; @@ -233,14 +232,11 @@ int xe_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) if (num_vfs && pci_num_vf(pdev)) return -EBUSY; - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); if (num_vfs > 0) - ret = pf_enable_vfs(xe, num_vfs); + return pf_enable_vfs(xe, num_vfs); else - ret = pf_disable_vfs(xe); - xe_pm_runtime_put(xe); - - return ret; + return pf_disable_vfs(xe); } /** diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c index bad751217e1e..e84bdde9bc80 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c @@ -70,9 +70,8 @@ static ssize_t from_file_write_to_xe_call(struct file *file, const char __user * if (ret < 0) return ret; if (yes) { - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ret = call(xe); - xe_pm_runtime_put(xe); } if (ret < 0) return ret; @@ -209,9 +208,8 @@ static ssize_t from_file_write_to_vf_call(struct file *file, const char __user * if (ret < 0) return ret; if (yes) { - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ret = call(xe, vfid); - xe_pm_runtime_put(xe); } if (ret < 0) return ret; diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c index c0b767ac735c..3d140506ba36 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c @@ -389,16 +389,12 @@ static ssize_t xe_sriov_dev_attr_store(struct kobject *kobj, struct attribute *a struct xe_sriov_dev_attr *vattr = to_xe_sriov_dev_attr(attr); struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj); struct xe_device *xe = vkobj->xe; - ssize_t ret; if (!vattr->store) return -EPERM; - xe_pm_runtime_get(xe); - ret = xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, buf, count); - xe_pm_runtime_put(xe); - - return ret; + guard(xe_pm_runtime)(xe); + return xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, buf, count); } static ssize_t xe_sriov_vf_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) @@ -423,18 +419,14 @@ static ssize_t xe_sriov_vf_attr_store(struct kobject *kobj, struct attribute *at struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj); struct xe_device *xe = vkobj->xe; unsigned int vfid = vkobj->vfid; - ssize_t ret; xe_sriov_pf_assert_vfid(xe, vfid); if (!vattr->store) return -EPERM; - xe_pm_runtime_get(xe); - ret = xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, vfid, buf, count); - xe_pm_runtime_get(xe); - - return ret; + guard(xe_pm_runtime)(xe); + return xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, vfid, buf, count); } static const struct sysfs_ops xe_sriov_dev_sysfs_ops = { diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index 33f4238604e1..052a5071e69f 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -477,8 +477,7 @@ void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p) if (!IS_VF_CCS_READY(xe)) return; - xe_pm_runtime_get(xe); - + guard(xe_pm_runtime)(xe); for_each_ccs_rw_ctx(ctx_id) { bb_pool = xe->sriov.vf.ccs.contexts[ctx_id].mem.ccs_bb_pool; if (!bb_pool) @@ -489,6 +488,4 @@ void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p) drm_suballoc_dump_debug_info(&bb_pool->base, p, xe_sa_manager_gpu_addr(bb_pool)); drm_puts(p, "\n"); } - - xe_pm_runtime_put(xe); } diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c index f3f478f14ff5..7f97db2f89bb 100644 --- a/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c +++ b/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c @@ -141,12 +141,11 @@ static int NAME##_set(void *data, u64 val) \ if (val > (TYPE)~0ull) \ return -EOVERFLOW; \ \ - xe_pm_runtime_get(xe); \ + guard(xe_pm_runtime)(xe); \ err = xe_sriov_pf_wait_ready(xe) ?: \ xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val); \ if (!err) \ xe_sriov_pf_provision_set_custom_mode(xe); \ - xe_pm_runtime_put(xe); \ \ return err; \ } \ From 3a344ddce25eb8b7cf63de8b5c54de10e5bb764f Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:44:04 -0800 Subject: [PATCH 036/187] drm/xe/tests: Use scope-based runtime PM Use scope-based handling of runtime PM in the kunit tests for consistency with other parts of the driver. Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-54-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/tests/xe_bo.c | 10 ++-------- drivers/gpu/drm/xe/tests/xe_dma_buf.c | 3 +-- drivers/gpu/drm/xe/tests/xe_migrate.c | 10 ++-------- drivers/gpu/drm/xe/tests/xe_mocs.c | 10 ++-------- 4 files changed, 7 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/xe/tests/xe_bo.c b/drivers/gpu/drm/xe/tests/xe_bo.c index 2294cf89f3e1..2278e589a493 100644 --- a/drivers/gpu/drm/xe/tests/xe_bo.c +++ b/drivers/gpu/drm/xe/tests/xe_bo.c @@ -185,8 +185,7 @@ static int ccs_test_run_device(struct xe_device *xe) return 0; } - xe_pm_runtime_get(xe); - + guard(xe_pm_runtime)(xe); for_each_tile(tile, xe, id) { /* For igfx run only for primary tile */ if (!IS_DGFX(xe) && id > 0) @@ -194,8 +193,6 @@ static int ccs_test_run_device(struct xe_device *xe) ccs_test_run_tile(xe, tile, test); } - xe_pm_runtime_put(xe); - return 0; } @@ -356,13 +353,10 @@ static int evict_test_run_device(struct xe_device *xe) return 0; } - xe_pm_runtime_get(xe); - + guard(xe_pm_runtime)(xe); for_each_tile(tile, xe, id) evict_test_run_tile(xe, tile, test); - xe_pm_runtime_put(xe); - return 0; } diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c index 5df98de5ba3c..954b6b911ea0 100644 --- a/drivers/gpu/drm/xe/tests/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c @@ -266,7 +266,7 @@ static int dma_buf_run_device(struct xe_device *xe) const struct dma_buf_test_params *params; struct kunit *test = kunit_get_current_test(); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); for (params = test_params; params->mem_mask; ++params) { struct dma_buf_test_params p = *params; @@ -274,7 +274,6 @@ static int dma_buf_run_device(struct xe_device *xe) test->priv = &p; xe_test_dmabuf_import_same_driver(xe); } - xe_pm_runtime_put(xe); /* A non-zero return would halt iteration over driver devices */ return 0; diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c index 5904d658d1f2..34e2f0f4631f 100644 --- a/drivers/gpu/drm/xe/tests/xe_migrate.c +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -344,8 +344,7 @@ static int migrate_test_run_device(struct xe_device *xe) struct xe_tile *tile; int id; - xe_pm_runtime_get(xe); - + guard(xe_pm_runtime)(xe); for_each_tile(tile, xe, id) { struct xe_migrate *m = tile->migrate; struct drm_exec *exec = XE_VALIDATION_OPT_OUT; @@ -356,8 +355,6 @@ static int migrate_test_run_device(struct xe_device *xe) xe_vm_unlock(m->q->vm); } - xe_pm_runtime_put(xe); - return 0; } @@ -759,13 +756,10 @@ static int validate_ccs_test_run_device(struct xe_device *xe) return 0; } - xe_pm_runtime_get(xe); - + guard(xe_pm_runtime)(xe); for_each_tile(tile, xe, id) validate_ccs_test_run_tile(xe, tile, test); - xe_pm_runtime_put(xe); - return 0; } diff --git a/drivers/gpu/drm/xe/tests/xe_mocs.c b/drivers/gpu/drm/xe/tests/xe_mocs.c index 28374330b894..daf3c6836c75 100644 --- a/drivers/gpu/drm/xe/tests/xe_mocs.c +++ b/drivers/gpu/drm/xe/tests/xe_mocs.c @@ -115,8 +115,7 @@ static int mocs_kernel_test_run_device(struct xe_device *xe) unsigned int flags; int id; - xe_pm_runtime_get(xe); - + guard(xe_pm_runtime)(xe); for_each_gt(gt, xe, id) { flags = live_mocs_init(&mocs, gt); if (flags & HAS_GLOBAL_MOCS) @@ -125,8 +124,6 @@ static int mocs_kernel_test_run_device(struct xe_device *xe) read_l3cc_table(gt, &mocs.table); } - xe_pm_runtime_put(xe); - return 0; } @@ -150,8 +147,7 @@ static int mocs_reset_test_run_device(struct xe_device *xe) int id; struct kunit *test = kunit_get_current_test(); - xe_pm_runtime_get(xe); - + guard(xe_pm_runtime)(xe); for_each_gt(gt, xe, id) { flags = live_mocs_init(&mocs, gt); kunit_info(test, "mocs_reset_test before reset\n"); @@ -169,8 +165,6 @@ static int mocs_reset_test_run_device(struct xe_device *xe) read_l3cc_table(gt, &mocs.table); } - xe_pm_runtime_put(xe); - return 0; } From f50c11ba35da84450b3206b50618ecb6482968dc Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:44:05 -0800 Subject: [PATCH 037/187] drm/xe/sysfs: Use scope-based runtime power management Switch sysfs to use scope-based runtime power management to slightly simplify the code. v2: - Drop unnecessary local variables. (Gustavo) Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-55-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_device_sysfs.c | 33 ++++++++----------- drivers/gpu/drm/xe/xe_gt_freq.c | 27 +++++---------- drivers/gpu/drm/xe/xe_gt_throttle.c | 9 ++--- drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c | 16 ++++----- 4 files changed, 31 insertions(+), 54 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_device_sysfs.c b/drivers/gpu/drm/xe/xe_device_sysfs.c index ec9c06b06fb5..a73e0e957cb0 100644 --- a/drivers/gpu/drm/xe/xe_device_sysfs.c +++ b/drivers/gpu/drm/xe/xe_device_sysfs.c @@ -57,9 +57,8 @@ vram_d3cold_threshold_store(struct device *dev, struct device_attribute *attr, drm_dbg(&xe->drm, "vram_d3cold_threshold: %u\n", vram_d3cold_threshold); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ret = xe_pm_set_vram_threshold(xe, vram_d3cold_threshold); - xe_pm_runtime_put(xe); return ret ?: count; } @@ -84,33 +83,31 @@ lb_fan_control_version_show(struct device *dev, struct device_attribute *attr, c u16 major = 0, minor = 0, hotfix = 0, build = 0; int ret; - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0), &cap, NULL); if (ret) - goto out; + return ret; if (REG_FIELD_GET(V1_FAN_PROVISIONED, cap)) { ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_LOW, 0), &ver_low, NULL); if (ret) - goto out; + return ret; ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_HIGH, 0), &ver_high, NULL); if (ret) - goto out; + return ret; major = REG_FIELD_GET(MAJOR_VERSION_MASK, ver_low); minor = REG_FIELD_GET(MINOR_VERSION_MASK, ver_low); hotfix = REG_FIELD_GET(HOTFIX_VERSION_MASK, ver_high); build = REG_FIELD_GET(BUILD_VERSION_MASK, ver_high); } -out: - xe_pm_runtime_put(xe); - return ret ?: sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build); + return sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build); } static DEVICE_ATTR_ADMIN_RO(lb_fan_control_version); @@ -123,33 +120,31 @@ lb_voltage_regulator_version_show(struct device *dev, struct device_attribute *a u16 major = 0, minor = 0, hotfix = 0, build = 0; int ret; - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0), &cap, NULL); if (ret) - goto out; + return ret; if (REG_FIELD_GET(VR_PARAMS_PROVISIONED, cap)) { ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_LOW, 0), &ver_low, NULL); if (ret) - goto out; + return ret; ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_HIGH, 0), &ver_high, NULL); if (ret) - goto out; + return ret; major = REG_FIELD_GET(MAJOR_VERSION_MASK, ver_low); minor = REG_FIELD_GET(MINOR_VERSION_MASK, ver_low); hotfix = REG_FIELD_GET(HOTFIX_VERSION_MASK, ver_high); build = REG_FIELD_GET(BUILD_VERSION_MASK, ver_high); } -out: - xe_pm_runtime_put(xe); - return ret ?: sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build); + return sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build); } static DEVICE_ATTR_ADMIN_RO(lb_voltage_regulator_version); @@ -233,9 +228,8 @@ auto_link_downgrade_capable_show(struct device *dev, struct device_attribute *at struct xe_device *xe = pdev_to_xe_device(pdev); u32 cap, val; - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); val = xe_mmio_read32(xe_root_tile_mmio(xe), BMG_PCIE_CAP); - xe_pm_runtime_put(xe); cap = REG_FIELD_GET(LINK_DOWNGRADE, val); return sysfs_emit(buf, "%u\n", cap == DOWNGRADE_CAPABLE); @@ -251,11 +245,10 @@ auto_link_downgrade_status_show(struct device *dev, struct device_attribute *att u32 val = 0; int ret; - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ret = xe_pcode_read(xe_device_get_root_tile(xe), PCODE_MBOX(DGFX_PCODE_STATUS, DGFX_GET_INIT_STATUS, 0), &val, NULL); - xe_pm_runtime_put(xe); return ret ?: sysfs_emit(buf, "%u\n", REG_FIELD_GET(DGFX_LINK_DOWNGRADE_STATUS, val)); } diff --git a/drivers/gpu/drm/xe/xe_gt_freq.c b/drivers/gpu/drm/xe/xe_gt_freq.c index 849ea6c86e8e..6284a4daf00a 100644 --- a/drivers/gpu/drm/xe/xe_gt_freq.c +++ b/drivers/gpu/drm/xe/xe_gt_freq.c @@ -70,9 +70,8 @@ static ssize_t act_freq_show(struct kobject *kobj, struct xe_guc_pc *pc = dev_to_pc(dev); u32 freq; - xe_pm_runtime_get(dev_to_xe(dev)); + guard(xe_pm_runtime)(dev_to_xe(dev)); freq = xe_guc_pc_get_act_freq(pc); - xe_pm_runtime_put(dev_to_xe(dev)); return sysfs_emit(buf, "%d\n", freq); } @@ -86,9 +85,8 @@ static ssize_t cur_freq_show(struct kobject *kobj, u32 freq; ssize_t ret; - xe_pm_runtime_get(dev_to_xe(dev)); + guard(xe_pm_runtime)(dev_to_xe(dev)); ret = xe_guc_pc_get_cur_freq(pc, &freq); - xe_pm_runtime_put(dev_to_xe(dev)); if (ret) return ret; @@ -113,9 +111,8 @@ static ssize_t rpe_freq_show(struct kobject *kobj, struct xe_guc_pc *pc = dev_to_pc(dev); u32 freq; - xe_pm_runtime_get(dev_to_xe(dev)); + guard(xe_pm_runtime)(dev_to_xe(dev)); freq = xe_guc_pc_get_rpe_freq(pc); - xe_pm_runtime_put(dev_to_xe(dev)); return sysfs_emit(buf, "%d\n", freq); } @@ -128,9 +125,8 @@ static ssize_t rpa_freq_show(struct kobject *kobj, struct xe_guc_pc *pc = dev_to_pc(dev); u32 freq; - xe_pm_runtime_get(dev_to_xe(dev)); + guard(xe_pm_runtime)(dev_to_xe(dev)); freq = xe_guc_pc_get_rpa_freq(pc); - xe_pm_runtime_put(dev_to_xe(dev)); return sysfs_emit(buf, "%d\n", freq); } @@ -154,9 +150,8 @@ static ssize_t min_freq_show(struct kobject *kobj, u32 freq; ssize_t ret; - xe_pm_runtime_get(dev_to_xe(dev)); + guard(xe_pm_runtime)(dev_to_xe(dev)); ret = xe_guc_pc_get_min_freq(pc, &freq); - xe_pm_runtime_put(dev_to_xe(dev)); if (ret) return ret; @@ -175,9 +170,8 @@ static ssize_t min_freq_store(struct kobject *kobj, if (ret) return ret; - xe_pm_runtime_get(dev_to_xe(dev)); + guard(xe_pm_runtime)(dev_to_xe(dev)); ret = xe_guc_pc_set_min_freq(pc, freq); - xe_pm_runtime_put(dev_to_xe(dev)); if (ret) return ret; @@ -193,9 +187,8 @@ static ssize_t max_freq_show(struct kobject *kobj, u32 freq; ssize_t ret; - xe_pm_runtime_get(dev_to_xe(dev)); + guard(xe_pm_runtime)(dev_to_xe(dev)); ret = xe_guc_pc_get_max_freq(pc, &freq); - xe_pm_runtime_put(dev_to_xe(dev)); if (ret) return ret; @@ -214,9 +207,8 @@ static ssize_t max_freq_store(struct kobject *kobj, if (ret) return ret; - xe_pm_runtime_get(dev_to_xe(dev)); + guard(xe_pm_runtime)(dev_to_xe(dev)); ret = xe_guc_pc_set_max_freq(pc, freq); - xe_pm_runtime_put(dev_to_xe(dev)); if (ret) return ret; @@ -243,9 +235,8 @@ static ssize_t power_profile_store(struct kobject *kobj, struct xe_guc_pc *pc = dev_to_pc(dev); int err; - xe_pm_runtime_get(dev_to_xe(dev)); + guard(xe_pm_runtime)(dev_to_xe(dev)); err = xe_guc_pc_set_power_profile(pc, buff); - xe_pm_runtime_put(dev_to_xe(dev)); return err ?: count; } diff --git a/drivers/gpu/drm/xe/xe_gt_throttle.c b/drivers/gpu/drm/xe/xe_gt_throttle.c index 82c5fbcdfbe3..096a6187ff12 100644 --- a/drivers/gpu/drm/xe/xe_gt_throttle.c +++ b/drivers/gpu/drm/xe/xe_gt_throttle.c @@ -85,7 +85,7 @@ u32 xe_gt_throttle_get_limit_reasons(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); struct xe_reg reg; - u32 val, mask; + u32 mask; if (xe_gt_is_media_type(gt)) reg = MTL_MEDIA_PERF_LIMIT_REASONS; @@ -97,11 +97,8 @@ u32 xe_gt_throttle_get_limit_reasons(struct xe_gt *gt) else mask = GT0_PERF_LIMIT_REASONS_MASK; - xe_pm_runtime_get(xe); - val = xe_mmio_read32(>->mmio, reg) & mask; - xe_pm_runtime_put(xe); - - return val; + guard(xe_pm_runtime)(xe); + return xe_mmio_read32(>->mmio, reg) & mask; } static bool is_throttled_by(struct xe_gt *gt, u32 mask) diff --git a/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c index 640950172088..cb45cdceef67 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c +++ b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c @@ -43,16 +43,14 @@ static ssize_t xe_hw_engine_class_sysfs_attr_show(struct kobject *kobj, { struct xe_device *xe = kobj_to_xe(kobj); struct kobj_attribute *kattr; - ssize_t ret = -EIO; kattr = container_of(attr, struct kobj_attribute, attr); if (kattr->show) { - xe_pm_runtime_get(xe); - ret = kattr->show(kobj, kattr, buf); - xe_pm_runtime_put(xe); + guard(xe_pm_runtime)(xe); + return kattr->show(kobj, kattr, buf); } - return ret; + return -EIO; } static ssize_t xe_hw_engine_class_sysfs_attr_store(struct kobject *kobj, @@ -62,16 +60,14 @@ static ssize_t xe_hw_engine_class_sysfs_attr_store(struct kobject *kobj, { struct xe_device *xe = kobj_to_xe(kobj); struct kobj_attribute *kattr; - ssize_t ret = -EIO; kattr = container_of(attr, struct kobj_attribute, attr); if (kattr->store) { - xe_pm_runtime_get(xe); - ret = kattr->store(kobj, kattr, buf, count); - xe_pm_runtime_put(xe); + guard(xe_pm_runtime)(xe); + return kattr->store(kobj, kattr, buf, count); } - return ret; + return -EIO; } static const struct sysfs_ops xe_hw_engine_class_sysfs_ops = { From 423fb66fac7dbb5ee2b26c40e0d21c0ffb0d7533 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 18 Nov 2025 08:44:06 -0800 Subject: [PATCH 038/187] drm/xe/debugfs: Use scope-based runtime PM Switch the debugfs code to use scope-based runtime PM where possible, for consistency with other parts of the driver. v2: - Drop unnecessary 'ret' variables. (Gustavo) Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20251118164338.3572146-56-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_debugfs.c | 16 +++++----------- drivers/gpu/drm/xe/xe_gsc_debugfs.c | 3 +-- drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c | 12 ++++-------- drivers/gpu/drm/xe/xe_guc_debugfs.c | 8 ++------ drivers/gpu/drm/xe/xe_huc_debugfs.c | 3 +-- drivers/gpu/drm/xe/xe_tile_debugfs.c | 8 ++------ 6 files changed, 15 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index e91da9589c5f..1d5a2a43a9d7 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -68,7 +68,7 @@ static int info(struct seq_file *m, void *data) struct xe_gt *gt; u8 id; - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); drm_printf(&p, "graphics_verx100 %d\n", xe->info.graphics_verx100); drm_printf(&p, "media_verx100 %d\n", xe->info.media_verx100); @@ -95,7 +95,6 @@ static int info(struct seq_file *m, void *data) gt->info.engine_mask); } - xe_pm_runtime_put(xe); return 0; } @@ -110,9 +109,8 @@ static int sriov_info(struct seq_file *m, void *data) static int workarounds(struct xe_device *xe, struct drm_printer *p) { - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); xe_wa_device_dump(xe, p); - xe_pm_runtime_put(xe); return 0; } @@ -134,7 +132,7 @@ static int dgfx_pkg_residencies_show(struct seq_file *m, void *data) xe = node_to_xe(m->private); p = drm_seq_file_printer(m); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); mmio = xe_root_tile_mmio(xe); static const struct { u32 offset; @@ -151,7 +149,6 @@ static int dgfx_pkg_residencies_show(struct seq_file *m, void *data) for (int i = 0; i < ARRAY_SIZE(residencies); i++) read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p); - xe_pm_runtime_put(xe); return 0; } @@ -163,7 +160,7 @@ static int dgfx_pcie_link_residencies_show(struct seq_file *m, void *data) xe = node_to_xe(m->private); p = drm_seq_file_printer(m); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); mmio = xe_root_tile_mmio(xe); static const struct { @@ -178,7 +175,6 @@ static int dgfx_pcie_link_residencies_show(struct seq_file *m, void *data) for (int i = 0; i < ARRAY_SIZE(residencies); i++) read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p); - xe_pm_runtime_put(xe); return 0; } @@ -277,16 +273,14 @@ static ssize_t wedged_mode_set(struct file *f, const char __user *ubuf, xe->wedged.mode = wedged_mode; - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); for_each_gt(gt, xe, id) { ret = xe_guc_ads_scheduler_policy_toggle_reset(>->uc.guc.ads); if (ret) { xe_gt_err(gt, "Failed to update GuC ADS scheduler policy. GuC may still cause engine reset even with wedged_mode=2\n"); - xe_pm_runtime_put(xe); return -EIO; } } - xe_pm_runtime_put(xe); return size; } diff --git a/drivers/gpu/drm/xe/xe_gsc_debugfs.c b/drivers/gpu/drm/xe/xe_gsc_debugfs.c index 461d7e99c2b3..b13928b50eb9 100644 --- a/drivers/gpu/drm/xe/xe_gsc_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gsc_debugfs.c @@ -37,9 +37,8 @@ static int gsc_info(struct seq_file *m, void *data) struct xe_device *xe = gsc_to_xe(gsc); struct drm_printer p = drm_seq_file_printer(m); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); xe_gsc_print_info(gsc, &p); - xe_pm_runtime_put(xe); return 0; } diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c index 5278ea4fd655..0fd863609848 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c @@ -123,11 +123,10 @@ static int POLICY##_set(void *data, u64 val) \ if (val > (TYPE)~0ull) \ return -EOVERFLOW; \ \ - xe_pm_runtime_get(xe); \ + guard(xe_pm_runtime)(xe); \ err = xe_gt_sriov_pf_policy_set_##POLICY(gt, val); \ if (!err) \ xe_sriov_pf_provision_set_custom_mode(xe); \ - xe_pm_runtime_put(xe); \ \ return err; \ } \ @@ -189,12 +188,11 @@ static int CONFIG##_set(void *data, u64 val) \ if (val > (TYPE)~0ull) \ return -EOVERFLOW; \ \ - xe_pm_runtime_get(xe); \ + guard(xe_pm_runtime)(xe); \ err = xe_sriov_pf_wait_ready(xe) ?: \ xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val); \ if (!err) \ xe_sriov_pf_provision_set_custom_mode(xe); \ - xe_pm_runtime_put(xe); \ \ return err; \ } \ @@ -249,11 +247,10 @@ static int set_threshold(void *data, u64 val, enum xe_guc_klv_threshold_index in if (val > (u32)~0ull) return -EOVERFLOW; - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); err = xe_gt_sriov_pf_config_set_threshold(gt, vfid, index, val); if (!err) xe_sriov_pf_provision_set_custom_mode(xe); - xe_pm_runtime_put(xe); return err; } @@ -358,9 +355,8 @@ static ssize_t control_write(struct file *file, const char __user *buf, size_t c xe_gt_assert(gt, sizeof(cmd) > strlen(control_cmds[n].cmd)); if (sysfs_streq(cmd, control_cmds[n].cmd)) { - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ret = control_cmds[n].fn ? (*control_cmds[n].fn)(gt, vfid) : 0; - xe_pm_runtime_put(xe); break; } } diff --git a/drivers/gpu/drm/xe/xe_guc_debugfs.c b/drivers/gpu/drm/xe/xe_guc_debugfs.c index 0b102ab46c4d..efaca259d3e8 100644 --- a/drivers/gpu/drm/xe/xe_guc_debugfs.c +++ b/drivers/gpu/drm/xe/xe_guc_debugfs.c @@ -70,13 +70,9 @@ static int guc_debugfs_show(struct seq_file *m, void *data) struct xe_gt *gt = grandparent->d_inode->i_private; struct xe_device *xe = gt_to_xe(gt); int (*print)(struct xe_guc *, struct drm_printer *) = node->info_ent->data; - int ret; - xe_pm_runtime_get(xe); - ret = print(>->uc.guc, &p); - xe_pm_runtime_put(xe); - - return ret; + guard(xe_pm_runtime)(xe); + return print(>->uc.guc, &p); } static int guc_log(struct xe_guc *guc, struct drm_printer *p) diff --git a/drivers/gpu/drm/xe/xe_huc_debugfs.c b/drivers/gpu/drm/xe/xe_huc_debugfs.c index 3a888a40188b..df9c4d79b710 100644 --- a/drivers/gpu/drm/xe/xe_huc_debugfs.c +++ b/drivers/gpu/drm/xe/xe_huc_debugfs.c @@ -37,9 +37,8 @@ static int huc_info(struct seq_file *m, void *data) struct xe_device *xe = huc_to_xe(huc); struct drm_printer p = drm_seq_file_printer(m); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); xe_huc_print_info(huc, &p); - xe_pm_runtime_put(xe); return 0; } diff --git a/drivers/gpu/drm/xe/xe_tile_debugfs.c b/drivers/gpu/drm/xe/xe_tile_debugfs.c index fff242a5ae56..39eeca75dded 100644 --- a/drivers/gpu/drm/xe/xe_tile_debugfs.c +++ b/drivers/gpu/drm/xe/xe_tile_debugfs.c @@ -82,13 +82,9 @@ int xe_tile_debugfs_show_with_rpm(struct seq_file *m, void *data) struct drm_info_node *node = m->private; struct xe_tile *tile = node_to_tile(node); struct xe_device *xe = tile_to_xe(tile); - int ret; - xe_pm_runtime_get(xe); - ret = xe_tile_debugfs_simple_show(m, data); - xe_pm_runtime_put(xe); - - return ret; + guard(xe_pm_runtime)(xe); + return xe_tile_debugfs_simple_show(m, data); } static int ggtt(struct xe_tile *tile, struct drm_printer *p) From 28aeaed130e8e587fd1b73b6d66ca41ccc5a1a31 Mon Sep 17 00:00:00 2001 From: Sanjay Yadav Date: Tue, 18 Nov 2025 17:19:00 +0530 Subject: [PATCH 039/187] drm/xe/oa: Fix potential UAF in xe_oa_add_config_ioctl() In xe_oa_add_config_ioctl(), we accessed oa_config->id after dropping metrics_lock. Since this lock protects the lifetime of oa_config, an attacker could guess the id and call xe_oa_remove_config_ioctl() with perfect timing, freeing oa_config before we dereference it, leading to a potential use-after-free. Fix this by caching the id in a local variable while holding the lock. v2: (Matt A) - Dropped mutex_unlock(&oa->metrics_lock) ordering change from xe_oa_remove_config_ioctl() Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/6614 Fixes: cdf02fe1a94a7 ("drm/xe/oa/uapi: Add/remove OA config perf ops") Cc: # v6.11+ Suggested-by: Matthew Auld Signed-off-by: Sanjay Yadav Reviewed-by: Matthew Auld Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20251118114859.3379952-2-sanjay.kumar.yadav@intel.com --- drivers/gpu/drm/xe/xe_oa.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 87a2bf53d661..890c363282ae 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -2403,11 +2403,13 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi goto sysfs_err; } + id = oa_config->id; + + drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, id); + mutex_unlock(&oa->metrics_lock); - drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, oa_config->id); - - return oa_config->id; + return id; sysfs_err: mutex_unlock(&oa->metrics_lock); From 62519b77aecad22b525eda482660ffa127e7ad80 Mon Sep 17 00:00:00 2001 From: Mika Kuoppala Date: Thu, 20 Nov 2025 18:14:35 +0200 Subject: [PATCH 040/187] drm/xe: Fix memory leak when handling pagefault vma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the pagefault handling code was moved to a new file, an extra drm_exec_init() was added to the VMA path. This call is unnecessary because xe_validation_ctx_init() already performs a drm_exec_init(), resulting in a memory leak reported by kmemleak. Remove the redundant drm_exec_init() from the VMA pagefault handling code. Fixes: fb544b844508 ("drm/xe: Implement xe_pagefault_queue_work") Cc: Matthew Brost Cc: Stuart Summers Cc: Lucas De Marchi Cc: "Thomas Hellström" Cc: Rodrigo Vivi Cc: Sumit Semwal Cc: "Christian König" Cc: intel-xe@lists.freedesktop.org Cc: linux-media@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Cc: linaro-mm-sig@lists.linaro.org Signed-off-by: Mika Kuoppala Reviewed-by: Thomas Hellström Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251120161435.3674556-1-mika.kuoppala@linux.intel.com --- drivers/gpu/drm/xe/xe_pagefault.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index fe3e40145012..afb06598b6e1 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -102,7 +102,6 @@ static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma, /* Lock VM and BOs dma-resv */ xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, (struct xe_val_flags) {}); - drm_exec_init(&exec, 0, 0); drm_exec_until_all_locked(&exec) { err = xe_pagefault_begin(&exec, vma, tile->mem.vram, needs_vram == 1); From 2e4ad5b0667244f496783c58de0995b9562d3344 Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Mon, 10 Nov 2025 18:45:23 +0000 Subject: [PATCH 041/187] drm/xe/guc: Fix resource leak in xe_guc_ct_init_noalloc() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xe_guc_ct_init_noalloc() allocates the CT workqueue and other helpers before it tries to initialize ct->lock. If drmm_mutex_init() fails we currently bail out without releasing those resources because the guc_ct_fini() hasn’t been registered yet. Since destroy_workqueue() in guc_ct_fini() may flush the workqueue, which in turn can take the ct lock, the initialization sequence is restructured to first initialize the ct->lock, then set up all CT state, and finally register guc_ct_fini(). v2: guc_ct_fini() does take ct lock. (Matt) v3: move primelockdep() together with drmm_mutex_init(). (Lucas) Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Cc: Lucas De Marchi Cc: Matthew Brost Signed-off-by: Shuicheng Lin Reviewed-by: Lucas De Marchi Link: https://patch.msgid.link/20251110184522.1581001-2-shuicheng.lin@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_guc_ct.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 2697d711adb2..ef46c272aeb2 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -225,6 +225,12 @@ int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE)); + err = drmm_mutex_init(&xe->drm, &ct->lock); + if (err) + return err; + + primelockdep(ct); + ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM); if (!ct->g2h_wq) return -ENOMEM; @@ -240,12 +246,6 @@ int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) init_waitqueue_head(&ct->wq); init_waitqueue_head(&ct->g2h_fence_wq); - err = drmm_mutex_init(&xe->drm, &ct->lock); - if (err) - return err; - - primelockdep(ct); - err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); if (err) return err; From 64fdf496a6929a0a194387d2bb5efaf5da2b542f Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Tue, 18 Nov 2025 11:08:11 -0800 Subject: [PATCH 042/187] drm/xe/guc: Fix stack_depot usage Add missing stack_depot_init() call when CONFIG_DRM_XE_DEBUG_GUC is enabled to fix the following call stack: [] BUG: kernel NULL pointer dereference, address: 0000000000000000 [] Workqueue: drm_sched_run_job_work [gpu_sched] [] RIP: 0010:stack_depot_save_flags+0x172/0x870 [] Call Trace: [] [] fast_req_track+0x58/0xb0 [xe] Fixes: 16b7e65d299d ("drm/xe/guc: Track FAST_REQ H2Gs to report where errors came from") Tested-by: Sagar Ghuge Cc: stable@vger.kernel.org # v6.17+ Reviewed-by: Stuart Summers Link: https://patch.msgid.link/20251118-fix-debug-guc-v1-1-9f780c6bedf8@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_guc_ct.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index ef46c272aeb2..4ac434ad216f 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -242,6 +242,9 @@ int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) spin_lock_init(&ct->dead.lock); INIT_WORK(&ct->dead.worker, ct_dead_worker_func); +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) + stack_depot_init(); +#endif #endif init_waitqueue_head(&ct->wq); init_waitqueue_head(&ct->g2h_fence_wq); From ea944d57eac746f9bd9056134751708c48084207 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Wed, 19 Nov 2025 07:21:57 -0800 Subject: [PATCH 043/187] drm/xe/guc_ct: Cleanup ifdef'ry Better split CONFIG_DRM_XE_DEBUG and CONFIG_DRM_XE_DEBUG_GUC optional parts from the main code, creating smaller ct_dead_* and fast_req_* interfaces. Reviewed-by: Daniele Ceraolo Spurio Link: https://patch.msgid.link/20251119152157.1675188-2-lucas.demarchi@intel.com Signed-off-by: Lucas De Marchi --- drivers/gpu/drm/xe/xe_guc_ct.c | 206 +++++++++++++++++---------------- 1 file changed, 106 insertions(+), 100 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 4ac434ad216f..02b4588fece0 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -42,6 +42,21 @@ static void ct_exit_safe_mode(struct xe_guc_ct *ct); static void guc_ct_change_state(struct xe_guc_ct *ct, enum xe_guc_ct_state state); +static struct xe_guc *ct_to_guc(struct xe_guc_ct *ct) +{ + return container_of(ct, struct xe_guc, ct); +} + +static struct xe_gt *ct_to_gt(struct xe_guc_ct *ct) +{ + return container_of(ct, struct xe_gt, uc.guc.ct); +} + +static struct xe_device *ct_to_xe(struct xe_guc_ct *ct) +{ + return gt_to_xe(ct_to_gt(ct)); +} + #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) enum { /* Internal states, not error conditions */ @@ -68,14 +83,101 @@ enum { static void ct_dead_worker_func(struct work_struct *w); static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code); -#define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code) +static void ct_dead_fini(struct xe_guc_ct *ct) +{ + cancel_work_sync(&ct->dead.worker); +} + +static void ct_dead_init(struct xe_guc_ct *ct) +{ + spin_lock_init(&ct->dead.lock); + INIT_WORK(&ct->dead.worker, ct_dead_worker_func); + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) + stack_depot_init(); +#endif +} + +static void fast_req_stack_save(struct xe_guc_ct *ct, unsigned int slot) +{ +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) + unsigned long entries[SZ_32]; + unsigned int n; + + n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); + /* May be called under spinlock, so avoid sleeping */ + ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT); +#endif +} + +static void fast_req_dump(struct xe_guc_ct *ct, u16 fence, unsigned int slot) +{ + struct xe_gt *gt = ct_to_gt(ct); +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) + char *buf __cleanup(kfree) = kmalloc(SZ_4K, GFP_NOWAIT); + + if (buf && stack_depot_snprint(ct->fast_req[slot].stack, buf, SZ_4K, 0)) + xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s\n", + fence, ct->fast_req[slot].action, buf); + else + xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n", + fence, ct->fast_req[slot].action); #else + xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n", + fence, ct->fast_req[slot].action); +#endif +} + +static void fast_req_report(struct xe_guc_ct *ct, u16 fence) +{ + u16 fence_min = U16_MAX, fence_max = 0; + struct xe_gt *gt = ct_to_gt(ct); + unsigned int n; + + lockdep_assert_held(&ct->lock); + + for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) { + if (ct->fast_req[n].fence < fence_min) + fence_min = ct->fast_req[n].fence; + if (ct->fast_req[n].fence > fence_max) + fence_max = ct->fast_req[n].fence; + + if (ct->fast_req[n].fence != fence) + continue; + + return fast_req_dump(ct, fence, n); + } + + xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n", + fence, fence_min, fence_max, ct->fence_seqno); +} + +static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) +{ + unsigned int slot = fence % ARRAY_SIZE(ct->fast_req); + + fast_req_stack_save(ct, slot); + ct->fast_req[slot].fence = fence; + ct->fast_req[slot].action = action; +} + +#define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code) + +#else + +static void ct_dead_fini(struct xe_guc_ct *ct) { } +static void ct_dead_init(struct xe_guc_ct *ct) { } + +static void fast_req_report(struct xe_guc_ct *ct, u16 fence) { } +static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { } + #define CT_DEAD(ct, ctb, reason) \ do { \ struct guc_ctb *_ctb = (ctb); \ if (_ctb) \ _ctb->info.broken = true; \ } while (0) + #endif /* Used when a CT send wants to block and / or receive data */ @@ -112,24 +214,6 @@ static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) return g2h_fence->seqno == ~0x0; } -static struct xe_guc * -ct_to_guc(struct xe_guc_ct *ct) -{ - return container_of(ct, struct xe_guc, ct); -} - -static struct xe_gt * -ct_to_gt(struct xe_guc_ct *ct) -{ - return container_of(ct, struct xe_gt, uc.guc.ct); -} - -static struct xe_device * -ct_to_xe(struct xe_guc_ct *ct) -{ - return gt_to_xe(ct_to_gt(ct)); -} - /** * DOC: GuC CTB Blob * @@ -199,9 +283,7 @@ static void guc_ct_fini(struct drm_device *drm, void *arg) { struct xe_guc_ct *ct = arg; -#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) - cancel_work_sync(&ct->dead.worker); -#endif + ct_dead_fini(ct); ct_exit_safe_mode(ct); destroy_workqueue(ct->g2h_wq); xa_destroy(&ct->fence_lookup); @@ -239,13 +321,8 @@ int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) xa_init(&ct->fence_lookup); INIT_WORK(&ct->g2h_worker, g2h_worker_func); INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func); -#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) - spin_lock_init(&ct->dead.lock); - INIT_WORK(&ct->dead.worker, ct_dead_worker_func); -#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) - stack_depot_init(); -#endif -#endif + + ct_dead_init(ct); init_waitqueue_head(&ct->wq); init_waitqueue_head(&ct->g2h_fence_wq); @@ -747,28 +824,6 @@ static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) spin_unlock_irq(&ct->fast_lock); } -#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) -static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) -{ - unsigned int slot = fence % ARRAY_SIZE(ct->fast_req); -#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) - unsigned long entries[SZ_32]; - unsigned int n; - - n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); - - /* May be called under spinlock, so avoid sleeping */ - ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT); -#endif - ct->fast_req[slot].fence = fence; - ct->fast_req[slot].action = action; -} -#else -static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) -{ -} -#endif - /* * The CT protocol accepts a 16 bits fence. This field is fully owned by the * driver, the GuC will just copy it to the reply message. Since we need to @@ -1338,55 +1393,6 @@ static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) return 0; } -#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) -static void fast_req_report(struct xe_guc_ct *ct, u16 fence) -{ - u16 fence_min = U16_MAX, fence_max = 0; - struct xe_gt *gt = ct_to_gt(ct); - bool found = false; - unsigned int n; -#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) - char *buf; -#endif - - lockdep_assert_held(&ct->lock); - - for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) { - if (ct->fast_req[n].fence < fence_min) - fence_min = ct->fast_req[n].fence; - if (ct->fast_req[n].fence > fence_max) - fence_max = ct->fast_req[n].fence; - - if (ct->fast_req[n].fence != fence) - continue; - found = true; - -#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) - buf = kmalloc(SZ_4K, GFP_NOWAIT); - if (buf && stack_depot_snprint(ct->fast_req[n].stack, buf, SZ_4K, 0)) - xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s", - fence, ct->fast_req[n].action, buf); - else - xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n", - fence, ct->fast_req[n].action); - kfree(buf); -#else - xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n", - fence, ct->fast_req[n].action); -#endif - break; - } - - if (!found) - xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n", - fence, fence_min, fence_max, ct->fence_seqno); -} -#else -static void fast_req_report(struct xe_guc_ct *ct, u16 fence) -{ -} -#endif - static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) { struct xe_gt *gt = ct_to_gt(ct); From 96b93ac214f9dd66294d975d86c5dee256faef91 Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Mon, 17 Nov 2025 11:48:43 -0800 Subject: [PATCH 044/187] drm/xe: Fix conversion from clock ticks to milliseconds When tick counts are large and multiplication by MSEC_PER_SEC is larger than 64 bits, the conversion from clock ticks to milliseconds can go bad. Use mul_u64_u32_div() instead. Cc: Ashutosh Dixit Signed-off-by: Harish Chegondi Suggested-by: Umesh Nerlige Ramappa Fixes: 49cc215aad7f ("drm/xe: Add xe_gt_clock_interval_to_ms helper") Reviewed-by: Ashutosh Dixit Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/1562f1b62d5be3fbaee100f09107f3cc49e40dd1.1763408584.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/xe_gt_clock.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_clock.c b/drivers/gpu/drm/xe/xe_gt_clock.c index 00f5972c14dc..bfc25c46f798 100644 --- a/drivers/gpu/drm/xe/xe_gt_clock.c +++ b/drivers/gpu/drm/xe/xe_gt_clock.c @@ -74,11 +74,6 @@ int xe_gt_clock_init(struct xe_gt *gt) return 0; } -static u64 div_u64_roundup(u64 n, u32 d) -{ - return div_u64(n + d - 1, d); -} - /** * xe_gt_clock_interval_to_ms - Convert sampled GT clock ticks to msec * @@ -89,5 +84,5 @@ static u64 div_u64_roundup(u64 n, u32 d) */ u64 xe_gt_clock_interval_to_ms(struct xe_gt *gt, u64 count) { - return div_u64_roundup(count * MSEC_PER_SEC, gt->info.reference_clock); + return mul_u64_u32_div(count, MSEC_PER_SEC, gt->info.reference_clock); } From 0f4435a1f46efc3177eb082cd3f73e29da5ab86a Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Sat, 15 Nov 2025 16:13:22 +0100 Subject: [PATCH 045/187] drm/xe/pf: Use div_u64 when calculating GGTT profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will fix the following error seen on some 32-bit config: "ERROR: modpost: "__udivdi3" [drivers/gpu/drm/xe/xe.ko] undefined!" Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202511150929.3vUi6PEJ-lkp@intel.com/ Fixes: e448372e8a8e ("drm/xe/pf: Use migration-friendly GGTT auto-provisioning") Signed-off-by: Michal Wajdeczko Reviewed-by: Piotr Piórkowski Link: https://patch.msgid.link/20251115151323.10828-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index 62f6cc45a764..59c5c6b4d994 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -711,7 +711,7 @@ static u64 pf_profile_fair_ggtt(struct xe_gt *gt, unsigned int num_vfs) if (num_vfs > 56) return SZ_64M - SZ_8M; - return rounddown_pow_of_two(shareable / num_vfs); + return rounddown_pow_of_two(div_u64(shareable, num_vfs)); } /** From 5a062505aa0ed5f9124c22f07da6ba58950475b2 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Sat, 15 Nov 2025 16:26:58 +0100 Subject: [PATCH 046/187] drm/xe/pf: Fix .bulk_profile/sched_priority description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .bulk_profile/sched_priority file is always write-only, unlike the profile/sched_priority files which can be either read-write or read-only (in case of PF or VFs respectively). Fixes: 6b514ed2d9a7 ("drm/xe/pf: Add documentation for sriov_admin attributes") Signed-off-by: Michal Wajdeczko Reviewed-by: Piotr Piórkowski Link: https://patch.msgid.link/20251115152659.10853-1-michal.wajdeczko@intel.com --- Documentation/ABI/testing/sysfs-driver-intel-xe-sriov | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov b/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov index 2fd7e9b7bacc..7f5ef9eada53 100644 --- a/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov +++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov @@ -119,7 +119,7 @@ Description: The GT preemption timeout (PT) in [us] to be applied to all functions. See sriov_admin/{pf,vf}/profile/preempt_timeout_us for more details. - sched_priority: (RW/RO) string + sched_priority: (WO) string The GT scheduling priority to be applied for all functions. See sriov_admin/{pf,vf}/profile/sched_priority for more details. From f4c8298cf5f5c2458f700546456601f3ca9b282b Mon Sep 17 00:00:00 2001 From: Zhanjun Dong Date: Wed, 5 Nov 2025 18:31:43 -0500 Subject: [PATCH 047/187] drm/xe/guc: Cleanup GuC log buffer macros and helpers Cleanup GuC log buffer macros and helpers, add Xe style macro prefix. Update buffer type values to align with the GuC specification Update buffer offset calculation. Remove helper functions, replaced with macros. Signed-off-by: Zhanjun Dong Reviewed-by: Michal Wajdeczko Signed-off-by: Daniele Ceraolo Spurio Link: https://patch.msgid.link/20251105233143.1168759-1-zhanjun.dong@intel.com --- drivers/gpu/drm/xe/abi/guc_log_abi.h | 42 +++++++++-- drivers/gpu/drm/xe/xe_guc.c | 23 +++--- drivers/gpu/drm/xe/xe_guc_capture.c | 16 ++--- drivers/gpu/drm/xe/xe_guc_fwif.h | 6 +- drivers/gpu/drm/xe/xe_guc_log.c | 100 ++------------------------- drivers/gpu/drm/xe/xe_guc_log.h | 29 +++++--- 6 files changed, 85 insertions(+), 131 deletions(-) diff --git a/drivers/gpu/drm/xe/abi/guc_log_abi.h b/drivers/gpu/drm/xe/abi/guc_log_abi.h index 554630b7ccd9..fbf212d59a40 100644 --- a/drivers/gpu/drm/xe/abi/guc_log_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_log_abi.h @@ -8,11 +8,45 @@ #include +/** + * DOC: GuC Log buffer Layout + * + * The in-memory log buffer layout is as follows:: + * + * +===============================+ 0000h + * | Crash dump state header | ^ + * +-------------------------------+ 32B | + * | Debug state header | | + * +-------------------------------+ 64B 4KB + * | Capture state header | | + * +-------------------------------+ 96B | + * | | v + * +===============================+ <--- EVENT_DATA_OFFSET + * | Event logs(raw data) | ^ + * | | | + * | | EVENT_DATA_BUFFER_SIZE + * | | | + * | | v + * +===============================+ <--- CRASH_DUMP_OFFSET + * | Crash Dump(raw data) | ^ + * | | | + * | | CRASH_DUMP_BUFFER_SIZE + * | | | + * | | v + * +===============================+ <--- STATE_CAPTURE_OFFSET + * | Error state capture(raw data) | ^ + * | | | + * | | STATE_CAPTURE_BUFFER_SIZE + * | | | + * | | v + * +===============================+ Total: GUC_LOG_SIZE + */ + /* GuC logging buffer types */ -enum guc_log_buffer_type { - GUC_LOG_BUFFER_CRASH_DUMP, - GUC_LOG_BUFFER_DEBUG, - GUC_LOG_BUFFER_CAPTURE, +enum guc_log_type { + GUC_LOG_TYPE_EVENT_DATA, + GUC_LOG_TYPE_CRASH_DUMP, + GUC_LOG_TYPE_STATE_CAPTURE, }; #define GUC_LOG_BUFFER_TYPE_MAX 3 diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index cf92de1c88a7..d6672cf30d3e 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -104,7 +104,7 @@ static u32 guc_ctl_log_params_flags(struct xe_guc *guc) u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT; u32 flags; - #if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0) + #if (((XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE) % SZ_1M) == 0) #define LOG_UNIT SZ_1M #define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS #else @@ -112,7 +112,7 @@ static u32 guc_ctl_log_params_flags(struct xe_guc *guc) #define LOG_FLAG 0 #endif - #if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0) + #if (((XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE) % SZ_1M) == 0) #define CAPTURE_UNIT SZ_1M #define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS #else @@ -120,20 +120,21 @@ static u32 guc_ctl_log_params_flags(struct xe_guc *guc) #define CAPTURE_FLAG 0 #endif - BUILD_BUG_ON(!CRASH_BUFFER_SIZE); - BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT)); - BUILD_BUG_ON(!DEBUG_BUFFER_SIZE); - BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT)); - BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE); - BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT)); + BUILD_BUG_ON(!XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE, LOG_UNIT)); + BUILD_BUG_ON(!XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE, LOG_UNIT)); + BUILD_BUG_ON(!XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE, CAPTURE_UNIT)); flags = GUC_LOG_VALID | GUC_LOG_NOTIFY_ON_HALF_FULL | CAPTURE_FLAG | LOG_FLAG | - FIELD_PREP(GUC_LOG_CRASH, CRASH_BUFFER_SIZE / LOG_UNIT - 1) | - FIELD_PREP(GUC_LOG_DEBUG, DEBUG_BUFFER_SIZE / LOG_UNIT - 1) | - FIELD_PREP(GUC_LOG_CAPTURE, CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) | + FIELD_PREP(GUC_LOG_CRASH_DUMP, XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE / LOG_UNIT - 1) | + FIELD_PREP(GUC_LOG_EVENT_DATA, XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE / LOG_UNIT - 1) | + FIELD_PREP(GUC_LOG_STATE_CAPTURE, XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE / + CAPTURE_UNIT - 1) | FIELD_PREP(GUC_LOG_BUF_ADDR, offset); #undef LOG_UNIT diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c index 0c1fbe97b8bf..2cda92f7b323 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.c +++ b/drivers/gpu/drm/xe/xe_guc_capture.c @@ -843,7 +843,7 @@ static void check_guc_capture_size(struct xe_guc *guc) { int capture_size = guc_capture_output_size_est(guc); int spare_size = capture_size * GUC_CAPTURE_OVERBUFFER_MULTIPLIER; - u32 buffer_size = xe_guc_log_section_size_capture(&guc->log); + u32 buffer_size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE; /* * NOTE: capture_size is much smaller than the capture region @@ -949,7 +949,7 @@ guc_capture_init_node(struct xe_guc *guc, struct __guc_capture_parsed_output *no * ADS module also calls separately for PF vs VF. * * --> alloc B: GuC output capture buf (registered via guc_init_params(log_param)) - * Size = #define CAPTURE_BUFFER_SIZE (warns if on too-small) + * Size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE (warns if on too-small) * Note2: 'x 3' to hold multiple capture groups * * GUC Runtime notify capture: @@ -1367,7 +1367,7 @@ static int __guc_capture_flushlog_complete(struct xe_guc *guc) { u32 action[] = { XE_GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE, - GUC_LOG_BUFFER_CAPTURE + GUC_LOG_TYPE_STATE_CAPTURE }; return xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); @@ -1384,8 +1384,8 @@ static void __guc_capture_process_output(struct xe_guc *guc) u32 log_buf_state_offset; u32 src_data_offset; - log_buf_state_offset = sizeof(struct guc_log_buffer_state) * GUC_LOG_BUFFER_CAPTURE; - src_data_offset = xe_guc_get_log_buffer_offset(&guc->log, GUC_LOG_BUFFER_CAPTURE); + log_buf_state_offset = sizeof(struct guc_log_buffer_state) * GUC_LOG_TYPE_STATE_CAPTURE; + src_data_offset = XE_GUC_LOG_STATE_CAPTURE_OFFSET; /* * Make a copy of the state structure, inside GuC log buffer @@ -1395,15 +1395,15 @@ static void __guc_capture_process_output(struct xe_guc *guc) xe_map_memcpy_from(guc_to_xe(guc), &log_buf_state_local, &guc->log.bo->vmap, log_buf_state_offset, sizeof(struct guc_log_buffer_state)); - buffer_size = xe_guc_get_log_buffer_size(&guc->log, GUC_LOG_BUFFER_CAPTURE); + buffer_size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE; read_offset = log_buf_state_local.read_ptr; write_offset = log_buf_state_local.sampled_write_ptr; full_count = FIELD_GET(GUC_LOG_BUFFER_STATE_BUFFER_FULL_CNT, log_buf_state_local.flags); /* Bookkeeping stuff */ tmp = FIELD_GET(GUC_LOG_BUFFER_STATE_FLUSH_TO_FILE, log_buf_state_local.flags); - guc->log.stats[GUC_LOG_BUFFER_CAPTURE].flush += tmp; - new_overflow = xe_guc_check_log_buf_overflow(&guc->log, GUC_LOG_BUFFER_CAPTURE, + guc->log.stats[GUC_LOG_TYPE_STATE_CAPTURE].flush += tmp; + new_overflow = xe_guc_check_log_buf_overflow(&guc->log, GUC_LOG_TYPE_STATE_CAPTURE, full_count); /* Now copy the actual logs. */ diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h index c90dd266e9cf..7d93c2749485 100644 --- a/drivers/gpu/drm/xe/xe_guc_fwif.h +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -91,9 +91,9 @@ struct guc_update_exec_queue_policy { #define GUC_LOG_NOTIFY_ON_HALF_FULL BIT(1) #define GUC_LOG_CAPTURE_ALLOC_UNITS BIT(2) #define GUC_LOG_LOG_ALLOC_UNITS BIT(3) -#define GUC_LOG_CRASH REG_GENMASK(5, 4) -#define GUC_LOG_DEBUG REG_GENMASK(9, 6) -#define GUC_LOG_CAPTURE REG_GENMASK(11, 10) +#define GUC_LOG_CRASH_DUMP REG_GENMASK(5, 4) +#define GUC_LOG_EVENT_DATA REG_GENMASK(9, 6) +#define GUC_LOG_STATE_CAPTURE REG_GENMASK(11, 10) #define GUC_LOG_BUF_ADDR REG_GENMASK(31, 12) #define GUC_CTL_WA 1 diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c index 0c704a11078a..086cd60628f3 100644 --- a/drivers/gpu/drm/xe/xe_guc_log.c +++ b/drivers/gpu/drm/xe/xe_guc_log.c @@ -19,6 +19,8 @@ #include "xe_mmio.h" #include "xe_module.h" +#define GUC_LOG_CHUNK_SIZE SZ_2M + static struct xe_guc * log_to_guc(struct xe_guc_log *log) { @@ -37,33 +39,6 @@ log_to_xe(struct xe_guc_log *log) return gt_to_xe(log_to_gt(log)); } -static size_t guc_log_size(void) -{ - /* - * GuC Log buffer Layout - * - * +===============================+ 00B - * | Crash dump state header | - * +-------------------------------+ 32B - * | Debug state header | - * +-------------------------------+ 64B - * | Capture state header | - * +-------------------------------+ 96B - * | | - * +===============================+ PAGE_SIZE (4KB) - * | Crash Dump logs | - * +===============================+ + CRASH_SIZE - * | Debug logs | - * +===============================+ + DEBUG_SIZE - * | Capture logs | - * +===============================+ + CAPTURE_SIZE - */ - return PAGE_SIZE + CRASH_BUFFER_SIZE + DEBUG_BUFFER_SIZE + - CAPTURE_BUFFER_SIZE; -} - -#define GUC_LOG_CHUNK_SIZE SZ_2M - static struct xe_guc_log_snapshot *xe_guc_log_snapshot_alloc(struct xe_guc_log *log, bool atomic) { struct xe_guc_log_snapshot *snapshot; @@ -255,7 +230,7 @@ int xe_guc_log_init(struct xe_guc_log *log) struct xe_tile *tile = gt_to_tile(log_to_gt(log)); struct xe_bo *bo; - bo = xe_managed_bo_create_pin_map(xe, tile, guc_log_size(), + bo = xe_managed_bo_create_pin_map(xe, tile, GUC_LOG_SIZE, XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE | @@ -263,7 +238,7 @@ int xe_guc_log_init(struct xe_guc_log *log) if (IS_ERR(bo)) return PTR_ERR(bo); - xe_map_memset(xe, &bo->vmap, 0, 0, guc_log_size()); + xe_map_memset(xe, &bo->vmap, 0, 0, xe_bo_size(bo)); log->bo = bo; log->level = xe_modparam.guc_log_level; @@ -272,71 +247,6 @@ int xe_guc_log_init(struct xe_guc_log *log) ALLOW_ERROR_INJECTION(xe_guc_log_init, ERRNO); /* See xe_pci_probe() */ -static u32 xe_guc_log_section_size_crash(struct xe_guc_log *log) -{ - return CRASH_BUFFER_SIZE; -} - -static u32 xe_guc_log_section_size_debug(struct xe_guc_log *log) -{ - return DEBUG_BUFFER_SIZE; -} - -/** - * xe_guc_log_section_size_capture - Get capture buffer size within log sections. - * @log: The log object. - * - * This function will return the capture buffer size within log sections. - * - * Return: capture buffer size. - */ -u32 xe_guc_log_section_size_capture(struct xe_guc_log *log) -{ - return CAPTURE_BUFFER_SIZE; -} - -/** - * xe_guc_get_log_buffer_size - Get log buffer size for a type. - * @log: The log object. - * @type: The log buffer type - * - * Return: buffer size. - */ -u32 xe_guc_get_log_buffer_size(struct xe_guc_log *log, enum guc_log_buffer_type type) -{ - switch (type) { - case GUC_LOG_BUFFER_CRASH_DUMP: - return xe_guc_log_section_size_crash(log); - case GUC_LOG_BUFFER_DEBUG: - return xe_guc_log_section_size_debug(log); - case GUC_LOG_BUFFER_CAPTURE: - return xe_guc_log_section_size_capture(log); - } - return 0; -} - -/** - * xe_guc_get_log_buffer_offset - Get offset in log buffer for a type. - * @log: The log object. - * @type: The log buffer type - * - * This function will return the offset in the log buffer for a type. - * Return: buffer offset. - */ -u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_type type) -{ - enum guc_log_buffer_type i; - u32 offset = PAGE_SIZE;/* for the log_buffer_states */ - - for (i = GUC_LOG_BUFFER_CRASH_DUMP; i < GUC_LOG_BUFFER_TYPE_MAX; ++i) { - if (i == type) - break; - offset += xe_guc_get_log_buffer_size(log, i); - } - - return offset; -} - /** * xe_guc_check_log_buf_overflow - Check if log buffer overflowed * @log: The log object. @@ -350,7 +260,7 @@ u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_typ * * Return: True if overflowed. */ -bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, enum guc_log_buffer_type type, +bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, enum guc_log_type type, unsigned int full_cnt) { unsigned int prev_full_cnt = log->stats[type].sampled_overflow; diff --git a/drivers/gpu/drm/xe/xe_guc_log.h b/drivers/gpu/drm/xe/xe_guc_log.h index 98a47ac42b08..0bd5e89d75e0 100644 --- a/drivers/gpu/drm/xe/xe_guc_log.h +++ b/drivers/gpu/drm/xe/xe_guc_log.h @@ -13,14 +13,26 @@ struct drm_printer; struct xe_device; #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) -#define CRASH_BUFFER_SIZE SZ_1M -#define DEBUG_BUFFER_SIZE SZ_8M -#define CAPTURE_BUFFER_SIZE SZ_2M +#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE SZ_8M +#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE SZ_1M +#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE SZ_2M #else -#define CRASH_BUFFER_SIZE SZ_16K -#define DEBUG_BUFFER_SIZE SZ_64K -#define CAPTURE_BUFFER_SIZE SZ_1M +#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE SZ_64K +#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE SZ_16K +#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE SZ_1M #endif + +#define GUC_LOG_SIZE (SZ_4K + \ + XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE + \ + XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE + \ + XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE) + +#define XE_GUC_LOG_EVENT_DATA_OFFSET SZ_4K +#define XE_GUC_LOG_CRASH_DUMP_OFFSET (XE_GUC_LOG_EVENT_DATA_OFFSET + \ + XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE) +#define XE_GUC_LOG_STATE_CAPTURE_OFFSET (XE_GUC_LOG_CRASH_DUMP_OFFSET + \ + XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE) + /* * While we're using plain log level in i915, GuC controls are much more... * "elaborate"? We have a couple of bits for verbosity, separate bit for actual @@ -51,11 +63,8 @@ xe_guc_log_get_level(struct xe_guc_log *log) return log->level; } -u32 xe_guc_log_section_size_capture(struct xe_guc_log *log); -u32 xe_guc_get_log_buffer_size(struct xe_guc_log *log, enum guc_log_buffer_type type); -u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_type type); bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, - enum guc_log_buffer_type type, + enum guc_log_type type, unsigned int full_cnt); #endif From 69ef30d6c903c4b9cd75c50367acb1cd1f95d440 Mon Sep 17 00:00:00 2001 From: Zhanjun Dong Date: Mon, 27 Oct 2025 17:42:12 -0400 Subject: [PATCH 048/187] drm/xe/uc: Change assertion to error on huc authentication failure The fault injection test can cause the xe_huc_auth function to fail. This is an intentional failure, so in this scenario we don't want to throw an assert and taint the kernel, because that will impact CI execution. Signed-off-by: Zhanjun Dong Reviewed-by: Daniele Ceraolo Spurio Signed-off-by: Daniele Ceraolo Spurio Link: https://patch.msgid.link/20251027214212.2856903-1-zhanjun.dong@intel.com --- drivers/gpu/drm/xe/xe_uc.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c index 465bda355443..40aed4a66bac 100644 --- a/drivers/gpu/drm/xe/xe_uc.c +++ b/drivers/gpu/drm/xe/xe_uc.c @@ -218,9 +218,12 @@ int xe_uc_load_hw(struct xe_uc *uc) xe_guc_engine_activity_enable_stats(&uc->guc); - /* We don't fail the driver load if HuC fails to auth, but let's warn */ + /* We don't fail the driver load if HuC fails to auth */ ret = xe_huc_auth(&uc->huc, XE_HUC_AUTH_VIA_GUC); - xe_gt_assert(uc_to_gt(uc), !ret); + if (ret) + xe_gt_err(uc_to_gt(uc), + "HuC authentication failed (%pe), continuing with no HuC\n", + ERR_PTR(ret)); /* GSC load is async */ xe_gsc_load_start(&uc->gsc); From 79cb005c7134340d78e2fa8e526a6104fe492695 Mon Sep 17 00:00:00 2001 From: Lukasz Laguna Date: Mon, 24 Nov 2025 20:02:34 +0100 Subject: [PATCH 049/187] drm/xe: Add device flag to indicate standalone MERT The MERT subsystem manages memory accesses between host and device. On the Crescent Island platform, it requires direct management by the driver. Introduce a device flag and corresponding helpers to identify platforms with standalone MERT, enabling proper initialization and handling. Signed-off-by: Lukasz Laguna Reviewed-by: Ashutosh Dixit Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251124190237.20503-2-lukasz.laguna@intel.com --- drivers/gpu/drm/xe/xe_device.h | 5 +++++ drivers/gpu/drm/xe/xe_device_types.h | 2 ++ drivers/gpu/drm/xe/xe_pci.c | 2 ++ drivers/gpu/drm/xe/xe_pci_types.h | 1 + 4 files changed, 10 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h index 32cc6323b7f6..6604b89330d5 100644 --- a/drivers/gpu/drm/xe/xe_device.h +++ b/drivers/gpu/drm/xe/xe_device.h @@ -172,6 +172,11 @@ static inline bool xe_device_has_lmtt(struct xe_device *xe) return IS_DGFX(xe); } +static inline bool xe_device_has_mert(struct xe_device *xe) +{ + return xe->info.has_mert; +} + u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size); void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p); diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index a03cc83aa26f..1907f868e770 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -303,6 +303,8 @@ struct xe_device { u8 has_mbx_power_limits:1; /** @info.has_mem_copy_instr: Device supports MEM_COPY instruction */ u8 has_mem_copy_instr:1; + /** @info.has_mert: Device has standalone MERT */ + u8 has_mert:1; /** @info.has_pxp: Device has PXP support */ u8 has_pxp:1; /** @info.has_range_tlb_inval: Has range based TLB invalidations */ diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 10eb18bc740e..4b4c6aa800ec 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -407,6 +407,7 @@ static const struct xe_device_desc cri_desc = { .has_display = false, .has_flat_ccs = false, .has_mbx_power_limits = true, + .has_mert = true, .has_sriov = true, .max_gt_per_tile = 2, .require_force_probe = true, @@ -671,6 +672,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.has_heci_cscfi = desc->has_heci_cscfi; xe->info.has_late_bind = desc->has_late_bind; xe->info.has_llc = desc->has_llc; + xe->info.has_mert = desc->has_mert; xe->info.has_pxp = desc->has_pxp; xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) && desc->has_sriov; diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index 9892c063a9c5..d89ee5d82439 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -47,6 +47,7 @@ struct xe_device_desc { u8 has_llc:1; u8 has_mbx_power_limits:1; u8 has_mem_copy_instr:1; + u8 has_mert:1; u8 has_pxp:1; u8 has_sriov:1; u8 needs_scratch:1; From 0e72241a53e9188e88bb7b258266bccd615a4d60 Mon Sep 17 00:00:00 2001 From: Lukasz Laguna Date: Mon, 24 Nov 2025 20:02:35 +0100 Subject: [PATCH 050/187] drm/xe/pf: Configure LMTT in MERT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On platforms with standalone MERT, the PF driver needs to program LMTT in MERT's LMEM_CFG register. Signed-off-by: Lukasz Laguna Reviewed-by: Piotr Piórkowski Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251124190237.20503-3-lukasz.laguna@intel.com --- drivers/gpu/drm/xe/regs/xe_mert_regs.h | 13 +++++++++++++ drivers/gpu/drm/xe/xe_lmtt.c | 10 +++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/xe/regs/xe_mert_regs.h diff --git a/drivers/gpu/drm/xe/regs/xe_mert_regs.h b/drivers/gpu/drm/xe/regs/xe_mert_regs.h new file mode 100644 index 000000000000..5b7c15e08747 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_mert_regs.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_MERT_REGS_H_ +#define _XE_MERT_REGS_H_ + +#include "regs/xe_reg_defs.h" + +#define MERT_LMEM_CFG XE_REG(0x1448b0) + +#endif /* _XE_MERT_REGS_H_ */ diff --git a/drivers/gpu/drm/xe/xe_lmtt.c b/drivers/gpu/drm/xe/xe_lmtt.c index 4dc1de482eee..f50c5a4b9edf 100644 --- a/drivers/gpu/drm/xe/xe_lmtt.c +++ b/drivers/gpu/drm/xe/xe_lmtt.c @@ -8,6 +8,7 @@ #include #include "regs/xe_gt_regs.h" +#include "regs/xe_mert_regs.h" #include "xe_assert.h" #include "xe_bo.h" @@ -17,6 +18,7 @@ #include "xe_mmio.h" #include "xe_res_cursor.h" #include "xe_sriov.h" +#include "xe_tile.h" #include "xe_tile_sriov_printk.h" /** @@ -196,16 +198,22 @@ static void lmtt_setup_dir_ptr(struct xe_lmtt *lmtt) struct xe_device *xe = tile_to_xe(tile); dma_addr_t offset = xe_bo_main_addr(lmtt->pd->bo, XE_PAGE_SIZE); struct xe_gt *gt; + u32 config; u8 id; lmtt_debug(lmtt, "DIR offset %pad\n", &offset); lmtt_assert(lmtt, xe_bo_is_vram(lmtt->pd->bo)); lmtt_assert(lmtt, IS_ALIGNED(offset, SZ_64K)); + config = LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K); + for_each_gt_on_tile(gt, tile, id) xe_mmio_write32(>->mmio, GRAPHICS_VER(xe) >= 20 ? XE2_LMEM_CFG : LMEM_CFG, - LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K)); + config); + + if (xe_device_has_mert(xe) && xe_tile_is_root(tile)) + xe_mmio_write32(&tile->mmio, MERT_LMEM_CFG, config); } /** From 1fc30960150672b4c8f842ea00c75be766759bd8 Mon Sep 17 00:00:00 2001 From: Lukasz Laguna Date: Mon, 24 Nov 2025 20:02:36 +0100 Subject: [PATCH 051/187] drm/xe/pf: Add TLB invalidation support for MERT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for triggering and handling MERT TLB invalidation. After LMTT updates, the MERT TLB invalidation is initiated to ensure memory translations remain coherent. Completion of the invalidation is signaled via MERT interrupt (bit 13 in the GFX master interrupt register). Detect and handle this interrupt to properly synchronize the invalidation flow. Signed-off-by: Lukasz Laguna Reviewed-by: Piotr Piórkowski Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251124190237.20503-4-lukasz.laguna@intel.com --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/regs/xe_irq_regs.h | 1 + drivers/gpu/drm/xe/regs/xe_mert_regs.h | 3 ++ drivers/gpu/drm/xe/xe_device_types.h | 4 ++ drivers/gpu/drm/xe/xe_irq.c | 2 + drivers/gpu/drm/xe/xe_lmtt.c | 15 +++++- drivers/gpu/drm/xe/xe_mert.c | 71 ++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_mert.h | 32 ++++++++++++ drivers/gpu/drm/xe/xe_sriov_pf.c | 4 ++ 9 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/xe/xe_mert.c create mode 100644 drivers/gpu/drm/xe/xe_mert.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 0ca5c39ec904..dd42f4251049 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -173,6 +173,7 @@ xe-$(CONFIG_PCI_IOV) += \ xe_lmtt.o \ xe_lmtt_2l.o \ xe_lmtt_ml.o \ + xe_mert.o \ xe_pci_sriov.o \ xe_sriov_packet.o \ xe_sriov_pf.o \ diff --git a/drivers/gpu/drm/xe/regs/xe_irq_regs.h b/drivers/gpu/drm/xe/regs/xe_irq_regs.h index 2f97662d958d..9d74f454d3ff 100644 --- a/drivers/gpu/drm/xe/regs/xe_irq_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_irq_regs.h @@ -20,6 +20,7 @@ #define GU_MISC_IRQ REG_BIT(29) #define ERROR_IRQ(x) REG_BIT(26 + (x)) #define DISPLAY_IRQ REG_BIT(16) +#define SOC_H2DMEMINT_IRQ REG_BIT(13) #define I2C_IRQ REG_BIT(12) #define GT_DW_IRQ(x) REG_BIT(x) diff --git a/drivers/gpu/drm/xe/regs/xe_mert_regs.h b/drivers/gpu/drm/xe/regs/xe_mert_regs.h index 5b7c15e08747..aef66c04901d 100644 --- a/drivers/gpu/drm/xe/regs/xe_mert_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_mert_regs.h @@ -10,4 +10,7 @@ #define MERT_LMEM_CFG XE_REG(0x1448b0) +#define MERT_TLB_INV_DESC_A XE_REG(0x14cf7c) +#define MERT_TLB_INV_DESC_A_VALID REG_BIT(0) + #endif /* _XE_MERT_REGS_H_ */ diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 1907f868e770..e8de3f807ad9 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -17,6 +17,7 @@ #include "xe_late_bind_fw_types.h" #include "xe_lmtt_types.h" #include "xe_memirq_types.h" +#include "xe_mert.h" #include "xe_oa_types.h" #include "xe_pagefault_types.h" #include "xe_platform_types.h" @@ -220,6 +221,9 @@ struct xe_tile { /** @debugfs: debugfs directory associated with this tile */ struct dentry *debugfs; + + /** @mert: MERT-related data */ + struct xe_mert mert; }; /** diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c index 024e13e606ec..baf5d2c6e802 100644 --- a/drivers/gpu/drm/xe/xe_irq.c +++ b/drivers/gpu/drm/xe/xe_irq.c @@ -21,6 +21,7 @@ #include "xe_hw_error.h" #include "xe_i2c.h" #include "xe_memirq.h" +#include "xe_mert.h" #include "xe_mmio.h" #include "xe_pxp.h" #include "xe_sriov.h" @@ -525,6 +526,7 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg) xe_heci_csc_irq_handler(xe, master_ctl); xe_display_irq_handler(xe, master_ctl); xe_i2c_irq_handler(xe, master_ctl); + xe_mert_irq_handler(xe, master_ctl); gu_misc_iir = gu_misc_irq_ack(xe, master_ctl); } } diff --git a/drivers/gpu/drm/xe/xe_lmtt.c b/drivers/gpu/drm/xe/xe_lmtt.c index f50c5a4b9edf..3059ea6525bc 100644 --- a/drivers/gpu/drm/xe/xe_lmtt.c +++ b/drivers/gpu/drm/xe/xe_lmtt.c @@ -15,6 +15,7 @@ #include "xe_tlb_inval.h" #include "xe_lmtt.h" #include "xe_map.h" +#include "xe_mert.h" #include "xe_mmio.h" #include "xe_res_cursor.h" #include "xe_sriov.h" @@ -270,19 +271,29 @@ static int lmtt_invalidate_hw(struct xe_lmtt *lmtt) * @lmtt: the &xe_lmtt to invalidate * * Send requests to all GuCs on this tile to invalidate all TLBs. + * If the platform has a standalone MERT, also invalidate MERT's TLB. * * This function should be called only when running as a PF driver. */ void xe_lmtt_invalidate_hw(struct xe_lmtt *lmtt) { + struct xe_tile *tile = lmtt_to_tile(lmtt); + struct xe_device *xe = lmtt_to_xe(lmtt); int err; - lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt))); + lmtt_assert(lmtt, IS_SRIOV_PF(xe)); err = lmtt_invalidate_hw(lmtt); if (err) - xe_tile_sriov_err(lmtt_to_tile(lmtt), "LMTT invalidation failed (%pe)", + xe_tile_sriov_err(tile, "LMTT invalidation failed (%pe)", ERR_PTR(err)); + + if (xe_device_has_mert(xe) && xe_tile_is_root(tile)) { + err = xe_mert_invalidate_lmtt(tile); + if (err) + xe_tile_sriov_err(tile, "MERT LMTT invalidation failed (%pe)", + ERR_PTR(err)); + } } static void lmtt_write_pte(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pt, diff --git a/drivers/gpu/drm/xe/xe_mert.c b/drivers/gpu/drm/xe/xe_mert.c new file mode 100644 index 000000000000..304cc8421999 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mert.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright(c) 2025, Intel Corporation. All rights reserved. + */ + +#include "regs/xe_irq_regs.h" +#include "regs/xe_mert_regs.h" + +#include "xe_device.h" +#include "xe_mert.h" +#include "xe_mmio.h" +#include "xe_tile.h" + +/** + * xe_mert_invalidate_lmtt - Invalidate MERT LMTT + * @tile: the &xe_tile + * + * Trigger invalidation of the MERT LMTT and wait for completion. + * + * Return: 0 on success or -ETIMEDOUT in case of a timeout. + */ +int xe_mert_invalidate_lmtt(struct xe_tile *tile) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_mert *mert = &tile->mert; + const long timeout = HZ / 4; + unsigned long flags; + + xe_assert(xe, xe_device_has_mert(xe)); + xe_assert(xe, xe_tile_is_root(tile)); + + spin_lock_irqsave(&mert->lock, flags); + if (!mert->tlb_inv_triggered) { + mert->tlb_inv_triggered = true; + reinit_completion(&mert->tlb_inv_done); + xe_mmio_write32(&tile->mmio, MERT_TLB_INV_DESC_A, MERT_TLB_INV_DESC_A_VALID); + } + spin_unlock_irqrestore(&mert->lock, flags); + + if (!wait_for_completion_timeout(&mert->tlb_inv_done, timeout)) + return -ETIMEDOUT; + + return 0; +} + +/** + * xe_mert_irq_handler - Handler for MERT interrupts + * @xe: the &xe_device + * @master_ctl: interrupt register + * + * Handle interrupts generated by MERT. + */ +void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + unsigned long flags; + u32 reg_val; + + if (!(master_ctl & SOC_H2DMEMINT_IRQ)) + return; + + spin_lock_irqsave(&tile->mert.lock, flags); + if (tile->mert.tlb_inv_triggered) { + reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_INV_DESC_A); + if (!(reg_val & MERT_TLB_INV_DESC_A_VALID)) { + tile->mert.tlb_inv_triggered = false; + complete_all(&tile->mert.tlb_inv_done); + } + } + spin_unlock_irqrestore(&tile->mert.lock, flags); +} diff --git a/drivers/gpu/drm/xe/xe_mert.h b/drivers/gpu/drm/xe/xe_mert.h new file mode 100644 index 000000000000..2e14c5dec008 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mert.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright(c) 2025, Intel Corporation. All rights reserved. + */ + +#ifndef __XE_MERT_H__ +#define __XE_MERT_H__ + +#include +#include +#include + +struct xe_device; +struct xe_tile; + +struct xe_mert { + /** @lock: protects the TLB invalidation status */ + spinlock_t lock; + /** @tlb_inv_triggered: indicates if TLB invalidation was triggered */ + bool tlb_inv_triggered; + /** @mert.tlb_inv_done: completion of TLB invalidation */ + struct completion tlb_inv_done; +}; + +#ifdef CONFIG_PCI_IOV +int xe_mert_invalidate_lmtt(struct xe_tile *tile); +void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl); +#else +static inline void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl) { } +#endif + +#endif /* __XE_MERT_H__ */ diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.c b/drivers/gpu/drm/xe/xe_sriov_pf.c index 7c779d63179f..72423bb17e6f 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf.c @@ -90,6 +90,7 @@ bool xe_sriov_pf_readiness(struct xe_device *xe) */ int xe_sriov_pf_init_early(struct xe_device *xe) { + struct xe_mert *mert = &xe_device_get_root_tile(xe)->mert; int err; xe_assert(xe, IS_SRIOV_PF(xe)); @@ -111,6 +112,9 @@ int xe_sriov_pf_init_early(struct xe_device *xe) xe_sriov_pf_service_init(xe); + spin_lock_init(&mert->lock); + init_completion(&mert->tlb_inv_done); + return 0; } From 2e02254ef5f0c07980b39fb4c334b0e612b8581e Mon Sep 17 00:00:00 2001 From: Lukasz Laguna Date: Mon, 24 Nov 2025 20:02:37 +0100 Subject: [PATCH 052/187] drm/xe/pf: Handle MERT catastrophic errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MERT block triggers an interrupt when a catastrophic error occurs. Update the interrupt handler to read the MERT catastrophic error type and log appropriate debug message. Signed-off-by: Lukasz Laguna Reviewed-by: Piotr Piórkowski Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251124190237.20503-5-lukasz.laguna@intel.com --- drivers/gpu/drm/xe/regs/xe_mert_regs.h | 5 +++++ drivers/gpu/drm/xe/xe_mert.c | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/drivers/gpu/drm/xe/regs/xe_mert_regs.h b/drivers/gpu/drm/xe/regs/xe_mert_regs.h index aef66c04901d..c345e11ceea8 100644 --- a/drivers/gpu/drm/xe/regs/xe_mert_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_mert_regs.h @@ -10,6 +10,11 @@ #define MERT_LMEM_CFG XE_REG(0x1448b0) +#define MERT_TLB_CT_INTR_ERR_ID_PORT XE_REG(0x145190) +#define MERT_TLB_CT_VFID_MASK REG_GENMASK(16, 9) +#define MERT_TLB_CT_ERROR_MASK REG_GENMASK(5, 0) +#define MERT_TLB_CT_LMTT_FAULT 0x05 + #define MERT_TLB_INV_DESC_A XE_REG(0x14cf7c) #define MERT_TLB_INV_DESC_A_VALID REG_BIT(0) diff --git a/drivers/gpu/drm/xe/xe_mert.c b/drivers/gpu/drm/xe/xe_mert.c index 304cc8421999..f7689e922953 100644 --- a/drivers/gpu/drm/xe/xe_mert.c +++ b/drivers/gpu/drm/xe/xe_mert.c @@ -55,10 +55,21 @@ void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl) struct xe_tile *tile = xe_device_get_root_tile(xe); unsigned long flags; u32 reg_val; + u8 err; if (!(master_ctl & SOC_H2DMEMINT_IRQ)) return; + reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_CT_INTR_ERR_ID_PORT); + xe_mmio_write32(&tile->mmio, MERT_TLB_CT_INTR_ERR_ID_PORT, 0); + + err = FIELD_GET(MERT_TLB_CT_ERROR_MASK, reg_val); + if (err == MERT_TLB_CT_LMTT_FAULT) + drm_dbg(&xe->drm, "MERT catastrophic error: LMTT fault (VF%u)\n", + FIELD_GET(MERT_TLB_CT_VFID_MASK, reg_val)); + else if (err) + drm_dbg(&xe->drm, "MERT catastrophic error: Unexpected fault (0x%x)\n", err); + spin_lock_irqsave(&tile->mert.lock, flags); if (tile->mert.tlb_inv_triggered) { reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_INV_DESC_A); From 00937fe1921ab346b6f6a4beaa5c38e14733caa3 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 21 Nov 2025 07:27:50 -0800 Subject: [PATCH 053/187] drm/xe/vf: Start re-emission from first unsignaled job during VF migration The LRC software ring tail is reset to the first unsignaled pending job's head. Fix the re-emission logic to begin submitting from the first unsignaled job detected, rather than scanning all pending jobs, which can cause imbalance. v2: - Include missing local changes v3: - s/skip_replay/restore_replay (Tomasz) Fixes: c25c1010df88 ("drm/xe/vf: Replay GuC submission state on pause / unpause") Signed-off-by: Matthew Brost Reviewed-by: Tomasz Lis Link: https://patch.msgid.link/20251121152750.240557-1-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_gpu_scheduler.h | 5 +++-- drivers/gpu/drm/xe/xe_guc_submit.c | 25 ++++++++++++++----------- drivers/gpu/drm/xe/xe_sched_job_types.h | 4 ++-- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.h b/drivers/gpu/drm/xe/xe_gpu_scheduler.h index 9955397aaaa9..c7a77a3a9681 100644 --- a/drivers/gpu/drm/xe/xe_gpu_scheduler.h +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.h @@ -54,13 +54,14 @@ static inline void xe_sched_tdr_queue_imm(struct xe_gpu_scheduler *sched) static inline void xe_sched_resubmit_jobs(struct xe_gpu_scheduler *sched) { struct drm_sched_job *s_job; + bool restore_replay = false; list_for_each_entry(s_job, &sched->base.pending_list, list) { struct drm_sched_fence *s_fence = s_job->s_fence; struct dma_fence *hw_fence = s_fence->parent; - if (to_xe_sched_job(s_job)->skip_emit || - (hw_fence && !dma_fence_is_signaled(hw_fence))) + restore_replay |= to_xe_sched_job(s_job)->restore_replay; + if (restore_replay || (hw_fence && !dma_fence_is_signaled(hw_fence))) sched->base.ops->run_job(s_job); } } diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 7e0882074a99..713263497bb9 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -822,7 +822,7 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - if (!job->skip_emit || job->last_replay) { + if (!job->restore_replay || job->last_replay) { if (xe_exec_queue_is_parallel(q)) wq_item_append(q); else @@ -881,10 +881,10 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) { if (!exec_queue_registered(q)) register_exec_queue(q, GUC_CONTEXT_NORMAL); - if (!job->skip_emit) + if (!job->restore_replay) q->ring_ops->emit_job(job); submit_exec_queue(q, job); - job->skip_emit = false; + job->restore_replay = false; } /* @@ -2147,6 +2147,8 @@ static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) job = xe_sched_first_pending_job(sched); if (job) { + job->restore_replay = true; + /* * Adjust software tail so jobs submitted overwrite previous * position in ring buffer with new GGTT addresses. @@ -2236,17 +2238,18 @@ static void guc_exec_queue_unpause_prepare(struct xe_guc *guc, struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; - struct drm_sched_job *s_job; struct xe_sched_job *job = NULL; + bool restore_replay = false; - list_for_each_entry(s_job, &sched->base.pending_list, list) { - job = to_xe_sched_job(s_job); + list_for_each_entry(job, &sched->base.pending_list, drm.list) { + restore_replay |= job->restore_replay; + if (restore_replay) { + xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", + q->guc->id, xe_sched_job_seqno(job)); - xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", - q->guc->id, xe_sched_job_seqno(job)); - - q->ring_ops->emit_job(job); - job->skip_emit = true; + q->ring_ops->emit_job(job); + job->restore_replay = true; + } } if (job) diff --git a/drivers/gpu/drm/xe/xe_sched_job_types.h b/drivers/gpu/drm/xe/xe_sched_job_types.h index d26612abb4ca..7c4c54fe920a 100644 --- a/drivers/gpu/drm/xe/xe_sched_job_types.h +++ b/drivers/gpu/drm/xe/xe_sched_job_types.h @@ -63,8 +63,8 @@ struct xe_sched_job { bool ring_ops_flush_tlb; /** @ggtt: mapped in ggtt. */ bool ggtt; - /** @skip_emit: skip emitting the job */ - bool skip_emit; + /** @restore_replay: job being replayed for restore */ + bool restore_replay; /** @last_replay: last job being replayed */ bool last_replay; /** @ptrs: per instance pointers. */ From 07cf4b864f523f01d2bb522a05813df30b076ba8 Mon Sep 17 00:00:00 2001 From: Tomasz Lis Date: Mon, 24 Nov 2025 23:28:53 +0100 Subject: [PATCH 054/187] drm/xe: Protect against unset LRC when pausing submissions While pausing submissions, it is possible to encouner an exec queue which is during creation, and therefore doesn't have a valid xe_lrc struct reference. Protect agains such situation, by checking for NULL before access. Reviewed-by: Matthew Brost Fixes: c25c1010df88 ("drm/xe/vf: Replay GuC submission state on pause / unpause") Signed-off-by: Tomasz Lis Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251124222853.1900800-1-tomasz.lis@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 713263497bb9..9a0842398e95 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -2107,6 +2107,18 @@ static void guc_exec_queue_revert_pending_state_change(struct xe_guc *guc, q->guc->resume_time = 0; } +static void lrc_parallel_clear(struct xe_lrc *lrc) +{ + struct xe_device *xe = gt_to_xe(lrc->gt); + struct iosys_map map = xe_lrc_parallel_map(lrc); + int i; + + for (i = 0; i < WQ_SIZE / sizeof(u32); ++i) + parallel_write(xe, map, wq[i], + FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | + FIELD_PREP(WQ_LEN_MASK, 0)); +} + /* * This function is quite complex but only real way to ensure no state is lost * during VF resume flows. The function scans the queue state, make adjustments @@ -2130,8 +2142,8 @@ static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) guc_exec_queue_revert_pending_state_change(guc, q); if (xe_exec_queue_is_parallel(q)) { - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); + /* Pairs with WRITE_ONCE in __xe_exec_queue_init */ + struct xe_lrc *lrc = READ_ONCE(q->lrc[0]); /* * NOP existing WQ commands that may contain stale GGTT @@ -2139,10 +2151,8 @@ static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) * seems to get confused if the WQ head/tail pointers are * adjusted. */ - for (i = 0; i < WQ_SIZE / sizeof(u32); ++i) - parallel_write(xe, map, wq[i], - FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | - FIELD_PREP(WQ_LEN_MASK, 0)); + if (lrc) + lrc_parallel_clear(lrc); } job = xe_sched_first_pending_job(sched); From 5b12958b1ffa7db44c276b2d394f3ddb9e0ebaca Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Tue, 25 Nov 2025 13:26:24 +0530 Subject: [PATCH 055/187] drm/xe: Add helper to extend CPU-mirrored VMA range for merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce xe_vm_find_cpu_addr_mirror_vma_range(), which computes an extended range around a given range by including adjacent VMAs that are CPU-address-mirrored and have default memory attributes. This helper is useful for determining mergeable range without performing the actual merge. v2 - Add assert - Move unmap check to this patch v3 - Decrease offset to check by SZ_4K to avoid wrong vma return in fast lookup path v4 - *start should be >= SZ_4K (Matt) Cc: Matthew Brost Cc: Thomas Hellström Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251125075628.1182481-2-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_vm.c | 41 ++++++++++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_vm.h | 3 +++ 2 files changed, 44 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index f9989a7a710c..fc11a1a6f979 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4383,6 +4383,47 @@ int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) return xe_vm_alloc_vma(vm, &map_req, true); } +static bool is_cpu_addr_vma_with_default_attr(struct xe_vma *vma) +{ + return vma && xe_vma_is_cpu_addr_mirror(vma) && + !xe_svm_has_mapping(xe_vma_vm(vma), xe_vma_start(vma), xe_vma_end(vma)) && + xe_vma_has_default_mem_attrs(vma); +} + +/** + * xe_vm_find_cpu_addr_mirror_vma_range - Extend a VMA range to include adjacent CPU-mirrored VMAs + * @vm: VM to search within + * @start: Input/output pointer to the starting address of the range + * @end: Input/output pointer to the end address of the range + * + * Given a range defined by @start and @range, this function checks the VMAs + * immediately before and after the range. If those neighboring VMAs are + * CPU-address-mirrored and have default memory attributes, the function + * updates @start and @range to include them. This extended range can then + * be used for merging or other operations that require a unified VMA. + * + * The function does not perform the merge itself; it only computes the + * mergeable boundaries. + */ +void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm *vm, u64 *start, u64 *end) +{ + struct xe_vma *prev, *next; + + lockdep_assert_held(&vm->lock); + + if (*start >= SZ_4K) { + prev = xe_vm_find_vma_by_addr(vm, *start - SZ_4K); + if (is_cpu_addr_vma_with_default_attr(prev)) + *start = xe_vma_start(prev); + } + + if (*end < vm->size) { + next = xe_vm_find_vma_by_addr(vm, *end + 1); + if (is_cpu_addr_vma_with_default_attr(next)) + *end = xe_vma_end(next); + } +} + /** * xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma * @vm: Pointer to the xe_vm structure diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index ef8a5019574e..361f10b3c453 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -68,6 +68,9 @@ xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range); bool xe_vma_has_default_mem_attrs(struct xe_vma *vma); +void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm *vm, + u64 *start, + u64 *end); /** * xe_vm_has_scratch() - Whether the vm is configured for scratch PTEs * @vm: The vm From 4a938d3886d6dd29a54a2e6ebc03b6c2b7509ebf Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Tue, 25 Nov 2025 13:26:25 +0530 Subject: [PATCH 056/187] drm/xe: Merge adjacent default-attribute VMAs during garbage collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While restoring default memory attributes for VMAs during garbage collection, extend the target range by checking neighboring VMAs. If adjacent VMAs are CPU-address-mirrored and have default attributes, include them in the mergeable range to reduce fragmentation and improve VMA reuse. v2 -Rebase Cc: Matthew Brost Cc: Thomas Hellström Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251125075628.1182481-3-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_svm.c | 62 +++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 55c5a0eb82e1..e895a95af126 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -285,19 +285,21 @@ static int __xe_svm_garbage_collector(struct xe_vm *vm, return 0; } -static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 range_start, u64 range_end) +static void xe_vma_set_default_attributes(struct xe_vma *vma) +{ + vma->attr.preferred_loc.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE; + vma->attr.preferred_loc.migration_policy = DRM_XE_MIGRATE_ALL_PAGES; + vma->attr.pat_index = vma->attr.default_pat_index; + vma->attr.atomic_access = DRM_XE_ATOMIC_UNDEFINED; +} + +static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 start, u64 end) { struct xe_vma *vma; - struct xe_vma_mem_attr default_attr = { - .preferred_loc = { - .devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE, - .migration_policy = DRM_XE_MIGRATE_ALL_PAGES, - }, - .atomic_access = DRM_XE_ATOMIC_UNDEFINED, - }; - int err = 0; + bool has_default_attr; + int err; - vma = xe_vm_find_vma_by_addr(vm, range_start); + vma = xe_vm_find_vma_by_addr(vm, start); if (!vma) return -EINVAL; @@ -306,25 +308,33 @@ static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 range_start, u64 return 0; } - if (xe_vma_has_default_mem_attrs(vma)) - return 0; - vm_dbg(&vm->xe->drm, "Existing VMA start=0x%016llx, vma_end=0x%016llx", xe_vma_start(vma), xe_vma_end(vma)); - if (xe_vma_start(vma) == range_start && xe_vma_end(vma) == range_end) { - default_attr.pat_index = vma->attr.default_pat_index; - default_attr.default_pat_index = vma->attr.default_pat_index; - vma->attr = default_attr; - } else { - vm_dbg(&vm->xe->drm, "Split VMA start=0x%016llx, vma_end=0x%016llx", - range_start, range_end); - err = xe_vm_alloc_cpu_addr_mirror_vma(vm, range_start, range_end - range_start); - if (err) { - drm_warn(&vm->xe->drm, "VMA SPLIT failed: %pe\n", ERR_PTR(err)); - xe_vm_kill(vm, true); - return err; - } + has_default_attr = xe_vma_has_default_mem_attrs(vma); + + if (has_default_attr) { + if (xe_svm_has_mapping(vm, xe_vma_start(vma), xe_vma_end(vma))) + return 0; + + start = xe_vma_start(vma); + end = xe_vma_end(vma); + } else if (xe_vma_start(vma) == start && xe_vma_end(vma) == end) { + xe_vma_set_default_attributes(vma); + } + + xe_vm_find_cpu_addr_mirror_vma_range(vm, &start, &end); + + if (xe_vma_start(vma) == start && xe_vma_end(vma) == end && has_default_attr) + return 0; + + vm_dbg(&vm->xe->drm, "New VMA start=0x%016llx, vma_end=0x%016llx", start, end); + + err = xe_vm_alloc_cpu_addr_mirror_vma(vm, start, end - start); + if (err) { + drm_warn(&vm->xe->drm, "New VMA MAP failed: %pe\n", ERR_PTR(err)); + xe_vm_kill(vm, true); + return err; } /* From 0ae006dc48f3748fdee675052bc0e5e87b856ac2 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Tue, 25 Nov 2025 13:26:26 +0530 Subject: [PATCH 057/187] drm/xe/svm: Extend MAP range to reduce vma fragmentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR is set during VM_BIND_OP_MAP, the mapping logic now checks adjacent cpu_addr_mirror VMAs with default attributes and expands the mapping range accordingly. This ensures that bo_unmap operations ideally target the same area and helps reduce fragmentation by coalescing nearby compatible VMAs into a single mapping. Cc: Matthew Brost Cc: Thomas Hellström Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251125075628.1182481-4-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_vm.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index fc11a1a6f979..2681039f3221 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2233,6 +2233,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops, struct drm_gpuva_ops *ops; struct drm_gpuva_op *__op; struct drm_gpuvm_bo *vm_bo; + u64 range_start = addr; u64 range_end = addr + range; int err; @@ -2245,10 +2246,14 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops, switch (operation) { case DRM_XE_VM_BIND_OP_MAP: + if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) + xe_vm_find_cpu_addr_mirror_vma_range(vm, &range_start, &range_end); + + fallthrough; case DRM_XE_VM_BIND_OP_MAP_USERPTR: { struct drm_gpuvm_map_req map_req = { - .map.va.addr = addr, - .map.va.range = range, + .map.va.addr = range_start, + .map.va.range = range_end - range_start, .map.gem.obj = obj, .map.gem.offset = bo_offset_or_userptr, }; From dd62fe512d5980e86cdd5b77385012f5659775f0 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Tue, 25 Nov 2025 13:26:27 +0530 Subject: [PATCH 058/187] drm/xe/svm: Enable UNMAP for VMA merging operations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ALLOW UNMAP of VMAs associated with SVM mappings when the MAP operation is intended to merge adjacent CPU_ADDR_MIRROR VMAs. v2 - Remove mapping exist check in garbage collector Cc: Matthew Brost Cc: Thomas Hellström Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251125075628.1182481-5-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_svm.c | 3 --- drivers/gpu/drm/xe/xe_vm.c | 10 +++++++--- drivers/gpu/drm/xe/xe_vm_types.h | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index e895a95af126..46977ec1e0de 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -314,9 +314,6 @@ static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 start, u64 end) has_default_attr = xe_vma_has_default_mem_attrs(vma); if (has_default_attr) { - if (xe_svm_has_mapping(vm, xe_vma_start(vma), xe_vma_end(vma))) - return 0; - start = xe_vma_start(vma); end = xe_vma_end(vma); } else if (xe_vma_start(vma) == start && xe_vma_end(vma) == end) { diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 2681039f3221..a7a21966e390 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2246,8 +2246,10 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops, switch (operation) { case DRM_XE_VM_BIND_OP_MAP: - if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) + if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) { xe_vm_find_cpu_addr_mirror_vma_range(vm, &range_start, &range_end); + vops->flags |= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP; + } fallthrough; case DRM_XE_VM_BIND_OP_MAP_USERPTR: { @@ -2729,7 +2731,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, if (xe_vma_is_cpu_addr_mirror(vma) && xe_svm_has_mapping(vm, xe_vma_start(vma), - xe_vma_end(vma))) + xe_vma_end(vma)) && + !(vops->flags & XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP)) return -EBUSY; if (!xe_vma_is_cpu_addr_mirror(vma)) @@ -4315,6 +4318,8 @@ static int xe_vm_alloc_vma(struct xe_vm *vm, if (is_madvise) vops.flags |= XE_VMA_OPS_FLAG_MADVISE; + else + vops.flags |= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP; err = vm_bind_ioctl_ops_parse(vm, ops, &vops); if (err) @@ -4391,7 +4396,6 @@ int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) static bool is_cpu_addr_vma_with_default_attr(struct xe_vma *vma) { return vma && xe_vma_is_cpu_addr_mirror(vma) && - !xe_svm_has_mapping(xe_vma_vm(vma), xe_vma_start(vma), xe_vma_end(vma)) && xe_vma_has_default_mem_attrs(vma); } diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index ccd6cc090309..3bf912bfbdcc 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -467,6 +467,7 @@ struct xe_vma_ops { #define XE_VMA_OPS_FLAG_MADVISE BIT(1) #define XE_VMA_OPS_ARRAY_OF_BINDS BIT(2) #define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT BIT(3) +#define XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP BIT(4) u32 flags; #ifdef TEST_VM_OPS_ERROR /** @inject_error: inject error to test error handling */ From 7f08cc5b3cc3bf6416f8b55bff906f67ed75637d Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Tue, 25 Nov 2025 13:26:28 +0530 Subject: [PATCH 059/187] drm/xe/vm: Skip ufence association for CPU address mirror VMA during MAP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MAP operation for a CPU address mirror VMA does not require ufence association because such mappings are not GPU-synchronized and do not participate in GPU job completion signaling. Remove the unnecessary ufence addition for this case to avoid -EBUSY failure in check_ufence of unbind ops. Cc: Matthew Brost Cc: Thomas Hellström Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251125075628.1182481-6-himal.prasad.ghimiray@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_vm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index a7a21966e390..a70a4a1fa03c 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -3218,7 +3218,8 @@ static void op_add_ufence(struct xe_vm *vm, struct xe_vma_op *op, { switch (op->base.op) { case DRM_GPUVA_OP_MAP: - vma_add_ufence(op->map.vma, ufence); + if (!xe_vma_is_cpu_addr_mirror(op->map.vma)) + vma_add_ufence(op->map.vma, ufence); break; case DRM_GPUVA_OP_REMAP: if (op->remap.prev) From dacda0cf75d5f4aa8da5404c21dc8ab4b7667799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20Kempczy=C5=84ski?= Date: Tue, 25 Nov 2025 16:37:33 +0100 Subject: [PATCH 060/187] drm/xe: Add caching pagetable flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce device xe_caching_pt flag to selectively turn it on for supported platforms. It allows to eliminate version check and enable this feature for the future platforms. Signed-off-by: Zbigniew Kempczyński Cc: Matthew Auld Reviewed-by: Matthew Auld Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20251125153732.400766-2-zbigniew.kempczynski@intel.com --- drivers/gpu/drm/xe/xe_bo.c | 3 +-- drivers/gpu/drm/xe/xe_device_types.h | 2 ++ drivers/gpu/drm/xe/xe_pci.c | 6 ++++++ drivers/gpu/drm/xe/xe_pci_types.h | 1 + 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 465cf9fc7ce9..ea02374f8887 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -515,8 +515,7 @@ static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo, * non-coherent and require a CPU:WC mapping. */ if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_SCANOUT) || - (xe->info.graphics_verx100 >= 1270 && - bo->flags & XE_BO_FLAG_PAGETABLE)) + (!xe->info.has_cached_pt && bo->flags & XE_BO_FLAG_PAGETABLE)) caching = ttm_write_combined; } diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index e8de3f807ad9..9a9a8eb84a78 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -285,6 +285,8 @@ struct xe_device { u8 has_asid:1; /** @info.has_atomic_enable_pte_bit: Device has atomic enable PTE bit */ u8 has_atomic_enable_pte_bit:1; + /** @info.has_cached_pt: Supports caching pagetable */ + u8 has_cached_pt:1; /** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */ u8 has_device_atomics_on_smem:1; /** @info.has_fan_control: Device supports fan control */ diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 4b4c6aa800ec..94abefa3d9ca 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -168,6 +168,7 @@ static const struct xe_device_desc tgl_desc = { .pre_gmdid_media_ip = &media_ip_xem, PLATFORM(TIGERLAKE), .dma_mask_size = 39, + .has_cached_pt = true, .has_display = true, .has_llc = true, .has_sriov = true, @@ -182,6 +183,7 @@ static const struct xe_device_desc rkl_desc = { .pre_gmdid_media_ip = &media_ip_xem, PLATFORM(ROCKETLAKE), .dma_mask_size = 39, + .has_cached_pt = true, .has_display = true, .has_llc = true, .max_gt_per_tile = 1, @@ -197,6 +199,7 @@ static const struct xe_device_desc adl_s_desc = { .pre_gmdid_media_ip = &media_ip_xem, PLATFORM(ALDERLAKE_S), .dma_mask_size = 39, + .has_cached_pt = true, .has_display = true, .has_llc = true, .has_sriov = true, @@ -217,6 +220,7 @@ static const struct xe_device_desc adl_p_desc = { .pre_gmdid_media_ip = &media_ip_xem, PLATFORM(ALDERLAKE_P), .dma_mask_size = 39, + .has_cached_pt = true, .has_display = true, .has_llc = true, .has_sriov = true, @@ -235,6 +239,7 @@ static const struct xe_device_desc adl_n_desc = { .pre_gmdid_media_ip = &media_ip_xem, PLATFORM(ALDERLAKE_N), .dma_mask_size = 39, + .has_cached_pt = true, .has_display = true, .has_llc = true, .has_sriov = true, @@ -663,6 +668,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.vram_flags = desc->vram_flags; xe->info.is_dgfx = desc->is_dgfx; + xe->info.has_cached_pt = desc->has_cached_pt; xe->info.has_fan_control = desc->has_fan_control; /* runtime fusing may force flat_ccs to disabled later */ xe->info.has_flat_ccs = desc->has_flat_ccs; diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index d89ee5d82439..f19f35359696 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -37,6 +37,7 @@ struct xe_device_desc { u8 require_force_probe:1; u8 is_dgfx:1; + u8 has_cached_pt:1; u8 has_display:1; u8 has_fan_control:1; u8 has_flat_ccs:1; From 9fb1f1256e419fcd0e5000ea8aaa71a65575a90b Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 21 Nov 2025 17:25:01 -0800 Subject: [PATCH 061/187] drm/gpusvm: Limit the number of retries in drm_gpusvm_get_pages drm_gpusvm_get_pages should not be allowed to retry forever, cap the time spent in the function to HMM_RANGE_DEFAULT_TIMEOUT has this is essentially a wrapper around hmm_range_fault. Signed-off-by: Matthew Brost Reviewed-by: Himal Prasad Ghimiray Link: https://patch.msgid.link/20251122012502.382587-1-matthew.brost@intel.com --- drivers/gpu/drm/drm_gpusvm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index 73e550c8ff8c..39c8c50401dd 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -1288,6 +1288,9 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, DMA_BIDIRECTIONAL; retry: + if (time_after(jiffies, timeout)) + return -EBUSY; + hmm_range.notifier_seq = mmu_interval_read_begin(notifier); if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages)) goto set_seqno; From 6028f59620927aee2e15a424004012ae05c50684 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 21 Nov 2025 17:25:02 -0800 Subject: [PATCH 062/187] drm/xe: Covert return of -EBUSY to -ENOMEM in VM bind IOCTL xe_vma_userptr_pin_pages can return -EBUSY but -EBUSY has special meaning in VM bind IOCTLs that user fence is pending that is attached to the VMA. Convert -EBUSY to -ENOMEM in this case as -EBUSY in practice means we are low or out of memory. Signed-off-by: Matthew Brost Reviewed-by: Tejas Upadhyay Link: https://patch.msgid.link/20251122012502.382587-2-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index a70a4a1fa03c..8ab726289583 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2455,8 +2455,17 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, if (IS_ERR(vma)) return vma; - if (xe_vma_is_userptr(vma)) + if (xe_vma_is_userptr(vma)) { err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); + /* + * -EBUSY has dedicated meaning that a user fence + * attached to the VMA is busy, in practice + * xe_vma_userptr_pin_pages can only fail with -EBUSY if + * we are low on memory so convert this to -ENOMEM. + */ + if (err == -EBUSY) + err = -ENOMEM; + } } if (err) { prep_vma_destroy(vm, vma, false); From 8b8a6456ac253093a02dfe87590c446da7aa2c46 Mon Sep 17 00:00:00 2001 From: Jonathan Cavitt Date: Mon, 17 Nov 2025 19:01:15 +0000 Subject: [PATCH 063/187] drm/xe/xe_sriov_packet: Return int from pf_descriptor_init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pf_descriptor_init currently returns a size_t, which is an unsigned integer data type. This conflicts with it returning a negative errno value on failure. Make it return an int instead. This mirrors how pf_trailer_init is used later. Signed-off-by: Jonathan Cavitt Cc: Michał Winiarski Reviewed-by: Alex Zuo Link: https://patch.msgid.link/20251117190114.69953-2-jonathan.cavitt@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_sriov_packet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_sriov_packet.c b/drivers/gpu/drm/xe/xe_sriov_packet.c index bab994696896..2cefefaed9ba 100644 --- a/drivers/gpu/drm/xe/xe_sriov_packet.c +++ b/drivers/gpu/drm/xe/xe_sriov_packet.c @@ -358,7 +358,7 @@ ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, #define MIGRATION_DESCRIPTOR_DWORDS (GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \ GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN) -static size_t pf_descriptor_init(struct xe_device *xe, unsigned int vfid) +static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid) { struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid); struct xe_sriov_packet *data; From 8e2610d9a5edefb99b1a708796a8f733358e5898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Pi=C3=B3rkowski?= Date: Thu, 27 Nov 2025 08:36:43 +0100 Subject: [PATCH 064/187] drm/xe: Move VRAM MM debugfs creation to tile level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, VRAM TTM resource manager debugfs entries (vram0_mm / vram1_mm) were created globally in the XE debugfs root directory. But technically, each tile has an associated VRAM TTM manager, which it can own. Let's create VRAM memory manager debugfs entries directly under each tile's debugfs directory for better alignment with the per-tile memory layout. Signed-off-by: Piotr Piórkowski Reviewed-by: Stuart Summers Link: https://patch.msgid.link/20251127073643.144379-1-piotr.piorkowski@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_debugfs.c | 12 ------------ drivers/gpu/drm/xe/xe_tile_debugfs.c | 9 +++++++++ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index 1d5a2a43a9d7..0f8a96a05a8e 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -369,7 +369,6 @@ void xe_debugfs_register(struct xe_device *xe) struct ttm_resource_manager *man; struct xe_tile *tile; struct xe_gt *gt; - u32 mem_type; u8 tile_id; u8 id; @@ -397,17 +396,6 @@ void xe_debugfs_register(struct xe_device *xe) debugfs_create_file("disable_late_binding", 0600, root, xe, &disable_late_binding_fops); - for (mem_type = XE_PL_VRAM0; mem_type <= XE_PL_VRAM1; ++mem_type) { - man = ttm_manager_type(bdev, mem_type); - - if (man) { - char name[16]; - - snprintf(name, sizeof(name), "vram%d_mm", mem_type - XE_PL_VRAM0); - ttm_resource_manager_create_debugfs(man, root, name); - } - } - man = ttm_manager_type(bdev, XE_PL_TT); ttm_resource_manager_create_debugfs(man, root, "gtt_mm"); diff --git a/drivers/gpu/drm/xe/xe_tile_debugfs.c b/drivers/gpu/drm/xe/xe_tile_debugfs.c index 39eeca75dded..5df2f461b7b7 100644 --- a/drivers/gpu/drm/xe/xe_tile_debugfs.c +++ b/drivers/gpu/drm/xe/xe_tile_debugfs.c @@ -106,6 +106,13 @@ static const struct drm_info_list vf_safe_debugfs_list[] = { { "sa_info", .show = xe_tile_debugfs_show_with_rpm, .data = sa_info }, }; +static void tile_debugfs_create_vram_mm(struct xe_tile *tile) +{ + if (tile->mem.vram) + ttm_resource_manager_create_debugfs(&tile->mem.vram->ttm.manager, tile->debugfs, + "vram_mm"); +} + /** * xe_tile_debugfs_register - Register tile's debugfs attributes * @tile: the &xe_tile to register @@ -135,4 +142,6 @@ void xe_tile_debugfs_register(struct xe_tile *tile) drm_debugfs_create_files(vf_safe_debugfs_list, ARRAY_SIZE(vf_safe_debugfs_list), tile->debugfs, minor); + + tile_debugfs_create_vram_mm(tile); } From 99234edab8e1fd3fd2309193a3b3169970a7e770 Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Thu, 30 Oct 2025 17:53:54 +0530 Subject: [PATCH 065/187] drm/xe/vf: Update pause/unpause() helpers with VF naming Now that pause/unpause() helpers have been updated for VF migration usecase, update their naming to match the functionality and while at it, add IS_SRIOV_VF() assert to make sure they are not abused. v7: Add IS_SRIOV_VF() assert (Matthew Brost) Use "vf" suffix (Michal) Suggested-by: Matthew Brost Signed-off-by: Raag Jadav Reviewed-by: Matthew Brost Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251030122357.128825-2-raag.jadav@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_vf.c | 6 +++--- drivers/gpu/drm/xe/xe_guc_submit.c | 16 ++++++++++------ drivers/gpu/drm/xe/xe_guc_submit.h | 6 +++--- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index 4c73a077d314..97c29c55f885 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -1128,7 +1128,7 @@ static bool vf_post_migration_shutdown(struct xe_gt *gt) spin_unlock_irq(>->sriov.vf.migration.lock); xe_guc_ct_flush_and_stop(>->uc.guc.ct); - xe_guc_submit_pause(>->uc.guc); + xe_guc_submit_pause_vf(>->uc.guc); xe_tlb_inval_reset(>->tlb_inval); return false; @@ -1163,12 +1163,12 @@ static int vf_post_migration_fixups(struct xe_gt *gt) static void vf_post_migration_rearm(struct xe_gt *gt) { xe_guc_ct_restart(>->uc.guc.ct); - xe_guc_submit_unpause_prepare(>->uc.guc); + xe_guc_submit_unpause_prepare_vf(>->uc.guc); } static void vf_post_migration_kickstart(struct xe_gt *gt) { - xe_guc_submit_unpause(>->uc.guc); + xe_guc_submit_unpause_vf(>->uc.guc); } static void vf_post_migration_abort(struct xe_gt *gt) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 9a0842398e95..cc7559cab9b3 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -2169,14 +2169,15 @@ static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) } /** - * xe_guc_submit_pause - Stop further runs of submission tasks on given GuC. + * xe_guc_submit_pause_vf - Stop further runs of submission tasks for VF. * @guc: the &xe_guc struct instance whose scheduler is to be disabled */ -void xe_guc_submit_pause(struct xe_guc *guc) +void xe_guc_submit_pause_vf(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; + xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); xe_gt_assert(guc_to_gt(guc), vf_recovery(guc)); mutex_lock(&guc->submission_state.lock); @@ -2267,14 +2268,15 @@ static void guc_exec_queue_unpause_prepare(struct xe_guc *guc, } /** - * xe_guc_submit_unpause_prepare - Prepare unpause submission tasks on given GuC. + * xe_guc_submit_unpause_prepare_vf - Prepare unpause submission tasks for VF. * @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause */ -void xe_guc_submit_unpause_prepare(struct xe_guc *guc) +void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; + xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); xe_gt_assert(guc_to_gt(guc), vf_recovery(guc)); mutex_lock(&guc->submission_state.lock); @@ -2342,14 +2344,16 @@ static void guc_exec_queue_unpause(struct xe_guc *guc, struct xe_exec_queue *q) } /** - * xe_guc_submit_unpause - Allow further runs of submission tasks on given GuC. + * xe_guc_submit_unpause_vf - Allow further runs of submission tasks for VF. * @guc: the &xe_guc struct instance whose scheduler is to be enabled */ -void xe_guc_submit_unpause(struct xe_guc *guc) +void xe_guc_submit_unpause_vf(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; + xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); + mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { /* diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index b49a2748ec46..0e7996836131 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -20,10 +20,10 @@ int xe_guc_submit_reset_prepare(struct xe_guc *guc); void xe_guc_submit_reset_wait(struct xe_guc *guc); void xe_guc_submit_stop(struct xe_guc *guc); int xe_guc_submit_start(struct xe_guc *guc); -void xe_guc_submit_pause(struct xe_guc *guc); -void xe_guc_submit_unpause(struct xe_guc *guc); -void xe_guc_submit_unpause_prepare(struct xe_guc *guc); void xe_guc_submit_pause_abort(struct xe_guc *guc); +void xe_guc_submit_pause_vf(struct xe_guc *guc); +void xe_guc_submit_unpause_vf(struct xe_guc *guc); +void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc); void xe_guc_submit_wedge(struct xe_guc *guc); int xe_guc_read_stopped(struct xe_guc *guc); From 726ceb5716da7657a65e01270c494ecd4cb0b242 Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Thu, 30 Oct 2025 17:53:55 +0530 Subject: [PATCH 066/187] drm/xe/guc_submit: Introduce pause/unpause() helpers for PF Introduce pause/unpause() helpers which stop/start further runs of submission tasks on given GuC and can be called from PF context. This is in preparation of usecases where we simply need to stop/start the scheduler without losing GuC state and don't require dealing with VF migration. v7: Reword commit message (Matthew Brost) Signed-off-by: Raag Jadav Reviewed-by: Matthew Brost Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251030122357.128825-3-raag.jadav@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 30 ++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.h | 2 ++ 2 files changed, 32 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index cc7559cab9b3..3ca2558c8c96 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -2168,6 +2168,21 @@ static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) } } +/** + * xe_guc_submit_pause - Stop further runs of submission tasks on given GuC. + * @guc: the &xe_guc struct instance whose scheduler is to be disabled + */ +void xe_guc_submit_pause(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + xe_sched_submission_stop(&q->guc->sched); + mutex_unlock(&guc->submission_state.lock); +} + /** * xe_guc_submit_pause_vf - Stop further runs of submission tasks for VF. * @guc: the &xe_guc struct instance whose scheduler is to be disabled @@ -2343,6 +2358,21 @@ static void guc_exec_queue_unpause(struct xe_guc *guc, struct xe_exec_queue *q) xe_sched_submission_resume_tdr(sched); } +/** + * xe_guc_submit_unpause - Allow further runs of submission tasks on given GuC. + * @guc: the &xe_guc struct instance whose scheduler is to be enabled + */ +void xe_guc_submit_unpause(struct xe_guc *guc) +{ + struct xe_exec_queue *q; + unsigned long index; + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + xe_sched_submission_start(&q->guc->sched); + mutex_unlock(&guc->submission_state.lock); +} + /** * xe_guc_submit_unpause_vf - Allow further runs of submission tasks for VF. * @guc: the &xe_guc struct instance whose scheduler is to be enabled diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index 0e7996836131..100a7891b918 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -20,8 +20,10 @@ int xe_guc_submit_reset_prepare(struct xe_guc *guc); void xe_guc_submit_reset_wait(struct xe_guc *guc); void xe_guc_submit_stop(struct xe_guc *guc); int xe_guc_submit_start(struct xe_guc *guc); +void xe_guc_submit_pause(struct xe_guc *guc); void xe_guc_submit_pause_abort(struct xe_guc *guc); void xe_guc_submit_pause_vf(struct xe_guc *guc); +void xe_guc_submit_unpause(struct xe_guc *guc); void xe_guc_submit_unpause_vf(struct xe_guc *guc); void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc); void xe_guc_submit_wedge(struct xe_guc *guc); From e6d2fe31a76d4960b0005e8fcb5eb66d00ed2e14 Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Thu, 30 Oct 2025 17:53:56 +0530 Subject: [PATCH 067/187] drm/xe/pm: Assert on runtime suspend if VFs are enabled We hold an additional reference to the runtime PM to keep PF in D0 during VFs lifetime, as our VFs do not implement the PM capability. This means we should never be runtime suspending as long as VFs are enabled. v8: Add !IS_SRIOV_VF() assert (Matthew Brost) Suggested-by: Daniele Ceraolo Spurio Signed-off-by: Raag Jadav Reviewed-by: Matthew Brost Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251030122357.128825-4-raag.jadav@intel.com --- drivers/gpu/drm/xe/xe_pci.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 94abefa3d9ca..763f588521d0 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -1160,6 +1160,15 @@ static int xe_pci_runtime_suspend(struct device *dev) struct xe_device *xe = pdev_to_xe_device(pdev); int err; + /* + * We hold an additional reference to the runtime PM to keep PF in D0 + * during VFs lifetime, as our VFs do not implement the PM capability. + * This means we should never be runtime suspending as long as VFs are + * enabled. + */ + xe_assert(xe, !IS_SRIOV_VF(xe)); + xe_assert(xe, !pci_num_vf(pdev)); + err = xe_pm_runtime_suspend(xe); if (err) return err; From 43fb9e113bf11d78e7c817e2696705f458753c79 Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Thu, 30 Oct 2025 17:53:57 +0530 Subject: [PATCH 068/187] drm/xe/gt: Introduce runtime suspend/resume If power state is retained between suspend/resume cycle, we don't need to perform full GT re-initialization. Introduce runtime helpers for GT which greatly reduce suspend/resume delay. v2: Drop redundant xe_gt_sanitize() and xe_guc_ct_stop() (Daniele) Use runtime naming for guc helpers (Daniele) v3: Drop redundant logging, add kernel doc (Michal) Use runtime naming for ct helpers (Michal) v4: Fix tags (Rodrigo) v5: Include host_l2_vram workaround (Daniele) Reuse xe_guc_submit_enable/disable() helpers (Daniele) Co-developed-by: Riana Tauro Signed-off-by: Riana Tauro Signed-off-by: Raag Jadav Acked-by: Matthew Brost Reviewed-by: Rodrigo Vivi Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251030122357.128825-5-raag.jadav@intel.com --- drivers/gpu/drm/xe/xe_gt.c | 60 ++++++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_gt.h | 2 ++ drivers/gpu/drm/xe/xe_guc.c | 34 +++++++++++++++++++ drivers/gpu/drm/xe/xe_guc.h | 2 ++ drivers/gpu/drm/xe/xe_guc_ct.c | 27 +++++++++++++++ drivers/gpu/drm/xe/xe_guc_ct.h | 2 ++ drivers/gpu/drm/xe/xe_pm.c | 10 +++--- drivers/gpu/drm/xe/xe_uc.c | 28 ++++++++++++++++ drivers/gpu/drm/xe/xe_uc.h | 2 ++ 9 files changed, 162 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 514ed50e6d83..996756d1fbd5 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -956,6 +956,66 @@ int xe_gt_resume(struct xe_gt *gt) return 0; } +/** + * xe_gt_runtime_suspend() - GT runtime suspend + * @gt: the GT object + * + * Return: 0 on success, negative error code otherwise. + */ +int xe_gt_runtime_suspend(struct xe_gt *gt) +{ + unsigned int fw_ref; + int err = -ETIMEDOUT; + + xe_gt_dbg(gt, "runtime suspending\n"); + + fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) + goto err_force_wake; + + xe_uc_runtime_suspend(>->uc); + xe_gt_disable_host_l2_vram(gt); + + xe_force_wake_put(gt_to_fw(gt), fw_ref); + xe_gt_dbg(gt, "runtime suspended\n"); + + return 0; + +err_force_wake: + xe_force_wake_put(gt_to_fw(gt), fw_ref); + return err; +} + +/** + * xe_gt_runtime_resume() - GT runtime resume + * @gt: the GT object + * + * Return: 0 on success, negative error code otherwise. + */ +int xe_gt_runtime_resume(struct xe_gt *gt) +{ + unsigned int fw_ref; + int err = -ETIMEDOUT; + + xe_gt_dbg(gt, "runtime resuming\n"); + + fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) + goto err_force_wake; + + xe_gt_enable_host_l2_vram(gt); + xe_uc_runtime_resume(>->uc); + + xe_force_wake_put(gt_to_fw(gt), fw_ref); + xe_gt_dbg(gt, "runtime resumed\n"); + + return 0; + +err_force_wake: + xe_force_wake_put(gt_to_fw(gt), fw_ref); + return err; +} + struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, enum xe_engine_class class, u16 instance, bool logical) diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h index 9d710049da45..94969ddd9d88 100644 --- a/drivers/gpu/drm/xe/xe_gt.h +++ b/drivers/gpu/drm/xe/xe_gt.h @@ -58,6 +58,8 @@ int xe_gt_suspend(struct xe_gt *gt); void xe_gt_shutdown(struct xe_gt *gt); int xe_gt_resume(struct xe_gt *gt); void xe_gt_reset_async(struct xe_gt *gt); +int xe_gt_runtime_resume(struct xe_gt *gt); +int xe_gt_runtime_suspend(struct xe_gt *gt); void xe_gt_sanitize(struct xe_gt *gt); int xe_gt_sanitize_freq(struct xe_gt *gt); diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index d6672cf30d3e..88376bc2a483 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -1617,6 +1617,40 @@ int xe_guc_start(struct xe_guc *guc) return xe_guc_submit_start(guc); } +/** + * xe_guc_runtime_suspend() - GuC runtime suspend + * @guc: The GuC object + * + * Stop further runs of submission tasks on given GuC and runtime suspend + * GuC CT. + */ +void xe_guc_runtime_suspend(struct xe_guc *guc) +{ + xe_guc_submit_pause(guc); + xe_guc_submit_disable(guc); + xe_guc_ct_runtime_suspend(&guc->ct); +} + +/** + * xe_guc_runtime_resume() - GuC runtime resume + * @guc: The GuC object + * + * Runtime resume GuC CT and allow further runs of submission tasks on + * given GuC. + */ +void xe_guc_runtime_resume(struct xe_guc *guc) +{ + /* + * Runtime PM flows are not applicable for VFs, so it's safe to + * directly enable IRQ. + */ + guc_enable_irq(guc); + + xe_guc_ct_runtime_resume(&guc->ct); + xe_guc_submit_enable(guc); + xe_guc_submit_unpause(guc); +} + void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p) { struct xe_gt *gt = guc_to_gt(guc); diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h index e2d4c5f44ae3..fdb08658d05a 100644 --- a/drivers/gpu/drm/xe/xe_guc.h +++ b/drivers/gpu/drm/xe/xe_guc.h @@ -35,6 +35,8 @@ int xe_guc_upload(struct xe_guc *guc); int xe_guc_min_load_for_hwconfig(struct xe_guc *guc); int xe_guc_enable_communication(struct xe_guc *guc); int xe_guc_opt_in_features_enable(struct xe_guc *guc); +void xe_guc_runtime_suspend(struct xe_guc *guc); +void xe_guc_runtime_resume(struct xe_guc *guc); int xe_guc_suspend(struct xe_guc *guc); void xe_guc_notify(struct xe_guc *guc); int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr); diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 02b4588fece0..2a24d3fb6881 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -717,6 +717,33 @@ void xe_guc_ct_stop(struct xe_guc_ct *ct) stop_g2h_handler(ct); } +/** + * xe_guc_ct_runtime_suspend() - GuC CT runtime suspend + * @ct: the &xe_guc_ct + * + * Set GuC CT to disabled state. + */ +void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct) +{ + /* + * Since we're already in runtime suspend path, we shouldn't have pending + * messages. But if there happen to be any, we'd probably want them to be + * thrown as errors for further investigation. + */ + xe_guc_ct_disable(ct); +} + +/** + * xe_guc_ct_runtime_resume() - GuC CT runtime resume + * @ct: the &xe_guc_ct + * + * Restart GuC CT and set it to enabled state. + */ +void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct) +{ + xe_guc_ct_restart(ct); +} + static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) { struct guc_ctb *h2g = &ct->ctbs.h2g; diff --git a/drivers/gpu/drm/xe/xe_guc_ct.h b/drivers/gpu/drm/xe/xe_guc_ct.h index ca1ce2b3c354..5599939f8fe1 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.h +++ b/drivers/gpu/drm/xe/xe_guc_ct.h @@ -17,6 +17,8 @@ int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct); int xe_guc_ct_enable(struct xe_guc_ct *ct); int xe_guc_ct_restart(struct xe_guc_ct *ct); void xe_guc_ct_disable(struct xe_guc_ct *ct); +void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct); +void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct); void xe_guc_ct_stop(struct xe_guc_ct *ct); void xe_guc_ct_flush_and_stop(struct xe_guc_ct *ct); void xe_guc_ct_fast_path(struct xe_guc_ct *ct); diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index 766922530265..4390ba69610d 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -591,7 +591,7 @@ int xe_pm_runtime_suspend(struct xe_device *xe) } for_each_gt(gt, xe, id) { - err = xe_gt_suspend(gt); + err = xe->d3cold.allowed ? xe_gt_suspend(gt) : xe_gt_runtime_suspend(gt); if (err) goto out_resume; } @@ -633,10 +633,10 @@ int xe_pm_runtime_resume(struct xe_device *xe) xe_rpm_lockmap_acquire(xe); - for_each_gt(gt, xe, id) - xe_gt_idle_disable_c6(gt); - if (xe->d3cold.allowed) { + for_each_gt(gt, xe, id) + xe_gt_idle_disable_c6(gt); + err = xe_pcode_ready(xe, true); if (err) goto out; @@ -657,7 +657,7 @@ int xe_pm_runtime_resume(struct xe_device *xe) xe_irq_resume(xe); for_each_gt(gt, xe, id) - xe_gt_resume(gt); + xe->d3cold.allowed ? xe_gt_resume(gt) : xe_gt_runtime_resume(gt); xe_display_pm_runtime_resume(xe); diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c index 40aed4a66bac..157520ea1783 100644 --- a/drivers/gpu/drm/xe/xe_uc.c +++ b/drivers/gpu/drm/xe/xe_uc.c @@ -304,6 +304,34 @@ int xe_uc_suspend(struct xe_uc *uc) return xe_guc_suspend(&uc->guc); } +/** + * xe_uc_runtime_suspend() - UC runtime suspend + * @uc: the UC object + * + * Runtime suspend all UCs. + */ +void xe_uc_runtime_suspend(struct xe_uc *uc) +{ + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return; + + xe_guc_runtime_suspend(&uc->guc); +} + +/** + * xe_uc_runtime_resume() - UC runtime resume + * @uc: the UC object + * + * Runtime resume all UCs. + */ +void xe_uc_runtime_resume(struct xe_uc *uc) +{ + if (!xe_device_uc_enabled(uc_to_xe(uc))) + return; + + xe_guc_runtime_resume(&uc->guc); +} + /** * xe_uc_declare_wedged() - Declare UC wedged * @uc: the UC object diff --git a/drivers/gpu/drm/xe/xe_uc.h b/drivers/gpu/drm/xe/xe_uc.h index 21c9306098cf..5398da1a8097 100644 --- a/drivers/gpu/drm/xe/xe_uc.h +++ b/drivers/gpu/drm/xe/xe_uc.h @@ -14,6 +14,8 @@ int xe_uc_init_post_hwconfig(struct xe_uc *uc); int xe_uc_load_hw(struct xe_uc *uc); void xe_uc_gucrc_disable(struct xe_uc *uc); int xe_uc_reset_prepare(struct xe_uc *uc); +void xe_uc_runtime_resume(struct xe_uc *uc); +void xe_uc_runtime_suspend(struct xe_uc *uc); void xe_uc_stop_prepare(struct xe_uc *uc); void xe_uc_stop(struct xe_uc *uc); int xe_uc_start(struct xe_uc *uc); From 01c724aa7bf84e9d081a56e0cbf1d282678ce144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Winiarski?= Date: Thu, 27 Nov 2025 10:39:31 +0100 Subject: [PATCH 069/187] drm/xe/pf: Enable SR-IOV VF migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All of the necessary building blocks are now in place to support SR-IOV VF migration. Flip the enable/disable logic to match VF code and disable the feature only for platforms that don't meet the necessary prerequisites. To allow more testing and experiments, on DEBUG builds any missing prerequisites will be ignored. Reviewed-by: Michal Wajdeczko Link: https://patch.msgid.link/20251127093934.1462188-2-michal.winiarski@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c | 9 +++++ drivers/gpu/drm/xe/xe_sriov_pf_migration.c | 35 ++++++++++++++++--- drivers/gpu/drm/xe/xe_sriov_pf_migration.h | 1 + .../gpu/drm/xe/xe_sriov_pf_migration_types.h | 4 +-- 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c index d5d918ddce4f..3174a8dee779 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c @@ -17,6 +17,7 @@ #include "xe_gt_sriov_pf_helpers.h" #include "xe_gt_sriov_pf_migration.h" #include "xe_gt_sriov_printk.h" +#include "xe_guc.h" #include "xe_guc_buf.h" #include "xe_guc_ct.h" #include "xe_migrate.h" @@ -1023,6 +1024,12 @@ static void action_ring_cleanup(void *arg) ptr_ring_cleanup(r, destroy_pf_packet); } +static void pf_gt_migration_check_support(struct xe_gt *gt) +{ + if (GUC_FIRMWARE_VER(>->uc.guc) < MAKE_GUC_VER(70, 54, 0)) + xe_sriov_pf_migration_disable(gt_to_xe(gt), "requires GuC version >= 70.54.0"); +} + /** * xe_gt_sriov_pf_migration_init() - Initialize support for VF migration. * @gt: the &xe_gt @@ -1039,6 +1046,8 @@ int xe_gt_sriov_pf_migration_init(struct xe_gt *gt) xe_gt_assert(gt, IS_SRIOV_PF(xe)); + pf_gt_migration_check_support(gt); + if (!pf_migration_supported(gt)) return 0; diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c index de06cc690fc8..6c4b16409cc9 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c @@ -46,13 +46,37 @@ bool xe_sriov_pf_migration_supported(struct xe_device *xe) { xe_assert(xe, IS_SRIOV_PF(xe)); - return xe->sriov.pf.migration.supported; + return IS_ENABLED(CONFIG_DRM_XE_DEBUG) || !xe->sriov.pf.migration.disabled; } -static bool pf_check_migration_support(struct xe_device *xe) +/** + * xe_sriov_pf_migration_disable() - Turn off SR-IOV VF migration support on PF. + * @xe: the &xe_device instance. + * @fmt: format string for the log message, to be combined with following VAs. + */ +void xe_sriov_pf_migration_disable(struct xe_device *xe, const char *fmt, ...) { - /* XXX: for now this is for feature enabling only */ - return IS_ENABLED(CONFIG_DRM_XE_DEBUG); + struct va_format vaf; + va_list va_args; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + va_start(va_args, fmt); + vaf.fmt = fmt; + vaf.va = &va_args; + xe_sriov_notice(xe, "migration %s: %pV\n", + IS_ENABLED(CONFIG_DRM_XE_DEBUG) ? + "missing prerequisite" : "disabled", + &vaf); + va_end(va_args); + + xe->sriov.pf.migration.disabled = true; +} + +static void pf_migration_check_support(struct xe_device *xe) +{ + if (!xe_device_has_memirq(xe)) + xe_sriov_pf_migration_disable(xe, "requires memory-based IRQ support"); } static void pf_migration_cleanup(void *arg) @@ -77,7 +101,8 @@ int xe_sriov_pf_migration_init(struct xe_device *xe) xe_assert(xe, IS_SRIOV_PF(xe)); - xe->sriov.pf.migration.supported = pf_check_migration_support(xe); + pf_migration_check_support(xe); + if (!xe_sriov_pf_migration_supported(xe)) return 0; diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration.h b/drivers/gpu/drm/xe/xe_sriov_pf_migration.h index b806298a0bb6..f8f408df8481 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_migration.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration.h @@ -14,6 +14,7 @@ struct xe_sriov_packet; int xe_sriov_pf_migration_init(struct xe_device *xe); bool xe_sriov_pf_migration_supported(struct xe_device *xe); +void xe_sriov_pf_migration_disable(struct xe_device *xe, const char *fmt, ...); int xe_sriov_pf_migration_restore_produce(struct xe_device *xe, unsigned int vfid, struct xe_sriov_packet *data); struct xe_sriov_packet * diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h b/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h index 363d673ee1dd..7d9a8a278d91 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h @@ -14,8 +14,8 @@ * struct xe_sriov_pf_migration - Xe device level VF migration data */ struct xe_sriov_pf_migration { - /** @supported: indicates whether VF migration feature is supported */ - bool supported; + /** @disabled: indicates whether VF migration feature is disabled */ + bool disabled; }; /** From 8b3cce3ad9c78ce3dae1c178f99352d50e12a3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Winiarski?= Date: Thu, 27 Nov 2025 10:39:32 +0100 Subject: [PATCH 070/187] drm/xe/pci: Introduce a helper to allow VF access to PF xe_device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In certain scenarios (such as VF migration), VF driver needs to interact with PF driver. Add a helper to allow VF driver access to PF xe_device. Reviewed-by: Michal Wajdeczko Link: https://patch.msgid.link/20251127093934.1462188-3-michal.winiarski@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_pci.c | 17 +++++++++++++++++ drivers/gpu/drm/xe/xe_pci.h | 3 +++ 2 files changed, 20 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 763f588521d0..75e3588242c7 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -1239,6 +1239,23 @@ static struct pci_driver xe_pci_driver = { #endif }; +/** + * xe_pci_to_pf_device() - Get PF &xe_device. + * @pdev: the VF &pci_dev device + * + * Return: pointer to PF &xe_device, NULL otherwise. + */ +struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev) +{ + struct drm_device *drm; + + drm = pci_iov_get_pf_drvdata(pdev, &xe_pci_driver); + if (IS_ERR(drm)) + return NULL; + + return to_xe_device(drm); +} + int xe_register_pci_driver(void) { return pci_register_driver(&xe_pci_driver); diff --git a/drivers/gpu/drm/xe/xe_pci.h b/drivers/gpu/drm/xe/xe_pci.h index 611c1209b14c..11bcc5fe2c5b 100644 --- a/drivers/gpu/drm/xe/xe_pci.h +++ b/drivers/gpu/drm/xe/xe_pci.h @@ -6,7 +6,10 @@ #ifndef _XE_PCI_H_ #define _XE_PCI_H_ +struct pci_dev; + int xe_register_pci_driver(void); void xe_unregister_pci_driver(void); +struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev); #endif From 17f22465c5a5573724c942ca7147b4024631ef87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Winiarski?= Date: Thu, 27 Nov 2025 10:39:33 +0100 Subject: [PATCH 071/187] drm/xe/pf: Export helpers for VFIO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Device specific VFIO driver variant for Xe will implement VF migration. Export everything that's needed for migration ops. Reviewed-by: Michal Wajdeczko Link: https://patch.msgid.link/20251127093934.1462188-4-michal.winiarski@intel.com Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/Makefile | 4 + drivers/gpu/drm/xe/xe_sriov_vfio.c | 80 ++++++++++++++++ include/drm/intel/xe_sriov_vfio.h | 143 +++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+) create mode 100644 drivers/gpu/drm/xe/xe_sriov_vfio.c create mode 100644 include/drm/intel/xe_sriov_vfio.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index dd42f4251049..0eadfeda67d9 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -185,6 +185,10 @@ xe-$(CONFIG_PCI_IOV) += \ xe_sriov_pf_sysfs.o \ xe_tile_sriov_pf_debugfs.o +ifeq ($(CONFIG_PCI_IOV),y) + xe-$(CONFIG_XE_VFIO_PCI) += xe_sriov_vfio.o +endif + # include helpers for tests even when XE is built-in ifdef CONFIG_DRM_XE_KUNIT_TEST xe-y += tests/xe_kunit_helpers.o diff --git a/drivers/gpu/drm/xe/xe_sriov_vfio.c b/drivers/gpu/drm/xe/xe_sriov_vfio.c new file mode 100644 index 000000000000..e9a7615bb5c5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vfio.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include + +#include "xe_pci.h" +#include "xe_pm.h" +#include "xe_sriov_pf_control.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_migration.h" + +struct xe_device *xe_sriov_vfio_get_pf(struct pci_dev *pdev) +{ + return xe_pci_to_pf_device(pdev); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_get_pf, "xe-vfio-pci"); + +bool xe_sriov_vfio_migration_supported(struct xe_device *xe) +{ + if (!IS_SRIOV_PF(xe)) + return -EPERM; + + return xe_sriov_pf_migration_supported(xe); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_migration_supported, "xe-vfio-pci"); + +#define DEFINE_XE_SRIOV_VFIO_FUNCTION(_type, _func, _impl) \ +_type xe_sriov_vfio_##_func(struct xe_device *xe, unsigned int vfid) \ +{ \ + if (!IS_SRIOV_PF(xe)) \ + return -EPERM; \ + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) \ + return -EINVAL; \ + \ + guard(xe_pm_runtime_noresume)(xe); \ + \ + return xe_sriov_pf_##_impl(xe, vfid); \ +} \ +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_##_func, "xe-vfio-pci") + +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, wait_flr_done, control_wait_flr); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, suspend_device, control_pause_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_device, control_resume_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, stop_copy_enter, control_trigger_save_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, stop_copy_exit, control_finish_save_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_data_enter, control_trigger_restore_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_data_exit, control_finish_restore_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, error, control_stop_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(ssize_t, stop_copy_size, migration_size); + +ssize_t xe_sriov_vfio_data_read(struct xe_device *xe, unsigned int vfid, + char __user *buf, size_t len) +{ + if (!IS_SRIOV_PF(xe)) + return -EPERM; + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) + return -EINVAL; + + guard(xe_pm_runtime_noresume)(xe); + + return xe_sriov_pf_migration_read(xe, vfid, buf, len); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_data_read, "xe-vfio-pci"); + +ssize_t xe_sriov_vfio_data_write(struct xe_device *xe, unsigned int vfid, + const char __user *buf, size_t len) +{ + if (!IS_SRIOV_PF(xe)) + return -EPERM; + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) + return -EINVAL; + + guard(xe_pm_runtime_noresume)(xe); + + return xe_sriov_pf_migration_write(xe, vfid, buf, len); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_data_write, "xe-vfio-pci"); diff --git a/include/drm/intel/xe_sriov_vfio.h b/include/drm/intel/xe_sriov_vfio.h new file mode 100644 index 000000000000..e9814e8149fd --- /dev/null +++ b/include/drm/intel/xe_sriov_vfio.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_SRIOV_VFIO_H_ +#define _XE_SRIOV_VFIO_H_ + +#include + +struct pci_dev; +struct xe_device; + +/** + * xe_sriov_vfio_get_pf() - Get PF &xe_device. + * @pdev: the VF &pci_dev device + * + * Return: pointer to PF &xe_device, NULL otherwise. + */ +struct xe_device *xe_sriov_vfio_get_pf(struct pci_dev *pdev); + +/** + * xe_sriov_vfio_migration_supported() - Check if migration is supported. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * + * Return: true if migration is supported, false otherwise. + */ +bool xe_sriov_vfio_migration_supported(struct xe_device *xe); + +/** + * xe_sriov_vfio_wait_flr_done() - Wait for VF FLR completion. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * + * This function will wait until VF FLR is processed by PF on all tiles (or + * until timeout occurs). + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_vfio_wait_flr_done(struct xe_device *xe, unsigned int vfid); + +/** + * xe_sriov_vfio_suspend_device() - Suspend VF. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * + * This function will pause VF on all tiles/GTs. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_vfio_suspend_device(struct xe_device *xe, unsigned int vfid); + +/** + * xe_sriov_vfio_resume_device() - Resume VF. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * + * This function will resume VF on all tiles. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_vfio_resume_device(struct xe_device *xe, unsigned int vfid); + +/** + * xe_sriov_vfio_stop_copy_enter() - Initiate a VF device migration data save. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_vfio_stop_copy_enter(struct xe_device *xe, unsigned int vfid); + +/** + * xe_sriov_vfio_stop_copy_exit() - Finish a VF device migration data save. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_vfio_stop_copy_exit(struct xe_device *xe, unsigned int vfid); + +/** + * xe_sriov_vfio_resume_data_enter() - Initiate a VF device migration data restore. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_vfio_resume_data_enter(struct xe_device *xe, unsigned int vfid); + +/** + * xe_sriov_vfio_resume_data_exit() - Finish a VF device migration data restore. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_vfio_resume_data_exit(struct xe_device *xe, unsigned int vfid); + +/** + * xe_sriov_vfio_error() - Move VF device to error state. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * + * Reset is needed to move it out of error state. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_vfio_error(struct xe_device *xe, unsigned int vfid); + +/** + * xe_sriov_vfio_data_read() - Read migration data from the VF device. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * @buf: start address of userspace buffer + * @len: requested read size from userspace + * + * Return: number of bytes that has been successfully read, + * 0 if no more migration data is available, -errno on failure. + */ +ssize_t xe_sriov_vfio_data_read(struct xe_device *xe, unsigned int vfid, + char __user *buf, size_t len); +/** + * xe_sriov_vfio_data_write() - Write migration data to the VF device. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * @buf: start address of userspace buffer + * @len: requested write size from userspace + * + * Return: number of bytes that has been successfully written, -errno on failure. + */ +ssize_t xe_sriov_vfio_data_write(struct xe_device *xe, unsigned int vfid, + const char __user *buf, size_t len); +/** + * xe_sriov_vfio_stop_copy_size() - Get a size estimate of VF device migration data. + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() + * @vfid: the VF identifier (can't be 0) + * + * Return: migration data size in bytes or a negative error code on failure. + */ +ssize_t xe_sriov_vfio_stop_copy_size(struct xe_device *xe, unsigned int vfid); + +#endif From 2e38c50ae4929f0b954fee69d428db7121452867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Winiarski?= Date: Thu, 27 Nov 2025 10:39:34 +0100 Subject: [PATCH 072/187] vfio/xe: Add device specific vfio_pci driver variant for Intel graphics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In addition to generic VFIO PCI functionality, the driver implements VFIO migration uAPI, allowing userspace to enable migration for Intel Graphics SR-IOV Virtual Functions. The driver binds to VF device and uses API exposed by Xe driver to transfer the VF migration data under the control of PF device. Acked-by: Rodrigo Vivi Reviewed-by: Kevin Tian Reviewed-by: Alex Williamson Link: https://patch.msgid.link/20251127093934.1462188-5-michal.winiarski@intel.com Signed-off-by: Michał Winiarski --- MAINTAINERS | 7 + drivers/vfio/pci/Kconfig | 2 + drivers/vfio/pci/Makefile | 2 + drivers/vfio/pci/xe/Kconfig | 12 + drivers/vfio/pci/xe/Makefile | 3 + drivers/vfio/pci/xe/main.c | 573 +++++++++++++++++++++++++++++++++++ 6 files changed, 599 insertions(+) create mode 100644 drivers/vfio/pci/xe/Kconfig create mode 100644 drivers/vfio/pci/xe/Makefile create mode 100644 drivers/vfio/pci/xe/main.c diff --git a/MAINTAINERS b/MAINTAINERS index 46126ce2f968..5168abe31e8a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -26964,6 +26964,13 @@ L: virtualization@lists.linux.dev S: Maintained F: drivers/vfio/pci/virtio +VFIO XE PCI DRIVER +M: Michał Winiarski +L: kvm@vger.kernel.org +L: intel-xe@lists.freedesktop.org +S: Supported +F: drivers/vfio/pci/xe + VGA_SWITCHEROO R: Lukas Wunner S: Maintained diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index 2b0172f54665..c100f0ab87f2 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -67,4 +67,6 @@ source "drivers/vfio/pci/nvgrace-gpu/Kconfig" source "drivers/vfio/pci/qat/Kconfig" +source "drivers/vfio/pci/xe/Kconfig" + endmenu diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile index cf00c0a7e55c..f5d46aa9347b 100644 --- a/drivers/vfio/pci/Makefile +++ b/drivers/vfio/pci/Makefile @@ -19,3 +19,5 @@ obj-$(CONFIG_VIRTIO_VFIO_PCI) += virtio/ obj-$(CONFIG_NVGRACE_GPU_VFIO_PCI) += nvgrace-gpu/ obj-$(CONFIG_QAT_VFIO_PCI) += qat/ + +obj-$(CONFIG_XE_VFIO_PCI) += xe/ diff --git a/drivers/vfio/pci/xe/Kconfig b/drivers/vfio/pci/xe/Kconfig new file mode 100644 index 000000000000..cc9b6dac6ed3 --- /dev/null +++ b/drivers/vfio/pci/xe/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +config XE_VFIO_PCI + tristate "VFIO support for Intel Graphics" + depends on DRM_XE && PCI_IOV + select VFIO_PCI_CORE + help + This option enables device specific VFIO driver variant for Intel Graphics. + In addition to generic VFIO PCI functionality, it implements VFIO + migration uAPI allowing userspace to enable migration for + Intel Graphics SR-IOV Virtual Functions supported by the Xe driver. + + If you don't know what to do here, say N. diff --git a/drivers/vfio/pci/xe/Makefile b/drivers/vfio/pci/xe/Makefile new file mode 100644 index 000000000000..13aa0fd192cd --- /dev/null +++ b/drivers/vfio/pci/xe/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_XE_VFIO_PCI) += xe-vfio-pci.o +xe-vfio-pci-y := main.o diff --git a/drivers/vfio/pci/xe/main.c b/drivers/vfio/pci/xe/main.c new file mode 100644 index 000000000000..0156b53c678b --- /dev/null +++ b/drivers/vfio/pci/xe/main.c @@ -0,0 +1,573 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct xe_vfio_pci_migration_file { + struct file *filp; + /* serializes accesses to migration data */ + struct mutex lock; + struct xe_vfio_pci_core_device *xe_vdev; + u8 disabled:1; +}; + +struct xe_vfio_pci_core_device { + struct vfio_pci_core_device core_device; + struct xe_device *xe; + /* PF internal control uses vfid index starting from 1 */ + unsigned int vfid; + u8 deferred_reset:1; + /* protects migration state */ + struct mutex state_mutex; + enum vfio_device_mig_state mig_state; + /* protects the reset_done flow */ + spinlock_t reset_lock; + struct xe_vfio_pci_migration_file *migf; +}; + +#define xe_vdev_to_dev(xe_vdev) (&(xe_vdev)->core_device.pdev->dev) + +static void xe_vfio_pci_disable_file(struct xe_vfio_pci_migration_file *migf) +{ + mutex_lock(&migf->lock); + migf->disabled = true; + mutex_unlock(&migf->lock); +} + +static void xe_vfio_pci_put_file(struct xe_vfio_pci_core_device *xe_vdev) +{ + xe_vfio_pci_disable_file(xe_vdev->migf); + fput(xe_vdev->migf->filp); + xe_vdev->migf = NULL; +} + +static void xe_vfio_pci_reset(struct xe_vfio_pci_core_device *xe_vdev) +{ + if (xe_vdev->migf) + xe_vfio_pci_put_file(xe_vdev); + + xe_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; +} + +static void xe_vfio_pci_state_mutex_lock(struct xe_vfio_pci_core_device *xe_vdev) +{ + mutex_lock(&xe_vdev->state_mutex); +} + +/* + * This function is called in all state_mutex unlock cases to + * handle a 'deferred_reset' if exists. + */ +static void xe_vfio_pci_state_mutex_unlock(struct xe_vfio_pci_core_device *xe_vdev) +{ +again: + spin_lock(&xe_vdev->reset_lock); + if (xe_vdev->deferred_reset) { + xe_vdev->deferred_reset = false; + spin_unlock(&xe_vdev->reset_lock); + xe_vfio_pci_reset(xe_vdev); + goto again; + } + mutex_unlock(&xe_vdev->state_mutex); + spin_unlock(&xe_vdev->reset_lock); +} + +static void xe_vfio_pci_reset_done(struct pci_dev *pdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = pci_get_drvdata(pdev); + int ret; + + if (!pdev->is_virtfn) + return; + + /* + * VF FLR requires additional processing done by PF driver. + * The processing is done after FLR is already finished from PCIe + * perspective. + * In order to avoid a scenario where VF is used while PF processing + * is still in progress, additional synchronization point is needed. + */ + ret = xe_sriov_vfio_wait_flr_done(xe_vdev->xe, xe_vdev->vfid); + if (ret) + dev_err(&pdev->dev, "Failed to wait for FLR: %d\n", ret); + + if (!xe_vdev->vfid) + return; + + /* + * As the higher VFIO layers are holding locks across reset and using + * those same locks with the mm_lock we need to prevent ABBA deadlock + * with the state_mutex and mm_lock. + * In case the state_mutex was taken already we defer the cleanup work + * to the unlock flow of the other running context. + */ + spin_lock(&xe_vdev->reset_lock); + xe_vdev->deferred_reset = true; + if (!mutex_trylock(&xe_vdev->state_mutex)) { + spin_unlock(&xe_vdev->reset_lock); + return; + } + spin_unlock(&xe_vdev->reset_lock); + xe_vfio_pci_state_mutex_unlock(xe_vdev); + + xe_vfio_pci_reset(xe_vdev); +} + +static const struct pci_error_handlers xe_vfio_pci_err_handlers = { + .reset_done = xe_vfio_pci_reset_done, + .error_detected = vfio_pci_core_aer_err_detected, +}; + +static int xe_vfio_pci_open_device(struct vfio_device *core_vdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + struct vfio_pci_core_device *vdev = &xe_vdev->core_device; + int ret; + + ret = vfio_pci_core_enable(vdev); + if (ret) + return ret; + + xe_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; + + vfio_pci_core_finish_enable(vdev); + + return 0; +} + +static void xe_vfio_pci_close_device(struct vfio_device *core_vdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + + xe_vfio_pci_state_mutex_lock(xe_vdev); + xe_vfio_pci_reset(xe_vdev); + xe_vfio_pci_state_mutex_unlock(xe_vdev); + vfio_pci_core_close_device(core_vdev); +} + +static int xe_vfio_pci_release_file(struct inode *inode, struct file *filp) +{ + struct xe_vfio_pci_migration_file *migf = filp->private_data; + + mutex_destroy(&migf->lock); + kfree(migf); + + return 0; +} + +static ssize_t xe_vfio_pci_save_read(struct file *filp, char __user *buf, size_t len, loff_t *pos) +{ + struct xe_vfio_pci_migration_file *migf = filp->private_data; + ssize_t ret; + + if (pos) + return -ESPIPE; + + mutex_lock(&migf->lock); + if (migf->disabled) { + mutex_unlock(&migf->lock); + return -ENODEV; + } + + ret = xe_sriov_vfio_data_read(migf->xe_vdev->xe, migf->xe_vdev->vfid, buf, len); + mutex_unlock(&migf->lock); + + return ret; +} + +static const struct file_operations xe_vfio_pci_save_fops = { + .owner = THIS_MODULE, + .read = xe_vfio_pci_save_read, + .release = xe_vfio_pci_release_file, + .llseek = noop_llseek, +}; + +static ssize_t xe_vfio_pci_resume_write(struct file *filp, const char __user *buf, + size_t len, loff_t *pos) +{ + struct xe_vfio_pci_migration_file *migf = filp->private_data; + ssize_t ret; + + if (pos) + return -ESPIPE; + + mutex_lock(&migf->lock); + if (migf->disabled) { + mutex_unlock(&migf->lock); + return -ENODEV; + } + + ret = xe_sriov_vfio_data_write(migf->xe_vdev->xe, migf->xe_vdev->vfid, buf, len); + mutex_unlock(&migf->lock); + + return ret; +} + +static const struct file_operations xe_vfio_pci_resume_fops = { + .owner = THIS_MODULE, + .write = xe_vfio_pci_resume_write, + .release = xe_vfio_pci_release_file, + .llseek = noop_llseek, +}; + +static const char *vfio_dev_state_str(u32 state) +{ + switch (state) { + case VFIO_DEVICE_STATE_RUNNING: return "running"; + case VFIO_DEVICE_STATE_RUNNING_P2P: return "running_p2p"; + case VFIO_DEVICE_STATE_STOP_COPY: return "stopcopy"; + case VFIO_DEVICE_STATE_STOP: return "stop"; + case VFIO_DEVICE_STATE_RESUMING: return "resuming"; + case VFIO_DEVICE_STATE_ERROR: return "error"; + default: return ""; + } +} + +enum xe_vfio_pci_file_type { + XE_VFIO_FILE_SAVE = 0, + XE_VFIO_FILE_RESUME, +}; + +static struct xe_vfio_pci_migration_file * +xe_vfio_pci_alloc_file(struct xe_vfio_pci_core_device *xe_vdev, + enum xe_vfio_pci_file_type type) +{ + struct xe_vfio_pci_migration_file *migf; + const struct file_operations *fops; + int flags; + + migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT); + if (!migf) + return ERR_PTR(-ENOMEM); + + fops = type == XE_VFIO_FILE_SAVE ? &xe_vfio_pci_save_fops : &xe_vfio_pci_resume_fops; + flags = type == XE_VFIO_FILE_SAVE ? O_RDONLY : O_WRONLY; + migf->filp = anon_inode_getfile("xe_vfio_mig", fops, migf, flags); + if (IS_ERR(migf->filp)) { + kfree(migf); + return ERR_CAST(migf->filp); + } + + mutex_init(&migf->lock); + migf->xe_vdev = xe_vdev; + xe_vdev->migf = migf; + + stream_open(migf->filp->f_inode, migf->filp); + + return migf; +} + +static struct file * +xe_vfio_set_state(struct xe_vfio_pci_core_device *xe_vdev, u32 new) +{ + u32 cur = xe_vdev->mig_state; + int ret; + + dev_dbg(xe_vdev_to_dev(xe_vdev), + "state: %s->%s\n", vfio_dev_state_str(cur), vfio_dev_state_str(new)); + + /* + * "STOP" handling is reused for "RUNNING_P2P", as the device doesn't + * have the capability to selectively block outgoing p2p DMA transfers. + * While the device is allowing BAR accesses when the VF is stopped, it + * is not processing any new workload requests, effectively stopping + * any outgoing DMA transfers (not just p2p). + * Any VRAM / MMIO accesses occurring during "RUNNING_P2P" are kept and + * will be migrated to target VF during stop-copy. + */ + if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_RUNNING_P2P) { + ret = xe_sriov_vfio_suspend_device(xe_vdev->xe, xe_vdev->vfid); + if (ret) + goto err; + + return NULL; + } + + if ((cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_STOP) || + (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING_P2P)) + return NULL; + + if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_RUNNING) { + ret = xe_sriov_vfio_resume_device(xe_vdev->xe, xe_vdev->vfid); + if (ret) + goto err; + + return NULL; + } + + if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) { + struct xe_vfio_pci_migration_file *migf; + + migf = xe_vfio_pci_alloc_file(xe_vdev, XE_VFIO_FILE_SAVE); + if (IS_ERR(migf)) { + ret = PTR_ERR(migf); + goto err; + } + get_file(migf->filp); + + ret = xe_sriov_vfio_stop_copy_enter(xe_vdev->xe, xe_vdev->vfid); + if (ret) { + fput(migf->filp); + goto err; + } + + return migf->filp; + } + + if (cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP) { + if (xe_vdev->migf) + xe_vfio_pci_put_file(xe_vdev); + + ret = xe_sriov_vfio_stop_copy_exit(xe_vdev->xe, xe_vdev->vfid); + if (ret) + goto err; + + return NULL; + } + + if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) { + struct xe_vfio_pci_migration_file *migf; + + migf = xe_vfio_pci_alloc_file(xe_vdev, XE_VFIO_FILE_RESUME); + if (IS_ERR(migf)) { + ret = PTR_ERR(migf); + goto err; + } + get_file(migf->filp); + + ret = xe_sriov_vfio_resume_data_enter(xe_vdev->xe, xe_vdev->vfid); + if (ret) { + fput(migf->filp); + goto err; + } + + return migf->filp; + } + + if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) { + if (xe_vdev->migf) + xe_vfio_pci_put_file(xe_vdev); + + ret = xe_sriov_vfio_resume_data_exit(xe_vdev->xe, xe_vdev->vfid); + if (ret) + goto err; + + return NULL; + } + + WARN(true, "Unknown state transition %d->%d", cur, new); + return ERR_PTR(-EINVAL); + +err: + dev_dbg(xe_vdev_to_dev(xe_vdev), + "Failed to transition state: %s->%s err=%d\n", + vfio_dev_state_str(cur), vfio_dev_state_str(new), ret); + return ERR_PTR(ret); +} + +static struct file * +xe_vfio_pci_set_device_state(struct vfio_device *core_vdev, + enum vfio_device_mig_state new_state) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + enum vfio_device_mig_state next_state; + struct file *f = NULL; + int ret; + + xe_vfio_pci_state_mutex_lock(xe_vdev); + while (new_state != xe_vdev->mig_state) { + ret = vfio_mig_get_next_state(core_vdev, xe_vdev->mig_state, + new_state, &next_state); + if (ret) { + xe_sriov_vfio_error(xe_vdev->xe, xe_vdev->vfid); + f = ERR_PTR(ret); + break; + } + f = xe_vfio_set_state(xe_vdev, next_state); + if (IS_ERR(f)) + break; + + xe_vdev->mig_state = next_state; + + /* Multiple state transitions with non-NULL file in the middle */ + if (f && new_state != xe_vdev->mig_state) { + fput(f); + f = ERR_PTR(-EINVAL); + break; + } + } + xe_vfio_pci_state_mutex_unlock(xe_vdev); + + return f; +} + +static int xe_vfio_pci_get_device_state(struct vfio_device *core_vdev, + enum vfio_device_mig_state *curr_state) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + + xe_vfio_pci_state_mutex_lock(xe_vdev); + *curr_state = xe_vdev->mig_state; + xe_vfio_pci_state_mutex_unlock(xe_vdev); + + return 0; +} + +static int xe_vfio_pci_get_data_size(struct vfio_device *vdev, + unsigned long *stop_copy_length) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(vdev, struct xe_vfio_pci_core_device, core_device.vdev); + + xe_vfio_pci_state_mutex_lock(xe_vdev); + *stop_copy_length = xe_sriov_vfio_stop_copy_size(xe_vdev->xe, xe_vdev->vfid); + xe_vfio_pci_state_mutex_unlock(xe_vdev); + + return 0; +} + +static const struct vfio_migration_ops xe_vfio_pci_migration_ops = { + .migration_set_state = xe_vfio_pci_set_device_state, + .migration_get_state = xe_vfio_pci_get_device_state, + .migration_get_data_size = xe_vfio_pci_get_data_size, +}; + +static void xe_vfio_pci_migration_init(struct xe_vfio_pci_core_device *xe_vdev) +{ + struct vfio_device *core_vdev = &xe_vdev->core_device.vdev; + struct pci_dev *pdev = to_pci_dev(core_vdev->dev); + struct xe_device *xe = xe_sriov_vfio_get_pf(pdev); + + if (!xe) + return; + if (!xe_sriov_vfio_migration_supported(xe)) + return; + + mutex_init(&xe_vdev->state_mutex); + spin_lock_init(&xe_vdev->reset_lock); + + /* PF internal control uses vfid index starting from 1 */ + xe_vdev->vfid = pci_iov_vf_id(pdev) + 1; + xe_vdev->xe = xe; + + core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P; + core_vdev->mig_ops = &xe_vfio_pci_migration_ops; +} + +static void xe_vfio_pci_migration_fini(struct xe_vfio_pci_core_device *xe_vdev) +{ + if (!xe_vdev->vfid) + return; + + mutex_destroy(&xe_vdev->state_mutex); +} + +static int xe_vfio_pci_init_dev(struct vfio_device *core_vdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + + xe_vfio_pci_migration_init(xe_vdev); + + return vfio_pci_core_init_dev(core_vdev); +} + +static void xe_vfio_pci_release_dev(struct vfio_device *core_vdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + + xe_vfio_pci_migration_fini(xe_vdev); +} + +static const struct vfio_device_ops xe_vfio_pci_ops = { + .name = "xe-vfio-pci", + .init = xe_vfio_pci_init_dev, + .release = xe_vfio_pci_release_dev, + .open_device = xe_vfio_pci_open_device, + .close_device = xe_vfio_pci_close_device, + .ioctl = vfio_pci_core_ioctl, + .device_feature = vfio_pci_core_ioctl_feature, + .read = vfio_pci_core_read, + .write = vfio_pci_core_write, + .mmap = vfio_pci_core_mmap, + .request = vfio_pci_core_request, + .match = vfio_pci_core_match, + .match_token_uuid = vfio_pci_core_match_token_uuid, + .bind_iommufd = vfio_iommufd_physical_bind, + .unbind_iommufd = vfio_iommufd_physical_unbind, + .attach_ioas = vfio_iommufd_physical_attach_ioas, + .detach_ioas = vfio_iommufd_physical_detach_ioas, +}; + +static int xe_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct xe_vfio_pci_core_device *xe_vdev; + int ret; + + xe_vdev = vfio_alloc_device(xe_vfio_pci_core_device, core_device.vdev, &pdev->dev, + &xe_vfio_pci_ops); + if (IS_ERR(xe_vdev)) + return PTR_ERR(xe_vdev); + + dev_set_drvdata(&pdev->dev, &xe_vdev->core_device); + + ret = vfio_pci_core_register_device(&xe_vdev->core_device); + if (ret) { + vfio_put_device(&xe_vdev->core_device.vdev); + return ret; + } + + return 0; +} + +static void xe_vfio_pci_remove(struct pci_dev *pdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = pci_get_drvdata(pdev); + + vfio_pci_core_unregister_device(&xe_vdev->core_device); + vfio_put_device(&xe_vdev->core_device.vdev); +} + +#define INTEL_PCI_VFIO_DEVICE(_id) { \ + PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, (_id)) \ +} + +static const struct pci_device_id xe_vfio_pci_table[] = { + INTEL_PTL_IDS(INTEL_PCI_VFIO_DEVICE), + INTEL_WCL_IDS(INTEL_PCI_VFIO_DEVICE), + INTEL_BMG_IDS(INTEL_PCI_VFIO_DEVICE), + {} +}; +MODULE_DEVICE_TABLE(pci, xe_vfio_pci_table); + +static struct pci_driver xe_vfio_pci_driver = { + .name = "xe-vfio-pci", + .id_table = xe_vfio_pci_table, + .probe = xe_vfio_pci_probe, + .remove = xe_vfio_pci_remove, + .err_handler = &xe_vfio_pci_err_handlers, + .driver_managed_dma = true, +}; +module_pci_driver(xe_vfio_pci_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Michał Winiarski "); +MODULE_DESCRIPTION("VFIO PCI driver with migration support for Intel Graphics"); From 9d94c1cf6ef938abd4b849b66f8eab11e3c537ef Mon Sep 17 00:00:00 2001 From: Balasubramani Vivekanandan Date: Fri, 21 Nov 2025 15:38:23 +0530 Subject: [PATCH 073/187] drm/xe/xe3_lpg: Apply Wa_16028005424 Applied Wa_16028005424 to Graphics version from 30.00 to 30.05 Reviewed-by: Matt Roper Signed-off-by: Balasubramani Vivekanandan Link: https://patch.msgid.link/20251121100822.20076-2-balasubramani.vivekanandan@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/regs/xe_guc_regs.h | 3 +++ drivers/gpu/drm/xe/xe_wa.c | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/drivers/gpu/drm/xe/regs/xe_guc_regs.h b/drivers/gpu/drm/xe/regs/xe_guc_regs.h index 2118f7dec287..87984713dd12 100644 --- a/drivers/gpu/drm/xe/regs/xe_guc_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_guc_regs.h @@ -90,6 +90,9 @@ #define GUC_SEND_INTERRUPT XE_REG(0xc4c8) #define GUC_SEND_TRIGGER REG_BIT(0) +#define GUC_INTR_CHICKEN XE_REG(0xc50c) +#define DISABLE_SIGNALING_ENGINES REG_BIT(1) + #define GUC_BCS_RCS_IER XE_REG(0xc550) #define GUC_VCS2_VCS1_IER XE_REG(0xc554) #define GUC_WD_VECS_IER XE_REG(0xc558) diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index 3764abca3d4f..7fdb2b7cb8e4 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -15,6 +15,7 @@ #include "regs/xe_engine_regs.h" #include "regs/xe_gt_regs.h" +#include "regs/xe_guc_regs.h" #include "regs/xe_regs.h" #include "xe_device_types.h" #include "xe_force_wake.h" @@ -323,6 +324,10 @@ static const struct xe_rtp_entry_sr gt_was[] = { XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), RAMDFTUNIT_CLKGATE_DIS)), XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), }, + { XE_RTP_NAME("16028005424"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005)), + XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES)) + }, }; static const struct xe_rtp_entry_sr engine_was[] = { From 251be5fb4982ebb0f5a81b62d975bd770f3ad5c2 Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Fri, 14 Nov 2025 20:56:39 +0000 Subject: [PATCH 074/187] drm/xe: Fix freq kobject leak on sysfs_create_files failure Ensure gt->freq is released when sysfs_create_files() fails in xe_gt_freq_init(). Without this, the kobject would leak. Add kobject_put() before returning the error. Fixes: fdc81c43f0c1 ("drm/xe: use devm_add_action_or_reset() helper") Signed-off-by: Shuicheng Lin Reviewed-by: Alex Zuo Reviewed-by: Xin Wang Link: https://patch.msgid.link/20251114205638.2184529-2-shuicheng.lin@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_gt_freq.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_gt_freq.c b/drivers/gpu/drm/xe/xe_gt_freq.c index 6284a4daf00a..a40dd074106f 100644 --- a/drivers/gpu/drm/xe/xe_gt_freq.c +++ b/drivers/gpu/drm/xe/xe_gt_freq.c @@ -284,8 +284,10 @@ int xe_gt_freq_init(struct xe_gt *gt) return -ENOMEM; err = sysfs_create_files(gt->freq, freq_attrs); - if (err) + if (err) { + kobject_put(gt->freq); return err; + } err = devm_add_action_or_reset(xe->drm.dev, freq_fini, gt->freq); if (err) From 8b5502145351bde87f522df082b9e41356898ba3 Mon Sep 17 00:00:00 2001 From: Vinay Belgaumkar Date: Fri, 28 Nov 2025 21:25:48 -0800 Subject: [PATCH 075/187] drm/xe: Apply Wa_14020316580 in xe_gt_idle_enable_pg() Wa_14020316580 was getting clobbered by power gating init code later in the driver load sequence. Move the Wa so that it applies correctly. Fixes: 7cd05ef89c9d ("drm/xe/xe2hpm: Add initial set of workarounds") Suggested-by: Matt Roper Signed-off-by: Vinay Belgaumkar Reviewed-by: Riana Tauro Reviewed-by: Matt Roper Link: https://patch.msgid.link/20251129052548.70766-1-vinay.belgaumkar@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_gt_idle.c | 8 ++++++++ drivers/gpu/drm/xe/xe_wa.c | 8 -------- drivers/gpu/drm/xe/xe_wa_oob.rules | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c index 3ca7bd7c9bcd..c1c9bec3c487 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.c +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -5,6 +5,7 @@ #include +#include #include "xe_force_wake.h" #include "xe_device.h" #include "xe_gt.h" @@ -16,6 +17,7 @@ #include "xe_mmio.h" #include "xe_pm.h" #include "xe_sriov.h" +#include "xe_wa.h" /** * DOC: Xe GT Idle @@ -144,6 +146,12 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt) xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25); } + if (XE_GT_WA(gt, 14020316580)) + gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) | + VDN_MFXVDENC_POWERGATE_ENABLE(0) | + VDN_HCP_POWERGATE_ENABLE(2) | + VDN_MFXVDENC_POWERGATE_ENABLE(2)); + xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable); } diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index 7fdb2b7cb8e4..d606b058d588 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -271,14 +271,6 @@ static const struct xe_rtp_entry_sr gt_was[] = { XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)), XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), }, - { XE_RTP_NAME("14020316580"), - XE_RTP_RULES(MEDIA_VERSION(1301)), - XE_RTP_ACTIONS(CLR(POWERGATE_ENABLE, - VDN_HCP_POWERGATE_ENABLE(0) | - VDN_MFXVDENC_POWERGATE_ENABLE(0) | - VDN_HCP_POWERGATE_ENABLE(2) | - VDN_MFXVDENC_POWERGATE_ENABLE(2))), - }, { XE_RTP_NAME("14019449301"), XE_RTP_RULES(MEDIA_VERSION(1301), ENGINE_CLASS(VIDEO_DECODE)), XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)), diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index fb38eb3d6e9a..7ca7258eb5d8 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -76,3 +76,4 @@ 15015404425_disable PLATFORM(PANTHERLAKE), MEDIA_STEP(B0, FOREVER) 16026007364 MEDIA_VERSION(3000) +14020316580 MEDIA_VERSION(1301) From eafc150549ec312afd8dda86eb3e53de735049ba Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 26 Nov 2025 10:59:44 -0800 Subject: [PATCH 076/187] drm/xe: Add properties line to VM snapshot capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add properties line to VM snapshot capture which includes additional information about VMA being dumped. This is helpful for debug purposes but also to build a robust GPU hang replay tool. The current format is: []: | Permissions has two options, either "read_only" or "read_write". Type has two options, either "userptr" or "bo". Cc: José Roberto de Souza Signed-off-by: Matthew Brost Reviewed-by: Jonathan Cavitt Link: https://patch.msgid.link/20251126185952.546277-2-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 8ab726289583..880877b10d91 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4047,6 +4047,9 @@ struct xe_vm_snapshot { struct { u64 ofs, bo_ofs; unsigned long len; +#define XE_VM_SNAP_FLAG_USERPTR BIT(0) +#define XE_VM_SNAP_FLAG_READ_ONLY BIT(1) + unsigned long flags; struct xe_bo *bo; void *data; struct mm_struct *mm; @@ -4087,6 +4090,8 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) snap->snap[i].ofs = xe_vma_start(vma); snap->snap[i].len = xe_vma_size(vma); + snap->snap[i].flags = xe_vma_read_only(vma) ? + XE_VM_SNAP_FLAG_READ_ONLY : 0; if (bo) { snap->snap[i].bo = xe_bo_get(bo); snap->snap[i].bo_ofs = xe_vma_bo_offset(vma); @@ -4100,6 +4105,7 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) snap->snap[i].data = ERR_PTR(-EFAULT); snap->snap[i].bo_ofs = xe_vma_userptr(vma); + snap->snap[i].flags |= XE_VM_SNAP_FLAG_USERPTR; } else { snap->snap[i].data = ERR_PTR(-ENOENT); } @@ -4169,6 +4175,12 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) for (i = 0; i < snap->num_snaps; i++) { drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len); + drm_printf(p, "[%llx].properties: %s|%s\n", snap->snap[i].ofs, + snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ? + "read_only" : "read_write", + snap->snap[i].flags & XE_VM_SNAP_FLAG_USERPTR ? + "userptr" : "bo"); + if (IS_ERR(snap->snap[i].data)) { drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs, PTR_ERR(snap->snap[i].data)); From 819c9ffd425971e0e913e5ee690f3485a28c1e0b Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 26 Nov 2025 10:59:45 -0800 Subject: [PATCH 077/187] drm/xe: Add "null_sparse" type to VM snap properties MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add "null_sparse" type to VM snap properties indicating the VMA reads zero and writes are droppped. This is useful information for debug and will help build a robust GPU hang replay tool. The current format is: []: | Permissions has two options, either "read_only" or "read_write". Type has three options, either "userptr", "null_sparse", or "bo". Cc: José Roberto de Souza Signed-off-by: Matthew Brost Reviewed-by: Jonathan Cavitt Link: https://patch.msgid.link/20251126185952.546277-3-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 880877b10d91..95bf328382bb 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4049,6 +4049,7 @@ struct xe_vm_snapshot { unsigned long len; #define XE_VM_SNAP_FLAG_USERPTR BIT(0) #define XE_VM_SNAP_FLAG_READ_ONLY BIT(1) +#define XE_VM_SNAP_FLAG_IS_NULL BIT(2) unsigned long flags; struct xe_bo *bo; void *data; @@ -4106,6 +4107,8 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) snap->snap[i].bo_ofs = xe_vma_userptr(vma); snap->snap[i].flags |= XE_VM_SNAP_FLAG_USERPTR; + } else if (xe_vma_is_null(vma)) { + snap->snap[i].flags |= XE_VM_SNAP_FLAG_IS_NULL; } else { snap->snap[i].data = ERR_PTR(-ENOENT); } @@ -4126,7 +4129,8 @@ void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap) struct xe_bo *bo = snap->snap[i].bo; int err; - if (IS_ERR(snap->snap[i].data)) + if (IS_ERR(snap->snap[i].data) || + snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL) continue; snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER); @@ -4178,6 +4182,8 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) drm_printf(p, "[%llx].properties: %s|%s\n", snap->snap[i].ofs, snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ? "read_only" : "read_write", + snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL ? + "null_sparse" : snap->snap[i].flags & XE_VM_SNAP_FLAG_USERPTR ? "userptr" : "bo"); @@ -4187,6 +4193,9 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) continue; } + if (snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL) + continue; + drm_printf(p, "[%llx].data: ", snap->snap[i].ofs); for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) { From 81e66a55a8d7c57c664520074c770484054f43b2 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 26 Nov 2025 10:59:46 -0800 Subject: [PATCH 078/187] drm/xe: Add mem_region to properties line in VM snapshot capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add memory region to properties line in VM snapshot capture indicating where the memory is located. The memory region corresponds to regions in the uAPI. This is useful information for debug and will help build a robust GPU hang replay tool. The current format is: []: ||mem_region=0x%x Permissions has two options, either "read_only" or "read_write". Type has three options, either "userptr", "null_sparse", or "bo". Memory region is a bit mask of where the memory is located. Cc: José Roberto de Souza Signed-off-by: Matthew Brost Reviewed-by: Jonathan Cavitt Link: https://patch.msgid.link/20251126185952.546277-4-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 95bf328382bb..f2cf95d23bd0 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4051,6 +4051,7 @@ struct xe_vm_snapshot { #define XE_VM_SNAP_FLAG_READ_ONLY BIT(1) #define XE_VM_SNAP_FLAG_IS_NULL BIT(2) unsigned long flags; + int uapi_mem_region; struct xe_bo *bo; void *data; struct mm_struct *mm; @@ -4096,6 +4097,18 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) if (bo) { snap->snap[i].bo = xe_bo_get(bo); snap->snap[i].bo_ofs = xe_vma_bo_offset(vma); + switch (bo->ttm.resource->mem_type) { + case XE_PL_SYSTEM: + case XE_PL_TT: + snap->snap[i].uapi_mem_region = 0; + break; + case XE_PL_VRAM0: + snap->snap[i].uapi_mem_region = 1; + break; + case XE_PL_VRAM1: + snap->snap[i].uapi_mem_region = 2; + break; + } } else if (xe_vma_is_userptr(vma)) { struct mm_struct *mm = to_userptr_vma(vma)->userptr.notifier.mm; @@ -4107,10 +4120,13 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) snap->snap[i].bo_ofs = xe_vma_userptr(vma); snap->snap[i].flags |= XE_VM_SNAP_FLAG_USERPTR; + snap->snap[i].uapi_mem_region = 0; } else if (xe_vma_is_null(vma)) { snap->snap[i].flags |= XE_VM_SNAP_FLAG_IS_NULL; + snap->snap[i].uapi_mem_region = -1; } else { snap->snap[i].data = ERR_PTR(-ENOENT); + snap->snap[i].uapi_mem_region = -1; } i++; } @@ -4179,13 +4195,16 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) for (i = 0; i < snap->num_snaps; i++) { drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len); - drm_printf(p, "[%llx].properties: %s|%s\n", snap->snap[i].ofs, + drm_printf(p, "[%llx].properties: %s|%s|mem_region=0x%lx\n", + snap->snap[i].ofs, snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ? "read_only" : "read_write", snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL ? "null_sparse" : snap->snap[i].flags & XE_VM_SNAP_FLAG_USERPTR ? - "userptr" : "bo"); + "userptr" : "bo", + snap->snap[i].uapi_mem_region == -1 ? 0 : + BIT(snap->snap[i].uapi_mem_region)); if (IS_ERR(snap->snap[i].data)) { drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs, From 6cf5d14a0bf730d6e45c407b7e3bedb1a5d8812f Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 26 Nov 2025 10:59:47 -0800 Subject: [PATCH 079/187] drm/xe: Add pat_index to properties line in VM snapshot capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add pat index to properties line in VM snapshot capture indicating the VMA caching properites. This is useful information for debug and will help build a robust GPU hang replay tool. The current format is: []: ||mem_region=0x%x|pat_index=%d Permissions has two options, either "read_only" or "read_write". Type has three options, either "userptr", "null_sparse", or "bo". Memory region is a bit mask of where the memory is located. Pat index corresponds to the value setup upon VM bind. Cc: José Roberto de Souza Signed-off-by: Matthew Brost Reviewed-by: Jonathan Cavitt Link: https://patch.msgid.link/20251126185952.546277-5-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index f2cf95d23bd0..b298953d2eed 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4052,6 +4052,7 @@ struct xe_vm_snapshot { #define XE_VM_SNAP_FLAG_IS_NULL BIT(2) unsigned long flags; int uapi_mem_region; + int pat_index; struct xe_bo *bo; void *data; struct mm_struct *mm; @@ -4094,6 +4095,7 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) snap->snap[i].len = xe_vma_size(vma); snap->snap[i].flags = xe_vma_read_only(vma) ? XE_VM_SNAP_FLAG_READ_ONLY : 0; + snap->snap[i].pat_index = vma->attr.pat_index; if (bo) { snap->snap[i].bo = xe_bo_get(bo); snap->snap[i].bo_ofs = xe_vma_bo_offset(vma); @@ -4195,7 +4197,7 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) for (i = 0; i < snap->num_snaps; i++) { drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len); - drm_printf(p, "[%llx].properties: %s|%s|mem_region=0x%lx\n", + drm_printf(p, "[%llx].properties: %s|%s|mem_region=0x%lx|pat_index=%d\n", snap->snap[i].ofs, snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ? "read_only" : "read_write", @@ -4204,7 +4206,8 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) snap->snap[i].flags & XE_VM_SNAP_FLAG_USERPTR ? "userptr" : "bo", snap->snap[i].uapi_mem_region == -1 ? 0 : - BIT(snap->snap[i].uapi_mem_region)); + BIT(snap->snap[i].uapi_mem_region), + snap->snap[i].pat_index); if (IS_ERR(snap->snap[i].data)) { drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs, From 066a1ddd8559fa494bf5137a1ef00c8761fcacf8 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 26 Nov 2025 10:59:48 -0800 Subject: [PATCH 080/187] drm/xe: Add cpu_caching to properties line in VM snapshot capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add CPU caching to properties line in VM snapshot capture indicating the BO caching properites. This is useful information for debug and will help build a robust GPU hang replay tool. The current format is: []: ||mem_region=0x%x|pat_index=%d|cpu_caching=%d Permissions has two options, either "read_only" or "read_write". Type has three options, either "userptr", "null_sparse", or "bo". Memory region is a bit mask of where the memory is located. Pat index corresponds to the value setup upon VM bind. CPU caching corresponds to the value of BO setup upon creation. v2: - Save off cpu_caching value rather than looking at BO (Carlos) v4: - Fix NULL ptr dereference (Carlos) Cc: José Roberto de Souza Signed-off-by: Matthew Brost Reviewed-by: Jonathan Cavitt Link: https://patch.msgid.link/20251126185952.546277-6-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index b298953d2eed..8bf3f9d3d644 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4053,6 +4053,7 @@ struct xe_vm_snapshot { unsigned long flags; int uapi_mem_region; int pat_index; + int cpu_caching; struct xe_bo *bo; void *data; struct mm_struct *mm; @@ -4097,6 +4098,7 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) XE_VM_SNAP_FLAG_READ_ONLY : 0; snap->snap[i].pat_index = vma->attr.pat_index; if (bo) { + snap->snap[i].cpu_caching = bo->cpu_caching; snap->snap[i].bo = xe_bo_get(bo); snap->snap[i].bo_ofs = xe_vma_bo_offset(vma); switch (bo->ttm.resource->mem_type) { @@ -4197,7 +4199,7 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) for (i = 0; i < snap->num_snaps; i++) { drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len); - drm_printf(p, "[%llx].properties: %s|%s|mem_region=0x%lx|pat_index=%d\n", + drm_printf(p, "[%llx].properties: %s|%s|mem_region=0x%lx|pat_index=%d|cpu_caching=%d\n", snap->snap[i].ofs, snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ? "read_only" : "read_write", @@ -4207,7 +4209,8 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) "userptr" : "bo", snap->snap[i].uapi_mem_region == -1 ? 0 : BIT(snap->snap[i].uapi_mem_region), - snap->snap[i].pat_index); + snap->snap[i].pat_index, + snap->snap[i].cpu_caching); if (IS_ERR(snap->snap[i].data)) { drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs, From c87f586e55364d8ca8dae38d5d00f8dd267ca8bc Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 26 Nov 2025 10:59:49 -0800 Subject: [PATCH 081/187] drm/xe: Add VM.uapi_flags to VM snapshot capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add VM.uapi_flags to VM snapshot capture VM snapshot capture. This is useful information for debug and will help build a robust GPU hang replay tool. The current format is: VM.uapi_flags: 0x%x Cc: José Roberto de Souza Signed-off-by: Matthew Brost Reviewed-by: Jonathan Cavitt Link: https://patch.msgid.link/20251126185952.546277-7-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 8bf3f9d3d644..00ffd3f03983 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -4043,6 +4043,7 @@ int xe_vm_validate_protected(struct xe_vm *vm) } struct xe_vm_snapshot { + int uapi_flags; unsigned long num_snaps; struct { u64 ofs, bo_ofs; @@ -4082,6 +4083,13 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) goto out_unlock; } + if (vm->flags & XE_VM_FLAG_FAULT_MODE) + snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_FAULT_MODE; + if (vm->flags & XE_VM_FLAG_LR_MODE) + snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_LR_MODE; + if (vm->flags & XE_VM_FLAG_SCRATCH_PAGE) + snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; + snap->num_snaps = num_snaps; i = 0; drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { @@ -4196,6 +4204,7 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) return; } + drm_printf(p, "VM.uapi_flags: 0x%x\n", snap->uapi_flags); for (i = 0; i < snap->num_snaps; i++) { drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len); From b80961a86b40372b7cfb3065439377f7e7550e59 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 26 Nov 2025 10:59:50 -0800 Subject: [PATCH 082/187] drm/xe/uapi: Add DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE which accepts a user pointer to populate the exec queue state so that a GPU hang can be replayed via a Mesa tool. v2: Update the value for HANG_REPLAY_STATE flag Cc: José Roberto de Souza Signed-off-by: Matthew Brost Signed-off-by: Carlos Santa Reviewed-by: Jonathan Cavitt Acked-by: José Roberto de Souza Acked-by: Rodrigo Vivi Link: https://patch.msgid.link/20251126185952.546277-8-matthew.brost@intel.com --- include/uapi/drm/xe_drm.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 47853659a705..37881b1eb6ba 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -210,8 +210,12 @@ struct drm_xe_ext_set_property { /** @pad: MBZ */ __u32 pad; - /** @value: property value */ - __u64 value; + union { + /** @value: property value */ + __u64 value; + /** @ptr: pointer to user value */ + __u64 ptr; + }; /** @reserved: Reserved */ __u64 reserved[2]; @@ -1292,6 +1296,7 @@ struct drm_xe_exec_queue_create { #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE 1 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE 2 +#define DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE 3 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; From 7032361d8cb13ab48b2b79167967c3a29c075ce2 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 26 Nov 2025 10:59:51 -0800 Subject: [PATCH 083/187] drm/xe: Add replay_offset and replay_length lines to LRC HWCTX snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add replay_offset and replay_length lines to LRC HWCTX snapshot with the idea being this information can be used extract the data which needs to be pass to exec queue extension DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE so GPU hang can be replayed via a Mesa tool. The additional lines look like: [HWCTX].replay_offset: 0x%x [HWCTX].replay_length: 0x%x Cc: José Roberto de Souza Signed-off-by: Matthew Brost Reviewed-by: Jonathan Cavitt Link: https://patch.msgid.link/20251126185952.546277-9-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_lrc.c | 8 ++++++++ drivers/gpu/drm/xe/xe_lrc.h | 1 + drivers/gpu/drm/xe/xe_lrc_types.h | 3 +++ 3 files changed, 12 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index b5083c99dd50..2deca095607c 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -1402,6 +1402,9 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, kref_init(&lrc->refcount); lrc->gt = gt; + lrc->replay_size = xe_gt_lrc_size(gt, hwe->class); + if (xe_gt_has_indirect_ring_state(gt)) + lrc->replay_size -= LRC_INDIRECT_RING_STATE_SIZE; lrc->size = lrc_size; lrc->flags = 0; lrc->ring.size = ring_size; @@ -2235,6 +2238,8 @@ struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc) snapshot->lrc_bo = xe_bo_get(lrc->bo); snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc); snapshot->lrc_size = lrc->size; + snapshot->replay_offset = 0; + snapshot->replay_size = lrc->replay_size; snapshot->lrc_snapshot = NULL; snapshot->ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(lrc)); snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); @@ -2305,6 +2310,9 @@ void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer } drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE); + drm_printf(p, "\n\t[HWCTX].replay_offset: 0x%lx\n", snapshot->replay_offset); + drm_printf(p, "\n\t[HWCTX].replay_length: 0x%lx\n", snapshot->replay_size); + drm_puts(p, "\t[HWCTX].data: "); for (; i < snapshot->lrc_size; i += sizeof(u32)) { u32 *val = snapshot->lrc_snapshot + i; diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index 2fb628da5c43..c3288625d0c7 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -23,6 +23,7 @@ struct xe_lrc_snapshot { struct xe_bo *lrc_bo; void *lrc_snapshot; unsigned long lrc_size, lrc_offset; + unsigned long replay_size, replay_offset; u32 context_desc; u32 ring_addr; diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h index e9883706e004..a4373d280c39 100644 --- a/drivers/gpu/drm/xe/xe_lrc_types.h +++ b/drivers/gpu/drm/xe/xe_lrc_types.h @@ -25,6 +25,9 @@ struct xe_lrc { /** @size: size of the lrc and optional indirect ring state */ u32 size; + /** @replay_size: Size LRC needed for replaying a hang */ + u32 replay_size; + /** @gt: gt which this LRC belongs to */ struct xe_gt *gt; From 1026c1a73a9686ff35ac100039f94f0725622447 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 26 Nov 2025 10:59:52 -0800 Subject: [PATCH 084/187] drm/xe: Implement DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE which sets the exec queue default state to user data passed in. The intent is for a Mesa tool to use this replay GPU hangs. v2: - Enable the flag DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE - Fix the page size math calculation to avoid a crash v4: - Use vmemdup_user (Maarten) - Copy default state first into LRC, then replay state (Testing, Carlos) Cc: José Roberto de Souza Signed-off-by: Matthew Brost Reviewed-by: Maarten Lankhorst Reviewed-by: Jonathan Cavitt Link: https://patch.msgid.link/20251126185952.546277-10-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.c | 26 +++++++++++++-- drivers/gpu/drm/xe/xe_exec_queue_types.h | 3 ++ drivers/gpu/drm/xe/xe_execlist.c | 2 +- drivers/gpu/drm/xe/xe_lrc.c | 42 ++++++++++++++++-------- drivers/gpu/drm/xe/xe_lrc.h | 3 +- 5 files changed, 58 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 8724f8de67e2..226d07a3d852 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -79,6 +79,7 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q) if (q->xef) xe_file_put(q->xef); + kvfree(q->replay_state); kfree(q); } @@ -225,8 +226,8 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags) struct xe_lrc *lrc; xe_gt_sriov_vf_wait_valid_ggtt(q->gt); - lrc = xe_lrc_create(q->hwe, q->vm, xe_lrc_ring_size(), - q->msix_vec, flags); + lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state, + xe_lrc_ring_size(), q->msix_vec, flags); if (IS_ERR(lrc)) { err = PTR_ERR(lrc); goto err_lrc; @@ -567,6 +568,23 @@ exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM); } +static int exec_queue_set_hang_replay_state(struct xe_device *xe, + struct xe_exec_queue *q, + u64 value) +{ + size_t size = xe_gt_lrc_hang_replay_size(q->gt, q->class); + u64 __user *address = u64_to_user_ptr(value); + void *ptr; + + ptr = vmemdup_user(address, size); + if (XE_IOCTL_DBG(xe, IS_ERR(ptr))) + return PTR_ERR(ptr); + + q->replay_state = ptr; + + return 0; +} + typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, struct xe_exec_queue *q, u64 value); @@ -575,6 +593,7 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type, + [DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE] = exec_queue_set_hang_replay_state, }; static int exec_queue_user_ext_set_property(struct xe_device *xe, @@ -595,7 +614,8 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe, XE_IOCTL_DBG(xe, ext.pad) || XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE && - ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE)) + ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE && + ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE)) return -EINVAL; idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 771ffe35cd0c..3ba10632dcd6 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -167,6 +167,9 @@ struct xe_exec_queue { /** @ufence_timeline_value: User fence timeline value */ u64 ufence_timeline_value; + /** @replay_state: GPU hang replay state */ + void *replay_state; + /** @ops: submission backend exec queue operations */ const struct xe_exec_queue_ops *ops; diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c index 769d05517f93..46c17a18a3f4 100644 --- a/drivers/gpu/drm/xe/xe_execlist.c +++ b/drivers/gpu/drm/xe/xe_execlist.c @@ -269,7 +269,7 @@ struct xe_execlist_port *xe_execlist_port_create(struct xe_device *xe, port->hwe = hwe; - port->lrc = xe_lrc_create(hwe, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX, 0); + port->lrc = xe_lrc_create(hwe, NULL, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX, 0); if (IS_ERR(port->lrc)) { err = PTR_ERR(port->lrc); goto err; diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 2deca095607c..a05060f75e7e 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -91,13 +91,19 @@ gt_engine_needs_indirect_ctx(struct xe_gt *gt, enum xe_engine_class class) return false; } -size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class) +/** + * xe_gt_lrc_hang_replay_size() - Hang replay size + * @gt: The GT + * @class: Hardware engine class + * + * Determine size of GPU hang replay state for a GT and hardware engine class. + * + * Return: Size of GPU hang replay size + */ +size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class) { struct xe_device *xe = gt_to_xe(gt); - size_t size; - - /* Per-process HW status page (PPHWSP) */ - size = LRC_PPHWSP_SIZE; + size_t size = 0; /* Engine context image */ switch (class) { @@ -123,11 +129,18 @@ size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class) size += 1 * SZ_4K; } + return size; +} + +size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class) +{ + size_t size = xe_gt_lrc_hang_replay_size(gt, class); + /* Add indirect ring state page */ if (xe_gt_has_indirect_ring_state(gt)) size += LRC_INDIRECT_RING_STATE_SIZE; - return size; + return size + LRC_PPHWSP_SIZE; } /* @@ -1387,7 +1400,8 @@ setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) } static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - struct xe_vm *vm, u32 ring_size, u16 msix_vec, + struct xe_vm *vm, void *replay_state, u32 ring_size, + u16 msix_vec, u32 init_flags) { struct xe_gt *gt = hwe->gt; @@ -1402,9 +1416,7 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, kref_init(&lrc->refcount); lrc->gt = gt; - lrc->replay_size = xe_gt_lrc_size(gt, hwe->class); - if (xe_gt_has_indirect_ring_state(gt)) - lrc->replay_size -= LRC_INDIRECT_RING_STATE_SIZE; + lrc->replay_size = xe_gt_lrc_hang_replay_size(gt, hwe->class); lrc->size = lrc_size; lrc->flags = 0; lrc->ring.size = ring_size; @@ -1441,11 +1453,14 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, * scratch. */ map = __xe_lrc_pphwsp_map(lrc); - if (gt->default_lrc[hwe->class]) { + if (gt->default_lrc[hwe->class] || replay_state) { xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE); /* PPHWSP */ xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE, lrc_size - LRC_PPHWSP_SIZE); + if (replay_state) + xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, + replay_state, lrc->replay_size); } else { void *init_data = empty_lrc_data(hwe); @@ -1553,6 +1568,7 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, * xe_lrc_create - Create a LRC * @hwe: Hardware Engine * @vm: The VM (address space) + * @replay_state: GPU hang replay state * @ring_size: LRC ring size * @msix_vec: MSI-X interrupt vector (for platforms that support it) * @flags: LRC initialization flags @@ -1563,7 +1579,7 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, * upon failure. */ struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, - u32 ring_size, u16 msix_vec, u32 flags) + void *replay_state, u32 ring_size, u16 msix_vec, u32 flags) { struct xe_lrc *lrc; int err; @@ -1572,7 +1588,7 @@ struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, if (!lrc) return ERR_PTR(-ENOMEM); - err = xe_lrc_init(lrc, hwe, vm, ring_size, msix_vec, flags); + err = xe_lrc_init(lrc, hwe, vm, replay_state, ring_size, msix_vec, flags); if (err) { kfree(lrc); return ERR_PTR(err); diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index c3288625d0c7..a32472b92242 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -50,7 +50,7 @@ struct xe_lrc_snapshot { #define XE_LRC_CREATE_USER_CTX BIT(2) struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, - u32 ring_size, u16 msix_vec, u32 flags); + void *replay_state, u32 ring_size, u16 msix_vec, u32 flags); void xe_lrc_destroy(struct kref *ref); /** @@ -87,6 +87,7 @@ static inline size_t xe_lrc_ring_size(void) return SZ_16K; } +size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class); size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class); u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc); u32 xe_lrc_regs_offset(struct xe_lrc *lrc); From aa8225d4ee87d35eca89877981d8038ea2c0f402 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Wed, 26 Nov 2025 14:43:58 -0800 Subject: [PATCH 085/187] MAINTAINERS: Remove myself from xe maintainers As I'm leaving Intel soon, drop myself from the list of Xe maintainers. Also update the mailmap to switch to my kernel.org address. Acked-by: Jani Nikula Acked-by: Rodrigo Vivi Link: https://patch.msgid.link/20251126224357.2482051-2-lucas.demarchi@intel.com Signed-off-by: Lucas De Marchi --- .mailmap | 1 + MAINTAINERS | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index d2edd256b19d..1954a2faa8b2 100644 --- a/.mailmap +++ b/.mailmap @@ -471,6 +471,7 @@ Lorenzo Pieralisi Lorenzo Stoakes Luca Ceresoli Luca Weiss +Lucas De Marchi Lukasz Luba Luo Jie Lance Yang diff --git a/MAINTAINERS b/MAINTAINERS index 5168abe31e8a..9bad26fb99ba 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12483,7 +12483,6 @@ F: include/drm/intel/ F: include/uapi/drm/i915_drm.h INTEL DRM XE DRIVER (Lunar Lake and newer) -M: Lucas De Marchi M: Thomas Hellström M: Rodrigo Vivi L: intel-xe@lists.freedesktop.org From 2e2dab20dd664ea1ca6ed2342f6dc509496c9290 Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Mon, 1 Dec 2025 15:20:13 +0530 Subject: [PATCH 086/187] drm/xe/vf: Enable VF migration only on supported GuC versions Enable VF migration starting with GuC 70.54.0 (compatibility version 1.27.0) which supports additional VF2GUC_RESFIX_START message required to handle migration recovery in a more robust way. Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Tomasz Lis Reviewed-by: Michal Wajdeczko Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251201095011.21453-7-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_sriov_vf.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c index 284ce37ca92d..d56b8cfea50b 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf.c @@ -170,6 +170,26 @@ void xe_sriov_vf_init_early(struct xe_device *xe) vf_migration_init_early(xe); } +static bool vf_migration_init_late(struct xe_device *xe) +{ + struct xe_gt *gt = xe_root_mmio_gt(xe); + struct xe_uc_fw_version guc_version; + + if (!xe_sriov_vf_migration_supported(xe)) + return 0; + + xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version); + if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 27, 0)) { + xe_sriov_vf_migration_disable(xe, + "requires GuC ABI >= 1.27.0, but only %u.%u.%u found", + guc_version.major, guc_version.minor, + guc_version.patch); + return 0; + } + + return xe_sriov_vf_ccs_init(xe); +} + /** * xe_sriov_vf_init_late() - SR-IOV VF late initialization functions. * @xe: the &xe_device to initialize @@ -180,7 +200,7 @@ void xe_sriov_vf_init_early(struct xe_device *xe) */ int xe_sriov_vf_init_late(struct xe_device *xe) { - return xe_sriov_vf_ccs_init(xe); + return vf_migration_init_late(xe); } static int sa_info_vf_ccs(struct seq_file *m, void *data) From b5fbb94341a2b1d78b24713db454dcda2fc7194b Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Mon, 1 Dec 2025 15:20:14 +0530 Subject: [PATCH 087/187] drm/xe/vf: Introduce RESFIX start marker support In scenarios involving double migration, the VF KMD may encounter situations where it is instructed to re-migrate before having the opportunity to send RESFIX_DONE for the initial migration. This can occur when the fix-up for the prior migration is still underway, but the VF KMD is migrated again. Consequently, this may lead to the possibility of sending two migration notifications (i.e., pending fix-up for the first migration and a second notification for the new migration). Upon receiving the first RES_FIX notification, the GuC will resume VF submission on the GPU, potentially resulting in undefined behavior, such as system hangs or crashes. To avoid this, post migration, a marker is sent to the GUC prior to the start of resource fixups to indicate start of resource fixups. The same marker is sent along with RESFIX_DONE notification so that GUC can avoid submitting jobs to HW in case of double migration. Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Tomasz Lis Reviewed-by: Michal Wajdeczko Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251201095011.21453-8-satyanarayana.k.v.p@intel.com --- .../gpu/drm/xe/abi/guc_actions_sriov_abi.h | 67 +++++++++-- drivers/gpu/drm/xe/xe_gt_sriov_vf.c | 112 ++++++++++++------ drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h | 5 + drivers/gpu/drm/xe/xe_sriov_vf.c | 62 +++++++++- 4 files changed, 195 insertions(+), 51 deletions(-) diff --git a/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h index 0b28659d94e9..d9f21202e1a9 100644 --- a/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h @@ -502,13 +502,17 @@ #define VF2GUC_VF_RESET_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 /** - * DOC: VF2GUC_NOTIFY_RESFIX_DONE + * DOC: VF2GUC_RESFIX_DONE * - * This action is used by VF to notify the GuC that the VF KMD has completed - * post-migration recovery steps. + * This action is used by VF to inform the GuC that the VF KMD has completed + * post-migration recovery steps. From GuC VF compatibility 1.27.0 onwards, it + * shall only be sent after posting RESFIX_START and that both @MARKER fields + * must match. * * This message must be sent as `MMIO HXG Message`_. * + * Updated since GuC VF compatibility 1.27.0. + * * +---+-------+--------------------------------------------------------------+ * | | Bits | Description | * +===+=======+==============================================================+ @@ -516,9 +520,11 @@ * | +-------+--------------------------------------------------------------+ * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | * | +-------+--------------------------------------------------------------+ - * | | 27:16 | DATA0 = MBZ | + * | | 27:16 | DATA0 = MARKER = MBZ (only prior 1.27.0) | * | +-------+--------------------------------------------------------------+ - * | | 15:0 | ACTION = _`GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE` = 0x5508 | + * | | 27:16 | DATA0 = MARKER - can't be zero (1.27.0+) | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_VF2GUC_RESFIX_DONE` = 0x5508 | * +---+-------+--------------------------------------------------------------+ * * +---+-------+--------------------------------------------------------------+ @@ -531,13 +537,13 @@ * | | 27:0 | DATA0 = MBZ | * +---+-------+--------------------------------------------------------------+ */ -#define GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE 0x5508u +#define GUC_ACTION_VF2GUC_RESFIX_DONE 0x5508u -#define VF2GUC_NOTIFY_RESFIX_DONE_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN -#define VF2GUC_NOTIFY_RESFIX_DONE_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define VF2GUC_RESFIX_DONE_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN +#define VF2GUC_RESFIX_DONE_REQUEST_MSG_0_MARKER GUC_HXG_REQUEST_MSG_0_DATA0 -#define VF2GUC_NOTIFY_RESFIX_DONE_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN -#define VF2GUC_NOTIFY_RESFIX_DONE_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 +#define VF2GUC_RESFIX_DONE_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define VF2GUC_RESFIX_DONE_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 /** * DOC: VF2GUC_QUERY_SINGLE_KLV @@ -656,4 +662,45 @@ #define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN #define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_0_USED GUC_HXG_RESPONSE_MSG_0_DATA0 +/** + * DOC: VF2GUC_RESFIX_START + * + * This action is used by VF to inform the GuC that the VF KMD will be starting + * post-migration recovery fixups. The @MARKER sent with this action must match + * with the MARKER posted in the VF2GUC_RESFIX_DONE message. + * + * This message must be sent as `MMIO HXG Message`_. + * + * Available since GuC VF compatibility 1.27.0. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MARKER - can't be zero | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_VF2GUC_RESFIX_START` = 0x550F | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_VF2GUC_RESFIX_START 0x550Fu + +#define VF2GUC_RESFIX_START_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN +#define VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER GUC_HXG_REQUEST_MSG_0_DATA0 + +#define VF2GUC_RESFIX_START_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define VF2GUC_RESFIX_START_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 + #endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index 97c29c55f885..5bf13e41de80 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -299,12 +299,13 @@ void xe_gt_sriov_vf_guc_versions(struct xe_gt *gt, *found = gt->sriov.vf.guc_version; } -static int guc_action_vf_notify_resfix_done(struct xe_guc *guc) +static int guc_action_vf_resfix_start(struct xe_guc *guc, u16 marker) { u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = { FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | - FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE), + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_RESFIX_START) | + FIELD_PREP(VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER, marker), }; int ret; @@ -313,28 +314,41 @@ static int guc_action_vf_notify_resfix_done(struct xe_guc *guc) return ret > 0 ? -EPROTO : ret; } -/** - * vf_notify_resfix_done - Notify GuC about resource fixups apply completed. - * @gt: the &xe_gt struct instance linked to target GuC - * - * Returns: 0 if the operation completed successfully, or a negative error - * code otherwise. - */ -static int vf_notify_resfix_done(struct xe_gt *gt) +static int vf_resfix_start(struct xe_gt *gt, u16 marker) { struct xe_guc *guc = >->uc.guc; - int err; xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); - err = guc_action_vf_notify_resfix_done(guc); - if (unlikely(err)) - xe_gt_sriov_err(gt, "Failed to notify GuC about resource fixup done (%pe)\n", - ERR_PTR(err)); - else - xe_gt_sriov_dbg_verbose(gt, "sent GuC resource fixup done\n"); + xe_gt_sriov_dbg_verbose(gt, "Sending resfix start marker %u\n", marker); - return err; + return guc_action_vf_resfix_start(guc, marker); +} + +static int guc_action_vf_resfix_done(struct xe_guc *guc, u16 marker) +{ + u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_RESFIX_DONE) | + FIELD_PREP(VF2GUC_RESFIX_DONE_REQUEST_MSG_0_MARKER, marker), + }; + int ret; + + ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request)); + + return ret > 0 ? -EPROTO : ret; +} + +static int vf_resfix_done(struct xe_gt *gt, u16 marker) +{ + struct xe_guc *guc = >->uc.guc; + + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + xe_gt_sriov_dbg_verbose(gt, "Sending resfix done marker %u\n", marker); + + return guc_action_vf_resfix_done(guc, marker); } static int guc_action_query_single_klv(struct xe_guc *guc, u32 key, @@ -1162,6 +1176,13 @@ static int vf_post_migration_fixups(struct xe_gt *gt) static void vf_post_migration_rearm(struct xe_gt *gt) { + /* + * Make sure interrupts on the new HW are properly set. The GuC IRQ + * must be working at this point, since the recovery did started, + * but the rest was not enabled using the procedure from spec. + */ + xe_irq_resume(gt_to_xe(gt)); + xe_guc_ct_restart(>->uc.guc.ct); xe_guc_submit_unpause_prepare_vf(>->uc.guc); } @@ -1183,37 +1204,40 @@ static void vf_post_migration_abort(struct xe_gt *gt) xe_guc_submit_pause_abort(>->uc.guc); } -static int vf_post_migration_notify_resfix_done(struct xe_gt *gt) +static int vf_post_migration_resfix_done(struct xe_gt *gt, u16 marker) { - bool skip_resfix = false; - spin_lock_irq(>->sriov.vf.migration.lock); - if (gt->sriov.vf.migration.recovery_queued) { - skip_resfix = true; - xe_gt_sriov_dbg(gt, "another recovery imminent, resfix skipped\n"); - } else { + if (gt->sriov.vf.migration.recovery_queued) + xe_gt_sriov_dbg(gt, "another recovery imminent\n"); + else WRITE_ONCE(gt->sriov.vf.migration.recovery_inprogress, false); - } spin_unlock_irq(>->sriov.vf.migration.lock); - if (skip_resfix) - return -EAGAIN; + return vf_resfix_done(gt, marker); +} - /* - * Make sure interrupts on the new HW are properly set. The GuC IRQ - * must be working at this point, since the recovery did started, - * but the rest was not enabled using the procedure from spec. - */ - xe_irq_resume(gt_to_xe(gt)); +static int vf_post_migration_resfix_start(struct xe_gt *gt, u16 marker) +{ + return vf_resfix_start(gt, marker); +} - return vf_notify_resfix_done(gt); +static u16 vf_post_migration_next_resfix_marker(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + + BUILD_BUG_ON(1 + ((typeof(gt->sriov.vf.migration.resfix_marker))~0) > + FIELD_MAX(VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER)); + + /* add 1 to avoid zero-marker */ + return 1 + gt->sriov.vf.migration.resfix_marker++; } static void vf_post_migration_recovery(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); - int err; + u16 marker; bool retry; + int err; xe_gt_sriov_dbg(gt, "migration recovery in progress\n"); @@ -1227,15 +1251,27 @@ static void vf_post_migration_recovery(struct xe_gt *gt) goto fail; } + marker = vf_post_migration_next_resfix_marker(gt); + + err = vf_post_migration_resfix_start(gt, marker); + if (unlikely(err)) { + xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_START step (%pe)\n", + ERR_PTR(err)); + goto fail; + } + err = vf_post_migration_fixups(gt); if (err) goto fail; vf_post_migration_rearm(gt); - err = vf_post_migration_notify_resfix_done(gt); - if (err && err != -EAGAIN) + err = vf_post_migration_resfix_done(gt, marker); + if (err) { + xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_DONE step (%pe)\n", + ERR_PTR(err)); goto fail; + } vf_post_migration_kickstart(gt); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h index 420b0e6089de..db2f8b3ed3e9 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h @@ -52,6 +52,11 @@ struct xe_gt_sriov_vf_migration { wait_queue_head_t wq; /** @scratch: Scratch memory for VF recovery */ void *scratch; + /** + * @resfix_marker: Marker sent on start and on end of post-migration + * steps. + */ + u8 resfix_marker; /** @recovery_teardown: VF post migration recovery is being torn down */ bool recovery_teardown; /** @recovery_queued: VF post migration recovery in queued */ diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c index d56b8cfea50b..86423a799d57 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf.c @@ -49,11 +49,13 @@ * * As soon as Virtual GPU of the VM starts, the VF driver within receives * the MIGRATED interrupt and schedules post-migration recovery worker. - * That worker queries GuC for new provisioning (using MMIO communication), + * That worker sends `VF2GUC_RESFIX_START` action along with non-zero + * marker, queries GuC for new provisioning (using MMIO communication), * and applies fixups to any non-virtualized resources used by the VF. * * When the VF driver is ready to continue operation on the newly connected - * hardware, it sends `VF2GUC_NOTIFY_RESFIX_DONE` which causes it to + * hardware, it sends `VF2GUC_RESFIX_DONE` action along with the same + * marker which was sent with `VF2GUC_RESFIX_START` which causes it to * enter the long awaited `VF_RUNNING` state, and therefore start handling * CTB messages and scheduling workloads from the VF:: * @@ -102,12 +104,17 @@ * | [ ] new VF provisioning [ ] * | [ ]---------------------------> [ ] * | | [ ] + * | | VF2GUC_RESFIX_START [ ] + * | [ ] <---------------------------[ ] + * | [ ] [ ] + * | [ ] success [ ] + * | [ ]---------------------------> [ ] * | | VF driver applies post [ ] * | | migration fixups -------[ ] * | | | [ ] * | | -----> [ ] * | | [ ] - * | | VF2GUC_NOTIFY_RESFIX_DONE [ ] + * | | VF2GUC_RESFIX_DONE [ ] * | [ ] <---------------------------[ ] * | [ ] [ ] * | [ ] GuC sets new VF state to [ ] @@ -118,6 +125,55 @@ * | [ ]---------------------------> [ ] * | | | * | | | + * + * Handling of VF double migration flow is shown below:: + * + * GuC1 VF + * | | + * | [ ]<--- start fixups + * | VF2GUC_RESFIX_START(marker) [ ] + * [ ] <-------------------------------------------[ ] + * [ ] [ ] + * [ ]---\ [ ] + * [ ] store marker [ ] + * [ ]<--/ [ ] + * [ ] [ ] + * [ ] success [ ] + * [ ] ------------------------------------------> [ ] + * | [ ] + * | [ ]---\ + * | [ ] do fixups + * | [ ]<--/ + * | [ ] + * -------------- VF paused / saved ---------------- + * : + * + * GuC2 + * | + * ----------------- VF restored ------------------ + * | + * [ ] + * [ ]---\ + * [ ] reset marker + * [ ]<--/ + * [ ] + * ----------------- VF resumed ------------------ + * | [ ] + * | [ ] + * | VF2GUC_RESFIX_DONE(marker) [ ] + * [ ] <-------------------------------------------[ ] + * [ ] [ ] + * [ ]---\ [ ] + * [ ] check marker [ ] + * [ ] (mismatch) [ ] + * [ ]<--/ [ ] + * [ ] [ ] + * [ ] RESPONSE_VF_MIGRATED [ ] + * [ ] ------------------------------------------> [ ] + * | [ ]---\ + * | [ ] reschedule fixups + * | [ ]<--/ + * | | */ /** From 75e7d26281da46bc8570c61f44fe6093f120963a Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Mon, 1 Dec 2025 15:20:15 +0530 Subject: [PATCH 088/187] drm/xe/vf: Requeue recovery on GuC MIGRATION error during VF post-migration Handle GuC response `XE_GUC_RESPONSE_VF_MIGRATED` as a special case in the VF post-migration recovery flow. When this error occurs, it indicates that a new migration was detected while the resource fixup process was still in progress. Instead of failing immediately, requeue the VF into the recovery path to allow proper handling of the new migration event. This improves robustness of VF recovery in SR-IOV environments where migrations can overlap with resource fixup steps. Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Tomasz Lis Reviewed-by: Michal Wajdeczko Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251201095011.21453-9-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_vf.c | 3 +++ drivers/gpu/drm/xe/xe_guc.c | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index 5bf13e41de80..0b3ecb000ff7 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -1268,6 +1268,9 @@ static void vf_post_migration_recovery(struct xe_gt *gt) err = vf_post_migration_resfix_done(gt, marker); if (err) { + if (err == -EREMCHG) + goto queue; + xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_DONE step (%pe)\n", ERR_PTR(err)); goto fail; diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index 88376bc2a483..f0407bab9a0c 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -1484,6 +1484,12 @@ int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request, u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header); u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header); + if (unlikely(error == XE_GUC_RESPONSE_VF_MIGRATED)) { + xe_gt_dbg(gt, "GuC mmio request %#x rejected due to MIGRATION (hint %#x)\n", + request[0], hint); + return -EREMCHG; + } + xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n", request[0], error, hint); return -ENXIO; From 4c2768704710a5d4585941288e6a8159dd6dd33d Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Mon, 1 Dec 2025 15:20:16 +0530 Subject: [PATCH 089/187] drm/xe/vf: Add debugfs entries to test VF double migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VF migration sends a marker to the GUC before resource fixups begin, and repeats the marker with the RESFIX_DONE notification. This prevents the GUC from submitting jobs during double migration events. To reliably test double migration, a second migration must be triggered while fixups from the first migration are still in progress. Since fixups complete quickly, reproducing this scenario is difficult. Introduce debugfs controls to add delays in the post-fixup phase, creating a deterministic window for subsequent migrations. New debugfs entries: /sys/kernel/debug/dri/BDF/ ├── tile0 │ ├─gt0 │ │ ├──vf │ │ │ ├── resfix_stoppers resfix_stoppers: Predefined checkpoints that allow the migration process to pause at specific stages. The stages are given below. VF_MIGRATION_WAIT_RESFIX_START - BIT(0) VF_MIGRATION_WAIT_FIXUPS - BIT(1) VF_MIGRATION_WAIT_RESTART_JOBS - BIT(2) VF_MIGRATION_WAIT_RESFIX_DONE - BIT(3) Each state will pause with a 1-second delay per iteration, continuing until its corresponding bit is cleared. Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Tomasz Lis Acked-by: Adam Miszczak Reviewed-by: Michal Wajdeczko Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251201095011.21453-10-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_vf.c | 40 +++++++++++++++++++++ drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c | 12 +++++++ drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h | 8 +++++ 3 files changed, 60 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index 0b3ecb000ff7..3c806c8e5f3e 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -41,6 +42,37 @@ #define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo))) +#ifdef CONFIG_DRM_XE_DEBUG +enum VF_MIGRATION_WAIT_POINTS { + VF_MIGRATION_WAIT_RESFIX_START = BIT(0), + VF_MIGRATION_WAIT_FIXUPS = BIT(1), + VF_MIGRATION_WAIT_RESTART_JOBS = BIT(2), + VF_MIGRATION_WAIT_RESFIX_DONE = BIT(3), +}; + +#define VF_MIGRATION_WAIT_DELAY_IN_MS 1000 +static void vf_post_migration_inject_wait(struct xe_gt *gt, + enum VF_MIGRATION_WAIT_POINTS wait) +{ + while (gt->sriov.vf.migration.debug.resfix_stoppers & wait) { + xe_gt_dbg(gt, + "*TESTING* injecting %u ms delay due to resfix_stoppers=%#x, to continue clear %#x\n", + VF_MIGRATION_WAIT_DELAY_IN_MS, + gt->sriov.vf.migration.debug.resfix_stoppers, wait); + + msleep(VF_MIGRATION_WAIT_DELAY_IN_MS); + } +} + +#define VF_MIGRATION_INJECT_WAIT(gt, _POS) ({ \ + struct xe_gt *__gt = (gt); \ + vf_post_migration_inject_wait(__gt, VF_MIGRATION_WAIT_##_POS); \ + }) + +#else +#define VF_MIGRATION_INJECT_WAIT(_gt, ...) typecheck(struct xe_gt *, (_gt)) +#endif + static int guc_action_vf_reset(struct xe_guc *guc) { u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = { @@ -320,6 +352,8 @@ static int vf_resfix_start(struct xe_gt *gt, u16 marker) xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt))); + VF_MIGRATION_INJECT_WAIT(gt, RESFIX_START); + xe_gt_sriov_dbg_verbose(gt, "Sending resfix start marker %u\n", marker); return guc_action_vf_resfix_start(guc, marker); @@ -1158,6 +1192,8 @@ static int vf_post_migration_fixups(struct xe_gt *gt) void *buf = gt->sriov.vf.migration.scratch; int err; + VF_MIGRATION_INJECT_WAIT(gt, FIXUPS); + /* xe_gt_sriov_vf_query_config will fixup the GGTT addresses */ err = xe_gt_sriov_vf_query_config(gt); if (err) @@ -1176,6 +1212,8 @@ static int vf_post_migration_fixups(struct xe_gt *gt) static void vf_post_migration_rearm(struct xe_gt *gt) { + VF_MIGRATION_INJECT_WAIT(gt, RESTART_JOBS); + /* * Make sure interrupts on the new HW are properly set. The GuC IRQ * must be working at this point, since the recovery did started, @@ -1206,6 +1244,8 @@ static void vf_post_migration_abort(struct xe_gt *gt) static int vf_post_migration_resfix_done(struct xe_gt *gt, u16 marker) { + VF_MIGRATION_INJECT_WAIT(gt, RESFIX_DONE); + spin_lock_irq(>->sriov.vf.migration.lock); if (gt->sriov.vf.migration.recovery_queued) xe_gt_sriov_dbg(gt, "another recovery imminent\n"); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c index 2ed5b6780d30..507718326e1f 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c @@ -69,4 +69,16 @@ void xe_gt_sriov_vf_debugfs_register(struct xe_gt *gt, struct dentry *root) vfdentry->d_inode->i_private = gt; drm_debugfs_create_files(vf_info, ARRAY_SIZE(vf_info), vfdentry, minor); + + /* + * /sys/kernel/debug/dri/BDF/ + * ├── tile0 + * ├── gt0 + * ├── vf + * ├── resfix_stoppers + */ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { + debugfs_create_x8("resfix_stoppers", 0600, vfdentry, + >->sriov.vf.migration.debug.resfix_stoppers); + } } diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h index db2f8b3ed3e9..510c33116fbd 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h @@ -52,6 +52,14 @@ struct xe_gt_sriov_vf_migration { wait_queue_head_t wq; /** @scratch: Scratch memory for VF recovery */ void *scratch; + /** @debug: Debug hooks for delaying migration */ + struct { + /** + * @debug.resfix_stoppers: Stop and wait at different stages + * during post migration recovery + */ + u8 resfix_stoppers; + } debug; /** * @resfix_marker: Marker sent on start and on end of post-migration * steps. From 43109e398d2d4ac8a932ac46fde34e55cec53d4c Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Fri, 28 Nov 2025 13:52:12 +0530 Subject: [PATCH 090/187] drm/xe/gt: Use scope-based forcewake Switch runtime PM code to use scope-based forcewake for consistency with other parts of the driver. Signed-off-by: Raag Jadav Reviewed-by: Matt Roper Link: https://patch.msgid.link/20251128082212.294592-1-raag.jadav@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_gt.c | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 996756d1fbd5..1bfde7af90b3 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -964,26 +964,20 @@ int xe_gt_resume(struct xe_gt *gt) */ int xe_gt_runtime_suspend(struct xe_gt *gt) { - unsigned int fw_ref; - int err = -ETIMEDOUT; - xe_gt_dbg(gt, "runtime suspending\n"); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) - goto err_force_wake; + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + xe_gt_err(gt, "runtime suspend failed (%pe)\n", ERR_PTR(-ETIMEDOUT)); + return -ETIMEDOUT; + } xe_uc_runtime_suspend(>->uc); xe_gt_disable_host_l2_vram(gt); - xe_force_wake_put(gt_to_fw(gt), fw_ref); xe_gt_dbg(gt, "runtime suspended\n"); return 0; - -err_force_wake: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - return err; } /** @@ -994,26 +988,20 @@ int xe_gt_runtime_suspend(struct xe_gt *gt) */ int xe_gt_runtime_resume(struct xe_gt *gt) { - unsigned int fw_ref; - int err = -ETIMEDOUT; - xe_gt_dbg(gt, "runtime resuming\n"); - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) - goto err_force_wake; + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) { + xe_gt_err(gt, "runtime resume failed (%pe)\n", ERR_PTR(-ETIMEDOUT)); + return -ETIMEDOUT; + } xe_gt_enable_host_l2_vram(gt); xe_uc_runtime_resume(>->uc); - xe_force_wake_put(gt_to_fw(gt), fw_ref); xe_gt_dbg(gt, "runtime resumed\n"); return 0; - -err_force_wake: - xe_force_wake_put(gt_to_fw(gt), fw_ref); - return err; } struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, From 688035f83ef0a3b492b9ff9748518d6d45b4f107 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 2 Dec 2025 14:25:52 -0800 Subject: [PATCH 091/187] drm/xe/sync: Use for_each_tlb_inval() to calculate invalidation fences xe_sync_in_fence_get() uses the same kind of mismatched fence array allocation vs looping logic that was previously noted and changed by commit 0a4c2ddc711a ("drm/xe/vm: Use for_each_tlb_inval() to calculate invalidation fences"). As with that commit, the mismatch doesn't cause any problem at the moment since for_each_tlb_inval() loops the same number of times as XE_MAX_GT_PER_TILE (2). However we don't want to assume that these will always be the same in the future, so switch to using for_each_tlb_inval() in both places to future-proof the code. Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251202222551.1858930-2-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_sync.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index ff74528ca0c6..1fc4fa278b78 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -311,8 +311,11 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, struct xe_tile *tile; u8 id; - for_each_tile(tile, vm->xe, id) - num_fence += (1 + XE_MAX_GT_PER_TILE); + for_each_tile(tile, vm->xe, id) { + num_fence++; + for_each_tlb_inval(i) + num_fence++; + } fences = kmalloc_array(num_fence, sizeof(*fences), GFP_KERNEL); From 76ce2313709f13a6adbcaa1a43a8539c8f509f6a Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Tue, 2 Dec 2025 17:18:09 -0800 Subject: [PATCH 092/187] drm/xe: Do not reference loop variable directly Do not reference the loop variable job after the loop has exited. Instead, save the job from the last iteration of the loop. Fixes: 00937fe1921a ("drm/xe/vf: Start re-emission from first unsignaled job during VF migration") Reported-by: kernel test robot Reported-by: Dan Carpenter Closes: https://lore.kernel.org/r/202511291102.jnnKP6IB-lkp@intel.com/ Signed-off-by: Matthew Brost Reviewed-by: Dnyaneshwar Bhadane Link: https://patch.msgid.link/20251203011809.968893-1-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 3ca2558c8c96..f3f2c8556a66 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -2264,10 +2264,11 @@ static void guc_exec_queue_unpause_prepare(struct xe_guc *guc, struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; - struct xe_sched_job *job = NULL; + struct xe_sched_job *job = NULL, *__job; bool restore_replay = false; - list_for_each_entry(job, &sched->base.pending_list, drm.list) { + list_for_each_entry(__job, &sched->base.pending_list, drm.list) { + job = __job; restore_replay |= job->restore_replay; if (restore_replay) { xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", From 54da99e5c3a7e114df8dbbd42a0fee2b3ed8aa15 Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Fri, 21 Nov 2025 13:59:17 -0800 Subject: [PATCH 093/187] drm/xe/xe3: Remove graphics IP 30.01 from Wa_18041344222 IP list As per the updated WA database, Wa_18041344222 no longer applies to graphics IP version 30.01. Remove it. Cc: Matt Roper Cc: Gustavo Sousa Signed-off-by: Harish Chegondi Reviewed-by: Matt Roper Link: https://patch.msgid.link/280ab3e8dce8d7a40540ae634a5432694ac17ab0.1763762330.git.harish.chegondi@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_wa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index d606b058d588..914244943ac8 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -698,7 +698,7 @@ static const struct xe_rtp_entry_sr engine_was[] = { XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE)) }, { XE_RTP_NAME("18041344222"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3001), + XE_RTP_RULES(GRAPHICS_VERSION(3000), FUNC(xe_rtp_match_first_render_or_compute), FUNC(xe_rtp_match_not_sriov_vf), FUNC(xe_rtp_match_gt_has_discontiguous_dss_groups)), From 78d91ba6bd7968d4750dad57c62bf5225ddcb388 Mon Sep 17 00:00:00 2001 From: Sanjay Yadav Date: Thu, 4 Dec 2025 09:34:03 +0530 Subject: [PATCH 094/187] drm/xe/uapi: Add NO_COMPRESSION BO flag and query capability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION to let userspace opt out of CCS compression on a per-BO basis. When set, the driver maps this to XE_BO_FLAG_NO_COMPRESSION, skips CCS metadata allocation/clearing, and rejects compressed PAT indices at vm_bind. This avoids extra memory ops and manual CCS state handling for buffers. To allow userspace to detect at runtime whether the kernel supports this feature, add DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT and expose it via query_config() on Xe2+ platforms. Mesa PR: https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/38425 IGT PR: https://patchwork.freedesktop.org/patch/685180/ v2 - Changed error code from -EINVAL to -EOPNOTSUPP for unsupported flag usage on pre-Xe2 platforms - Fixed checkpatch warning in xe_vm.c - Fixed kernel-doc formatting in xe_drm.h v3 - Rebase - Updated commit title and description - Added UAPI for DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT and exposed it via query_config() v4 - Rebase v5 - Included Mesa PR and IGT PR in the commit description - Used xe_pat_index_get_comp_en() to extract the compression v6 - Added XE_IOCTL_DBG() checks for argument validation Suggested-by: Matthew Auld Suggested-by: José Roberto de Souza Acked-by: José Roberto de Souza Reviewed-by: Matthew Auld Signed-off-by: Sanjay Yadav Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20251204040402.2692921-2-sanjay.kumar.yadav@intel.com --- drivers/gpu/drm/xe/xe_bo.c | 15 +++++++++++++-- drivers/gpu/drm/xe/xe_bo.h | 1 + drivers/gpu/drm/xe/xe_query.c | 3 +++ drivers/gpu/drm/xe/xe_vm.c | 4 ++++ include/uapi/drm/xe_drm.h | 16 ++++++++++++++++ 5 files changed, 37 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index b67fd337ff19..6280e6a013ff 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -3178,7 +3178,8 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, args->flags & ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING | DRM_XE_GEM_CREATE_FLAG_SCANOUT | - DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM))) + DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM | + DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION))) return -EINVAL; if (XE_IOCTL_DBG(xe, args->handle)) @@ -3200,6 +3201,12 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT) bo_flags |= XE_BO_FLAG_SCANOUT; + if (args->flags & DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION) { + if (XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20)) + return -EOPNOTSUPP; + bo_flags |= XE_BO_FLAG_NO_COMPRESSION; + } + bo_flags |= args->placement << (ffs(XE_BO_FLAG_SYSTEM) - 1); /* CCS formats need physical placement at a 64K alignment in VRAM. */ @@ -3521,8 +3528,12 @@ bool xe_bo_needs_ccs_pages(struct xe_bo *bo) * Compression implies coh_none, therefore we know for sure that WB * memory can't currently use compression, which is likely one of the * common cases. + * Additionally, userspace may explicitly request no compression via the + * DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION flag, which should also disable + * CCS usage. */ - if (bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB) + if (bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB || + bo->flags & XE_BO_FLAG_NO_COMPRESSION) return false; return true; diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 911d5b90461a..8ab4474129c3 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -50,6 +50,7 @@ #define XE_BO_FLAG_GGTT3 BIT(23) #define XE_BO_FLAG_CPU_ADDR_MIRROR BIT(24) #define XE_BO_FLAG_FORCE_USER_VRAM BIT(25) +#define XE_BO_FLAG_NO_COMPRESSION BIT(26) /* this one is trigger internally only */ #define XE_BO_FLAG_INTERNAL_TEST BIT(30) diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index a7bf1fd6dd6a..6667403a8814 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -338,6 +338,9 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) if (xe->info.has_usm && IS_ENABLED(CONFIG_DRM_XE_GPUSVM)) config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR; + if (GRAPHICS_VER(xe) >= 20) + config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= + DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT; config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY; config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] = diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 00ffd3f03983..c2012d20faa6 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -3501,6 +3501,10 @@ static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo, { u16 coh_mode; + if (XE_IOCTL_DBG(xe, (bo->flags & XE_BO_FLAG_NO_COMPRESSION) && + xe_pat_index_get_comp_en(xe, pat_index))) + return -EINVAL; + if (XE_IOCTL_DBG(xe, range > xe_bo_size(bo)) || XE_IOCTL_DBG(xe, obj_offset > xe_bo_size(bo) - range)) { diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 37881b1eb6ba..0d99bb0cd20a 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -407,6 +407,9 @@ struct drm_xe_query_mem_regions { * has low latency hint support * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the * device has CPU address mirroring support + * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT - Flag is set if the + * device supports the userspace hint %DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION. + * This is exposed only on Xe2+. * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address @@ -425,6 +428,7 @@ struct drm_xe_query_config { #define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) #define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) #define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2) + #define DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT (1 << 3) #define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2 #define DRM_XE_QUERY_CONFIG_VA_BITS 3 #define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4 @@ -795,6 +799,17 @@ struct drm_xe_device_query { * need to use VRAM for display surfaces, therefore the kernel requires * setting this flag for such objects, otherwise an error is thrown on * small-bar systems. + * - %DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION - Allows userspace to + * hint that compression (CCS) should be disabled for the buffer being + * created. This can avoid unnecessary memory operations and CCS state + * management. + * On pre-Xe2 platforms, this flag is currently rejected as compression + * control is not supported via PAT index. On Xe2+ platforms, compression + * is controlled via PAT entries. If this flag is set, the driver will reject + * any VM bind that requests a PAT index enabling compression for this BO. + * Note: On dGPU platforms, there is currently no change in behavior with + * this flag, but future improvements may leverage it. The current benefit is + * primarily applicable to iGPU platforms. * * @cpu_caching supports the following values: * - %DRM_XE_GEM_CPU_CACHING_WB - Allocate the pages with write-back @@ -841,6 +856,7 @@ struct drm_xe_gem_create { #define DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING (1 << 0) #define DRM_XE_GEM_CREATE_FLAG_SCANOUT (1 << 1) #define DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM (1 << 2) +#define DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION (1 << 3) /** * @flags: Flags, currently a mask of memory instances of where BO can * be placed From ef7de33544a7a6783d7afe09496da362d1e90ba1 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 4 Dec 2025 10:41:36 +0100 Subject: [PATCH 095/187] drm/xe/pf: fix VFIO link error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Makefile logic for building xe_sriov_vfio.o was added incorrectly, as setting CONFIG_XE_VFIO_PCI=m means it doesn't get included into a built-in xe driver: ERROR: modpost: "xe_sriov_vfio_stop_copy_enter" [drivers/vfio/pci/xe/xe-vfio-pci.ko] undefined! ERROR: modpost: "xe_sriov_vfio_stop_copy_exit" [drivers/vfio/pci/xe/xe-vfio-pci.ko] undefined! ERROR: modpost: "xe_sriov_vfio_suspend_device" [drivers/vfio/pci/xe/xe-vfio-pci.ko] undefined! ERROR: modpost: "xe_sriov_vfio_wait_flr_done" [drivers/vfio/pci/xe/xe-vfio-pci.ko] undefined! ERROR: modpost: "xe_sriov_vfio_error" [drivers/vfio/pci/xe/xe-vfio-pci.ko] undefined! ERROR: modpost: "xe_sriov_vfio_resume_data_enter" [drivers/vfio/pci/xe/xe-vfio-pci.ko] undefined! ERROR: modpost: "xe_sriov_vfio_resume_device" [drivers/vfio/pci/xe/xe-vfio-pci.ko] undefined! ERROR: modpost: "xe_sriov_vfio_resume_data_exit" [drivers/vfio/pci/xe/xe-vfio-pci.ko] undefined! ERROR: modpost: "xe_sriov_vfio_data_write" [drivers/vfio/pci/xe/xe-vfio-pci.ko] undefined! ERROR: modpost: "xe_sriov_vfio_migration_supported" [drivers/vfio/pci/xe/xe-vfio-pci.ko] undefined! WARNING: modpost: suppressed 3 unresolved symbol warnings because there were too many) Check for CONFIG_XE_VFIO_PCI being enabled in the Makefile to decide whether to include the object instead. Fixes: 17f22465c5a5 ("drm/xe/pf: Export helpers for VFIO") Signed-off-by: Arnd Bergmann Reviewed-by: Michal Wajdeczko Signed-off-by: Thomas Hellström Link: https://patch.msgid.link/20251204094154.1029357-1-arnd@kernel.org --- drivers/gpu/drm/xe/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index dfc2ded01455..e5f3c2ec9e9a 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -185,8 +185,8 @@ xe-$(CONFIG_PCI_IOV) += \ xe_sriov_pf_sysfs.o \ xe_tile_sriov_pf_debugfs.o -ifeq ($(CONFIG_PCI_IOV),y) - xe-$(CONFIG_XE_VFIO_PCI) += xe_sriov_vfio.o +ifdef CONFIG_XE_VFIO_PCI + xe-$(CONFIG_PCI_IOV) += xe_sriov_vfio.o endif # include helpers for tests even when XE is built-in From 29bce9c8b41d5c378263a927acb9a9074d0e7a0e Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 4 Dec 2025 10:46:58 +0100 Subject: [PATCH 096/187] drm/xe: fix drm_gpusvm_init() arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Xe driver fails to build when CONFIG_DRM_XE_GPUSVM is disabled but CONFIG_DRM_GPUSVM is turned on, due to the clash of two commits: In file included from drivers/gpu/drm/xe/xe_vm_madvise.c:8: drivers/gpu/drm/xe/xe_svm.h: In function 'xe_svm_init': include/linux/stddef.h:8:14: error: passing argument 5 of 'drm_gpusvm_init' makes integer from pointer without a cast [-Wint-conversion] drivers/gpu/drm/xe/xe_svm.h:217:38: note: in expansion of macro 'NULL' 217 | NULL, NULL, 0, 0, 0, NULL, NULL, 0); | ^~~~ In file included from drivers/gpu/drm/xe/xe_bo_types.h:11, from drivers/gpu/drm/xe/xe_bo.h:11, from drivers/gpu/drm/xe/xe_vm_madvise.c:11: include/drm/drm_gpusvm.h:254:35: note: expected 'long unsigned int' but argument is of type 'void *' 254 | unsigned long mm_start, unsigned long mm_range, | ~~~~~~~~~~~~~~^~~~~~~~ In file included from drivers/gpu/drm/xe/xe_vm_madvise.c:14: drivers/gpu/drm/xe/xe_svm.h:216:16: error: too many arguments to function 'drm_gpusvm_init'; expected 10, have 11 216 | return drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", &vm->xe->drm, | ^~~~~~~~~~~~~~~ 217 | NULL, NULL, 0, 0, 0, NULL, NULL, 0); | ~ include/drm/drm_gpusvm.h:251:5: note: declared here Adapt the caller to the new argument list by removing the extraneous NULL argument. Fixes: 9e9787414882 ("drm/xe/userptr: replace xe_hmm with gpusvm") Fixes: 10aa5c806030 ("drm/gpusvm, drm/xe: Fix userptr to not allow device private pages") Signed-off-by: Arnd Bergmann Reviewed-by: Thomas Hellström Signed-off-by: Thomas Hellström Link: https://patch.msgid.link/20251204094704.1030933-1-arnd@kernel.org --- drivers/gpu/drm/xe/xe_svm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index 0955d2ac8d74..fa757dd07954 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -214,7 +214,7 @@ int xe_svm_init(struct xe_vm *vm) { #if IS_ENABLED(CONFIG_DRM_GPUSVM) return drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", &vm->xe->drm, - NULL, NULL, 0, 0, 0, NULL, NULL, 0); + NULL, 0, 0, 0, NULL, NULL, 0); #else return 0; #endif From 89f0b56f77c96c022cbc3b22d42664d0e93e3154 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Mon, 1 Dec 2025 18:51:11 -0800 Subject: [PATCH 097/187] drm/xe/oa: Use explicit struct initialization for struct xe_oa_regs Use explicit struct initialization for struct xe_oa_regs to reduce chance of error. Also add .oa_mmio_trg field to struct for completeness. Signed-off-by: Ashutosh Dixit Reviewed-by: Umesh Nerlige Ramappa Link: https://patch.msgid.link/20251202025115.373546-2-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_oa.c | 38 +++++++++++++++++--------------- drivers/gpu/drm/xe/xe_oa_types.h | 1 + 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 890c363282ae..75db5530cb55 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -2566,30 +2566,32 @@ static u32 __hwe_oa_unit(struct xe_hw_engine *hwe) static struct xe_oa_regs __oam_regs(u32 base) { return (struct xe_oa_regs) { - base, - OAM_HEAD_POINTER(base), - OAM_TAIL_POINTER(base), - OAM_BUFFER(base), - OAM_CONTEXT_CONTROL(base), - OAM_CONTROL(base), - OAM_DEBUG(base), - OAM_STATUS(base), - OAM_CONTROL_COUNTER_SEL_MASK, + .base = base, + .oa_head_ptr = OAM_HEAD_POINTER(base), + .oa_tail_ptr = OAM_TAIL_POINTER(base), + .oa_buffer = OAM_BUFFER(base), + .oa_ctx_ctrl = OAM_CONTEXT_CONTROL(base), + .oa_ctrl = OAM_CONTROL(base), + .oa_debug = OAM_DEBUG(base), + .oa_status = OAM_STATUS(base), + .oa_mmio_trg = OAM_MMIO_TRG(base), + .oa_ctrl_counter_select_mask = OAM_CONTROL_COUNTER_SEL_MASK, }; } static struct xe_oa_regs __oag_regs(void) { return (struct xe_oa_regs) { - 0, - OAG_OAHEADPTR, - OAG_OATAILPTR, - OAG_OABUFFER, - OAG_OAGLBCTXCTRL, - OAG_OACONTROL, - OAG_OA_DEBUG, - OAG_OASTATUS, - OAG_OACONTROL_OA_COUNTER_SEL_MASK, + .base = 0, + .oa_head_ptr = OAG_OAHEADPTR, + .oa_tail_ptr = OAG_OATAILPTR, + .oa_buffer = OAG_OABUFFER, + .oa_ctx_ctrl = OAG_OAGLBCTXCTRL, + .oa_ctrl = OAG_OACONTROL, + .oa_debug = OAG_OA_DEBUG, + .oa_status = OAG_OASTATUS, + .oa_mmio_trg = OAG_MMIOTRIGGER, + .oa_ctrl_counter_select_mask = OAG_OACONTROL_OA_COUNTER_SEL_MASK, }; } diff --git a/drivers/gpu/drm/xe/xe_oa_types.h b/drivers/gpu/drm/xe/xe_oa_types.h index cf080f412189..08cc8d7c2215 100644 --- a/drivers/gpu/drm/xe/xe_oa_types.h +++ b/drivers/gpu/drm/xe/xe_oa_types.h @@ -87,6 +87,7 @@ struct xe_oa_regs { struct xe_reg oa_ctrl; struct xe_reg oa_debug; struct xe_reg oa_status; + struct xe_reg oa_mmio_trg; u32 oa_ctrl_counter_select_mask; }; From 16e076b036583702bb47554d3931b5e674dd9a8e Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Mon, 1 Dec 2025 18:51:12 -0800 Subject: [PATCH 098/187] drm/xe/oa/uapi: Add gt_id to struct drm_xe_oa_unit gt_id was previously omitted from 'struct drm_xe_oa_unit' because it could be determine from hwe's attached to the OA unit. However, we now have OA units which don't have any hwe's attached to them. Hence add gt_id to 'struct drm_xe_oa_unit' in order to provide this needed information to userspace. Signed-off-by: Ashutosh Dixit Reviewed-by: Umesh Nerlige Ramappa Link: https://patch.msgid.link/20251202025115.373546-3-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_query.c | 4 +++- include/uapi/drm/xe_drm.h | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 6667403a8814..75490683bad2 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -685,7 +685,9 @@ static int query_oa_units(struct xe_device *xe, du->capabilities = DRM_XE_OA_CAPS_BASE | DRM_XE_OA_CAPS_SYNCS | DRM_XE_OA_CAPS_OA_BUFFER_SIZE | DRM_XE_OA_CAPS_WAIT_NUM_REPORTS | - DRM_XE_OA_CAPS_OAM; + DRM_XE_OA_CAPS_OAM | + DRM_XE_OA_CAPS_OA_UNIT_GT_ID; + du->gt_id = u->gt->info.id; j = 0; for_each_hw_engine(hwe, gt, hwe_id) { if (!xe_hw_engine_is_reserved(hwe) && diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 0d99bb0cd20a..876a076fa6c0 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1697,12 +1697,19 @@ struct drm_xe_oa_unit { #define DRM_XE_OA_CAPS_OA_BUFFER_SIZE (1 << 2) #define DRM_XE_OA_CAPS_WAIT_NUM_REPORTS (1 << 3) #define DRM_XE_OA_CAPS_OAM (1 << 4) +#define DRM_XE_OA_CAPS_OA_UNIT_GT_ID (1 << 5) /** @oa_timestamp_freq: OA timestamp freq */ __u64 oa_timestamp_freq; + /** @gt_id: gt id for this OA unit */ + __u16 gt_id; + + /** @reserved1: MBZ */ + __u16 reserved1[3]; + /** @reserved: MBZ */ - __u64 reserved[4]; + __u64 reserved[3]; /** @num_engines: number of engines in @eci array */ __u64 num_engines; From c7e269aa565f4f3fce652ff690db3676c50f5098 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Mon, 1 Dec 2025 18:51:13 -0800 Subject: [PATCH 099/187] drm/xe/oa: Allow exec_queue's to be specified only for OAG OA unit Exec_queue's are only used for OAR/OAC functionality for OAG unit. Make this requirement explicit, which avoids complications in the code for other (non-OAG) OA units. Signed-off-by: Ashutosh Dixit Reviewed-by: Umesh Nerlige Ramappa Link: https://patch.msgid.link/20251202025115.373546-4-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_oa.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 75db5530cb55..d63c04e2d492 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -2032,6 +2032,10 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f return ret; if (param.exec_queue_id > 0) { + /* An exec_queue is only needed for OAR/OAC functionality on OAG */ + if (XE_IOCTL_DBG(oa->xe, param.oa_unit->type != DRM_XE_OA_UNIT_TYPE_OAG)) + return -EINVAL; + param.exec_q = xe_exec_queue_lookup(xef, param.exec_queue_id); if (XE_IOCTL_DBG(oa->xe, !param.exec_q)) return -ENOENT; From ed455775c5a68b75e5f6ad6c8e0e3e9c98fd3f64 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Mon, 1 Dec 2025 18:51:14 -0800 Subject: [PATCH 100/187] drm/xe/rtp: Refactor OAG MMIO trigger register whitelisting Minor refactor of OAG MMIO trigger register whitelisting for code reuse with OAM MMIO trigger register whitelisting. Signed-off-by: Ashutosh Dixit Reviewed-by: Umesh Nerlige Ramappa Link: https://patch.msgid.link/20251202025115.373546-5-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_reg_whitelist.c | 41 +++++++++++++-------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c index 7ca360b2c20d..e8e47aa16a5d 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.c +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c @@ -67,28 +67,6 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(WHITELIST(CSBE_DEBUG_STATUS(RENDER_RING_BASE), 0)) }, - { XE_RTP_NAME("oa_reg_render"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), - ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(WHITELIST(OAG_MMIOTRIGGER, - RING_FORCE_TO_NONPRIV_ACCESS_RW), - WHITELIST(OAG_OASTATUS, - RING_FORCE_TO_NONPRIV_ACCESS_RD), - WHITELIST(OAG_OAHEADPTR, - RING_FORCE_TO_NONPRIV_ACCESS_RD | - RING_FORCE_TO_NONPRIV_RANGE_4)) - }, - { XE_RTP_NAME("oa_reg_compute"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), - ENGINE_CLASS(COMPUTE)), - XE_RTP_ACTIONS(WHITELIST(OAG_MMIOTRIGGER, - RING_FORCE_TO_NONPRIV_ACCESS_RW), - WHITELIST(OAG_OASTATUS, - RING_FORCE_TO_NONPRIV_ACCESS_RD), - WHITELIST(OAG_OAHEADPTR, - RING_FORCE_TO_NONPRIV_ACCESS_RD | - RING_FORCE_TO_NONPRIV_RANGE_4)) - }, { XE_RTP_NAME("14024997852"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(WHITELIST(FF_MODE, @@ -96,6 +74,25 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { WHITELIST(VFLSKPD, RING_FORCE_TO_NONPRIV_ACCESS_RW)) }, + +#define WHITELIST_OA_MMIO_TRG(trg, status, head) \ + WHITELIST(trg, RING_FORCE_TO_NONPRIV_ACCESS_RW), \ + WHITELIST(status, RING_FORCE_TO_NONPRIV_ACCESS_RD), \ + WHITELIST(head, RING_FORCE_TO_NONPRIV_ACCESS_RD | RING_FORCE_TO_NONPRIV_RANGE_4) + +#define WHITELIST_OAG_MMIO_TRG \ + WHITELIST_OA_MMIO_TRG(OAG_MMIOTRIGGER, OAG_OASTATUS, OAG_OAHEADPTR) + + { XE_RTP_NAME("oag_mmio_trg_rcs"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), + ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG) + }, + { XE_RTP_NAME("oag_mmio_trg_ccs"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), + ENGINE_CLASS(COMPUTE)), + XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG) + }, }; static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) From 8322adedc0f2ed98a1e12a8dbdfa4fbbb3f17fba Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Mon, 1 Dec 2025 18:51:15 -0800 Subject: [PATCH 101/187] drm/xe/rtp: Whitelist OAM MMIO trigger registers Whitelist OAM registers to enable userspace to execute MMIO triggers on OAM units. Signed-off-by: Ashutosh Dixit Reviewed-by: Umesh Nerlige Ramappa Link: https://patch.msgid.link/20251202025115.373546-6-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/regs/xe_oa_regs.h | 8 ++++++++ drivers/gpu/drm/xe/xe_oa.c | 7 +++---- drivers/gpu/drm/xe/xe_reg_whitelist.c | 21 +++++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/regs/xe_oa_regs.h b/drivers/gpu/drm/xe/regs/xe_oa_regs.h index e693a50706f8..638ab3b99eb0 100644 --- a/drivers/gpu/drm/xe/regs/xe_oa_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_oa_regs.h @@ -100,4 +100,12 @@ #define OAM_COMPRESSION_T3_CONTROL XE_REG(0x1c2e00) #define OAM_LAT_MEASURE_ENABLE REG_BIT(4) +/* Actual address is MEDIA_GT_GSI_OFFSET + the base addr below */ +#define XE_OAM_SAG_BASE 0x13000 +#define XE_OAM_SCMI_0_BASE 0x14000 +#define XE_OAM_SCMI_1_BASE 0x14800 +#define XE_OAM_SAG_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SAG_BASE) +#define XE_OAM_SCMI_0_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_0_BASE) +#define XE_OAM_SCMI_1_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_1_BASE) + #endif diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index d63c04e2d492..cc48663c2b48 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -2601,11 +2601,10 @@ static struct xe_oa_regs __oag_regs(void) static void __xe_oa_init_oa_units(struct xe_gt *gt) { - /* Actual address is MEDIA_GT_GSI_OFFSET + oam_base_addr[i] */ const u32 oam_base_addr[] = { - [XE_OAM_UNIT_SAG] = 0x13000, - [XE_OAM_UNIT_SCMI_0] = 0x14000, - [XE_OAM_UNIT_SCMI_1] = 0x14800, + [XE_OAM_UNIT_SAG] = XE_OAM_SAG_BASE, + [XE_OAM_UNIT_SCMI_0] = XE_OAM_SCMI_0_BASE, + [XE_OAM_UNIT_SCMI_1] = XE_OAM_SCMI_1_BASE, }; int i, num_units = gt->oa.num_oa_units; diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c index e8e47aa16a5d..da49c69076a4 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.c +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c @@ -83,6 +83,17 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { #define WHITELIST_OAG_MMIO_TRG \ WHITELIST_OA_MMIO_TRG(OAG_MMIOTRIGGER, OAG_OASTATUS, OAG_OAHEADPTR) +#define WHITELIST_OAM_MMIO_TRG \ + WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SAG_BASE_ADJ), \ + OAM_STATUS(XE_OAM_SAG_BASE_ADJ), \ + OAM_HEAD_POINTER(XE_OAM_SAG_BASE_ADJ)), \ + WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_0_BASE_ADJ), \ + OAM_STATUS(XE_OAM_SCMI_0_BASE_ADJ), \ + OAM_HEAD_POINTER(XE_OAM_SCMI_0_BASE_ADJ)), \ + WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_1_BASE_ADJ), \ + OAM_STATUS(XE_OAM_SCMI_1_BASE_ADJ), \ + OAM_HEAD_POINTER(XE_OAM_SCMI_1_BASE_ADJ)) + { XE_RTP_NAME("oag_mmio_trg_rcs"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), ENGINE_CLASS(RENDER)), @@ -93,6 +104,16 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { ENGINE_CLASS(COMPUTE)), XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG) }, + { XE_RTP_NAME("oam_mmio_trg_vcs"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED), + ENGINE_CLASS(VIDEO_DECODE)), + XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG) + }, + { XE_RTP_NAME("oam_mmio_trg_vecs"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED), + ENGINE_CLASS(VIDEO_ENHANCE)), + XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG) + }, }; static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) From c1daf37fec0d796ca01d0918ce65cc09fe4936f7 Mon Sep 17 00:00:00 2001 From: Zhanjun Dong Date: Thu, 27 Nov 2025 12:07:54 -0500 Subject: [PATCH 102/187] drm/xe/guc: Add log init config abi definitions Add GuC log init config (LIC) ABI definitions. Signed-off-by: Zhanjun Dong Reviewed-by: Julia Filipchuk Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251127170759.2620994-2-zhanjun.dong@intel.com --- drivers/gpu/drm/xe/abi/guc_lic_abi.h | 77 ++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 drivers/gpu/drm/xe/abi/guc_lic_abi.h diff --git a/drivers/gpu/drm/xe/abi/guc_lic_abi.h b/drivers/gpu/drm/xe/abi/guc_lic_abi.h new file mode 100644 index 000000000000..9169644093a2 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_lic_abi.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _ABI_GUC_LIC_ABI_H_ +#define _ABI_GUC_LIC_ABI_H_ + +#include + +/** + * enum guc_lic_type - Log Init Config KLV IDs. + */ +enum guc_lic_type { + /** + * @GUC_LIC_TYPE_GUC_SW_VERSION: GuC firmware version. Value + * is a 32 bit number represented by guc_sw_version. + */ + GUC_LIC_TYPE_GUC_SW_VERSION = 0x1, + /** + * @GUC_LIC_TYPE_GUC_DEVICE_ID: GuC device id. Value is a 32 + * bit. + */ + GUC_LIC_TYPE_GUC_DEVICE_ID = 0x2, + /** + * @GUC_LIC_TYPE_TSC_FREQUENCY: GuC timestamp counter + * frequency. Value is a 32 bit number representing frequency in + * kHz. This timestamp is utilized in log entries, timer and + * for engine utilization tracking. + */ + GUC_LIC_TYPE_TSC_FREQUENCY = 0x3, + /** + * @GUC_LIC_TYPE_GMD_ID: HW GMD ID. Value is a 32 bit number + * representing graphics, media and display HW architecture IDs. + */ + GUC_LIC_TYPE_GMD_ID = 0x4, + /** + * @GUC_LIC_TYPE_BUILD_PLATFORM_ID: GuC build platform ID. + * Value is 32 bits. + */ + GUC_LIC_TYPE_BUILD_PLATFORM_ID = 0x5, +}; + +/** + * struct guc_lic - GuC LIC (Log-Init-Config) structure. + * + * This is populated by the GUC at log init time and is located in the log + * buffer memory allocation. + */ +struct guc_lic { + /** + * @magic: A magic number set by GuC to identify that this + * structure contains valid information: magic = GUC_LIC_MAGIC. + */ + u32 magic; +#define GUC_LIC_MAGIC 0x8086900D + /** + * @version: The version of the this structure. + * Major and minor version number are represented as bit fields. + */ + u32 version; +#define GUC_LIC_VERSION_MASK_MAJOR GENMASK(31, 16) +#define GUC_LIC_VERSION_MASK_MINOR GENMASK(15, 0) + +#define GUC_LIC_VERSION_MAJOR 1u +#define GUC_LIC_VERSION_MINOR 0u + + /** @data_count: Number of dwords the `data` array contains. */ + u32 data_count; + /** + * @data: Array of dwords representing a list of LIC KLVs of + * type guc_klv_generic with keys represented by guc_lic_type + */ + u32 data[] __counted_by(data_count); +} __packed; + +#endif From 7eeb0e5408bda883ac01bf71c1105892c82060e4 Mon Sep 17 00:00:00 2001 From: Zhanjun Dong Date: Thu, 27 Nov 2025 12:07:55 -0500 Subject: [PATCH 103/187] drm/xe/guc: Add LFD related abi definitions Add GuC LFD (Log Format Descriptors) related ABI definitions. Signed-off-by: Zhanjun Dong Reviewed-by: Julia Filipchuk Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251127170759.2620994-3-zhanjun.dong@intel.com --- drivers/gpu/drm/xe/abi/guc_lfd_abi.h | 171 +++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 drivers/gpu/drm/xe/abi/guc_lfd_abi.h diff --git a/drivers/gpu/drm/xe/abi/guc_lfd_abi.h b/drivers/gpu/drm/xe/abi/guc_lfd_abi.h new file mode 100644 index 000000000000..b6ed20d5b508 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_lfd_abi.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _ABI_GUC_LFD_ABI_H_ +#define _ABI_GUC_LFD_ABI_H_ + +#include + +#include "guc_lic_abi.h" + +/* The current major version of GuC-Log-File format. */ +#define GUC_LFD_FORMAT_VERSION_MAJOR 0x0001 +/* The current minor version of GuC-Log-File format. */ +#define GUC_LFD_FORMAT_VERSION_MINOR 0x0000 + +/** enum guc_lfd_type - Log format descriptor type */ +enum guc_lfd_type { + /** + * @GUC_LFD_TYPE_FW_REQUIRED_RANGE_START: Start of range for + * required LFDs from GuC + * @GUC_LFD_TYPE_FW_VERSION: GuC Firmware Version structure. + * @GUC_LFD_TYPE_GUC_DEVICE_ID: GuC microcontroller device ID. + * @GUC_LFD_TYPE_TSC_FREQUENCY: Frequency of GuC timestamps. + * @GUC_LFD_TYPE_GMD_ID: HW GMD ID. + * @GUC_LFD_TYPE_BUILD_PLATFORM_ID: GuC build platform ID. + * @GUC_LFD_TYPE_FW_REQUIRED_RANGE_END: End of range for + * required LFDs from GuC + */ + GUC_LFD_TYPE_FW_REQUIRED_RANGE_START = 0x1, + GUC_LFD_TYPE_FW_VERSION = 0x1, + GUC_LFD_TYPE_GUC_DEVICE_ID = 0x2, + GUC_LFD_TYPE_TSC_FREQUENCY = 0x3, + GUC_LFD_TYPE_GMD_ID = 0x4, + GUC_LFD_TYPE_BUILD_PLATFORM_ID = 0x5, + GUC_LFD_TYPE_FW_REQUIRED_RANGE_END = 0x1FFF, + + /** + * @GUC_LFD_TYPE_FW_OPTIONAL_RANGE_START: Start of range for + * optional LFDs from GuC + * @GUC_LFD_TYPE_LOG_EVENTS_BUFFER: Log-event-entries buffer. + * @GUC_LFD_TYPE_FW_CRASH_DUMP: GuC generated crash-dump blob. + * @GUC_LFD_TYPE_FW_OPTIONAL_RANGE_END: End of range for + * optional LFDs from GuC + */ + GUC_LFD_TYPE_FW_OPTIONAL_RANGE_START = 0x2000, + GUC_LFD_TYPE_LOG_EVENTS_BUFFER = 0x2000, + GUC_LFD_TYPE_FW_CRASH_DUMP = 0x2001, + GUC_LFD_TYPE_FW_OPTIONAL_RANGE_END = 0x3FFF, + + /** + * @GUC_LFD_TYPE_KMD_REQUIRED_RANGE_START: Start of range for + * required KMD LFDs + * @GUC_LFD_TYPE_OS_ID: An identifier for the OS. + * @GUC_LFD_TYPE_KMD_REQUIRED_RANGE_END: End of this range for + * required KMD LFDs + */ + GUC_LFD_TYPE_KMD_REQUIRED_RANGE_START = 0x4000, + GUC_LFD_TYPE_OS_ID = 0x4000, + GUC_LFD_TYPE_KMD_REQUIRED_RANGE_END = 0x5FFF, + + /** + * @GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_START: Start of range for + * optional KMD LFDs + * @GUC_LFD_TYPE_BINARY_SCHEMA_FORMAT: Binary representation of + * GuC log-events schema. + * @GUC_LFD_TYPE_HOST_COMMENT: ASCII string containing comments + * from the host/KMD. + * @GUC_LFD_TYPE_TIMESTAMP_ANCHOR: A timestamp anchor, to convert + * between host and GuC timestamp. + * @GUC_LFD_TYPE_TIMESTAMP_ANCHOR_CONFIG: Timestamp anchor + * configuration, definition of timestamp frequency and bit width. + * @GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_END: End of this range for + * optional KMD LFDs + */ + GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_START = 0x6000, + GUC_LFD_TYPE_BINARY_SCHEMA_FORMAT = 0x6000, + GUC_LFD_TYPE_HOST_COMMENT = 0x6001, + GUC_LFD_TYPE_TIMESTAMP_ANCHOR = 0x6002, + GUC_LFD_TYPE_TIMESTAMP_ANCHOR_CONFIG = 0x6003, + GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_END = 0x7FFF, + + /* + * @GUC_LFD_TYPE_RESERVED_RANGE_START: Start of reserved range + * @GUC_LFD_TYPE_RESERVED_RANGE_END: End of reserved range + */ + GUC_LFD_TYPE_RESERVED_RANGE_START = 0x8000, + GUC_LFD_TYPE_RESERVED_RANGE_END = 0xFFFF, +}; + +/** enum guc_lfd_os_type - OS Type LFD-ID */ +enum guc_lfd_os_type { + /** @GUC_LFD_OS_TYPE_OSID_WIN: Windows OS */ + GUC_LFD_OS_TYPE_OSID_WIN = 0x1, + /** @GUC_LFD_OS_TYPE_OSID_LIN: Linux OS */ + GUC_LFD_OS_TYPE_OSID_LIN = 0x2, + /** @GUC_LFD_OS_TYPE_OSID_VMW: VMWare OS */ + GUC_LFD_OS_TYPE_OSID_VMW = 0x3, + /** @GUC_LFD_OS_TYPE_OSID_OTHER: Other */ + GUC_LFD_OS_TYPE_OSID_OTHER = 0x4, +}; + +/** struct guc_lfd_data - A generic header structure for all LFD blocks */ +struct guc_lfd_data { + /** @header: A 32 bits dword, contains multiple bit fields */ + u32 header; + /* LFD type. See guc_lfd_type */ +#define GUC_LFD_DATA_HEADER_MASK_TYPE GENMASK(31, 16) +#define GUC_LFD_DATA_HEADER_MASK_MAGIC GENMASK(15, 0) + + /** @data_count: Number of dwords the `data` field contains. */ + u32 data_count; + /** @data: Data defined by GUC_LFD_DATA_HEADER_MASK_TYPE */ + u32 data[] __counted_by(data_count); +} __packed; + +/** + * struct guc_lfd_data_log_events_buf - GuC Log Events Buffer. + * This is optional fw LFD data + */ +struct guc_lfd_data_log_events_buf { + /** + * @log_events_format_version: version of GuC log format of buffer + */ + u32 log_events_format_version; + /** + * @log_event: The log event data. + * Size in dwords is LFD block size - 1. + */ + u32 log_event[]; +} __packed; + +/** struct guc_lfd_data_os_info - OS Version Information. */ +struct guc_lfd_data_os_info { + /** + * @os_id: enum values to identify the OS brand. + * See guc_lfd_os_type for the range of types + */ + u32 os_id; + /** + * @build_version: ASCII string containing OS build version + * information based on os_id. String is padded with null + * characters to ensure its DWORD aligned. + * Size in dwords is LFD block size - 1. + */ + char build_version[]; +} __packed; + +/** + * struct guc_logfile_header - Header of GuC Log Streaming-LFD-File Format. + * This structure encapsulates the layout of the guc-log-file format + */ +struct guc_lfd_file_header { + /** + * @magic: A magic number set by producer of a GuC log file to + * identify that file is a valid guc-log-file containing a stream + * of LFDs. + */ + u64 magic; + /** @version: Version of this file format layout */ + u32 version; +#define GUC_LFD_FILE_HEADER_VERSION_MASK_MAJOR GENMASK(31, 16) +#define GUC_LFD_FILE_HEADER_VERSION_MASK_MINOR GENMASK(15, 0) + + /** @stream: A stream of one or more guc_lfd_data LFD blocks + */ + u32 stream[]; +} __packed; + +#endif From 9124732b14010f58849d6686d393246c864639ca Mon Sep 17 00:00:00 2001 From: Zhanjun Dong Date: Thu, 27 Nov 2025 12:07:56 -0500 Subject: [PATCH 104/187] drm/xe/guc: Add GuC log init config in LFD format Add support to output GuC log init config (LIC) in LFD format. Signed-off-by: Zhanjun Dong Reviewed-by: Julia Filipchuk Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251127170759.2620994-4-zhanjun.dong@intel.com --- drivers/gpu/drm/xe/xe_guc_log.c | 267 +++++++++++++++++++++++++++++++- 1 file changed, 266 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c index 086cd60628f3..6e840f1abe4e 100644 --- a/drivers/gpu/drm/xe/xe_guc_log.c +++ b/drivers/gpu/drm/xe/xe_guc_log.c @@ -7,8 +7,10 @@ #include +#include #include +#include "abi/guc_lfd_abi.h" #include "regs/xe_guc_regs.h" #include "xe_bo.h" #include "xe_devcoredump.h" @@ -19,7 +21,76 @@ #include "xe_mmio.h" #include "xe_module.h" -#define GUC_LOG_CHUNK_SIZE SZ_2M +#define GUC_LOG_CHUNK_SIZE SZ_2M + +/* Magic keys define */ +#define GUC_LFD_DRIVER_KEY_STREAMING 0x8086AAAA474C5346 +#define GUC_LFD_LOG_BUFFER_MARKER_2 0xDEADFEED +#define GUC_LFD_CRASH_DUMP_BUFFER_MARKER_2 0x8086DEAD +#define GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_2 0xBEEFFEED +#define GUC_LFD_LOG_BUFFER_MARKER_1V2 0xCABBA9E6 +#define GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_1V2 0xCABBA9F7 +#define GUC_LFD_DATA_HEADER_MAGIC 0x8086 + +/* LFD supported LIC type range */ +#define GUC_LIC_TYPE_FIRST GUC_LIC_TYPE_GUC_SW_VERSION +#define GUC_LIC_TYPE_LAST GUC_LIC_TYPE_BUILD_PLATFORM_ID +#define GUC_LFD_TYPE_FW_RANGE_FIRST GUC_LFD_TYPE_FW_VERSION +#define GUC_LFD_TYPE_FW_RANGE_LAST GUC_LFD_TYPE_BUILD_PLATFORM_ID + +#define GUC_LOG_BUFFER_STATE_HEADER_LENGTH 4096 +#define GUC_LOG_BUFFER_INIT_CONFIG 3 + +struct guc_log_buffer_entry_list { + u32 offset; + u32 rd_ptr; + u32 wr_ptr; + u32 wrap_offset; + u32 buf_size; +}; + +struct guc_lic_save { + u32 version; + /* + * Array of init config KLV values. + * Range from GUC_LOG_LIC_TYPE_FIRST to GUC_LOG_LIC_TYPE_LAST + */ + u32 values[GUC_LIC_TYPE_LAST - GUC_LIC_TYPE_FIRST + 1]; + struct guc_log_buffer_entry_list entry[GUC_LOG_BUFFER_INIT_CONFIG]; +}; + +static struct guc_log_buffer_entry_markers { + u32 key[2]; +} const entry_markers[GUC_LOG_BUFFER_INIT_CONFIG + 1] = { + {{ + GUC_LFD_LOG_BUFFER_MARKER_1V2, + GUC_LFD_LOG_BUFFER_MARKER_2 + }}, + {{ + GUC_LFD_LOG_BUFFER_MARKER_1V2, + GUC_LFD_CRASH_DUMP_BUFFER_MARKER_2 + }}, + {{ + GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_1V2, + GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_2 + }}, + {{ + GUC_LIC_MAGIC, + (FIELD_PREP_CONST(GUC_LIC_VERSION_MASK_MAJOR, GUC_LIC_VERSION_MAJOR) | + FIELD_PREP_CONST(GUC_LIC_VERSION_MASK_MINOR, GUC_LIC_VERSION_MINOR)) + }} +}; + +static struct guc_log_lic_lfd_map { + u32 lic; + u32 lfd; +} const lic_lfd_type_map[] = { + {GUC_LIC_TYPE_GUC_SW_VERSION, GUC_LFD_TYPE_FW_VERSION}, + {GUC_LIC_TYPE_GUC_DEVICE_ID, GUC_LFD_TYPE_GUC_DEVICE_ID}, + {GUC_LIC_TYPE_TSC_FREQUENCY, GUC_LFD_TYPE_TSC_FREQUENCY}, + {GUC_LIC_TYPE_GMD_ID, GUC_LFD_TYPE_GMD_ID}, + {GUC_LIC_TYPE_BUILD_PLATFORM_ID, GUC_LFD_TYPE_BUILD_PLATFORM_ID} +}; static struct xe_guc * log_to_guc(struct xe_guc_log *log) @@ -189,6 +260,200 @@ void xe_guc_log_snapshot_print(struct xe_guc_log_snapshot *snapshot, struct drm_ } } +static inline void lfd_output_binary(struct drm_printer *p, char *buf, int buf_size) +{ + seq_write(p->arg, buf, buf_size); +} + +static inline int xe_guc_log_add_lfd_header(struct guc_lfd_data *lfd) +{ + lfd->header = FIELD_PREP_CONST(GUC_LFD_DATA_HEADER_MASK_MAGIC, GUC_LFD_DATA_HEADER_MAGIC); + return offsetof(struct guc_lfd_data, data); +} + +static int xe_guc_log_add_typed_payload(struct drm_printer *p, u32 type, + u32 data_len, void *data) +{ + struct guc_lfd_data lfd; + int len; + + len = xe_guc_log_add_lfd_header(&lfd); + lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, type); + /* make length DW aligned */ + lfd.data_count = DIV_ROUND_UP(data_len, sizeof(u32)); + lfd_output_binary(p, (char *)&lfd, len); + + lfd_output_binary(p, data, data_len); + len += lfd.data_count * sizeof(u32); + + return len; +} + +static inline int lic_type_to_index(u32 lic_type) +{ + XE_WARN_ON(lic_type < GUC_LIC_TYPE_FIRST || lic_type > GUC_LIC_TYPE_LAST); + + return lic_type - GUC_LIC_TYPE_FIRST; +} + +static inline int lfd_type_to_index(u32 lfd_type) +{ + int i, lic_type = 0; + + XE_WARN_ON(lfd_type < GUC_LFD_TYPE_FW_RANGE_FIRST || lfd_type > GUC_LFD_TYPE_FW_RANGE_LAST); + + for (i = 0; i < ARRAY_SIZE(lic_lfd_type_map); i++) + if (lic_lfd_type_map[i].lfd == lfd_type) + lic_type = lic_lfd_type_map[i].lic; + + /* If not found, lic_type_to_index will warning invalid type */ + return lic_type_to_index(lic_type); +} + +static int xe_guc_log_add_klv(struct drm_printer *p, u32 lfd_type, + struct guc_lic_save *config) +{ + int klv_index = lfd_type_to_index(lfd_type); + + return xe_guc_log_add_typed_payload(p, lfd_type, sizeof(u32), &config->values[klv_index]); +} + +static int xe_guc_log_add_os_id(struct drm_printer *p, u32 id) +{ + struct guc_lfd_data_os_info os_id; + struct guc_lfd_data lfd; + int len, info_len, section_len; + char *version; + u32 blank = 0; + + len = xe_guc_log_add_lfd_header(&lfd); + lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_OS_ID); + + os_id.os_id = id; + section_len = offsetof(struct guc_lfd_data_os_info, build_version); + + version = init_utsname()->release; + info_len = strlen(version); + + /* make length DW aligned */ + lfd.data_count = DIV_ROUND_UP(section_len + info_len, sizeof(u32)); + lfd_output_binary(p, (char *)&lfd, len); + lfd_output_binary(p, (char *)&os_id, section_len); + lfd_output_binary(p, version, info_len); + + /* Padding with 0 */ + section_len = lfd.data_count * sizeof(u32) - section_len - info_len; + if (section_len) + lfd_output_binary(p, (char *)&blank, section_len); + + len += lfd.data_count * sizeof(u32); + return len; +} + +static void xe_guc_log_loop_log_init(struct guc_lic *init, struct guc_lic_save *config) +{ + struct guc_klv_generic_dw_t *p = (void *)init->data; + int i; + + for (i = 0; i < init->data_count;) { + int klv_len = FIELD_GET(GUC_KLV_0_LEN, p->kl) + 1; + int key = FIELD_GET(GUC_KLV_0_KEY, p->kl); + + if (key < GUC_LIC_TYPE_FIRST || key > GUC_LIC_TYPE_LAST) { + XE_WARN_ON(key < GUC_LIC_TYPE_FIRST || key > GUC_LIC_TYPE_LAST); + break; + } + config->values[lic_type_to_index(key)] = p->value; + i += klv_len + 1; /* Whole KLV structure length in dwords */ + p = (void *)((u32 *)p + klv_len); + } +} + +static int find_marker(u32 mark0, u32 mark1) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(entry_markers); i++) + if (mark0 == entry_markers[i].key[0] && mark1 == entry_markers[i].key[1]) + return i; + + return ARRAY_SIZE(entry_markers); +} + +static void xe_guc_log_load_lic(void *guc_log, struct guc_lic_save *config) +{ + u32 offset = GUC_LOG_BUFFER_STATE_HEADER_LENGTH; + struct guc_log_buffer_state *p = guc_log; + + config->version = p->version; + while (p->marker[0]) { + int index; + + index = find_marker(p->marker[0], p->marker[1]); + + if (index < ARRAY_SIZE(entry_markers)) { + if (index == GUC_LOG_BUFFER_INIT_CONFIG) { + /* Load log init config */ + xe_guc_log_loop_log_init((void *)p, config); + + /* LIC structure is the last */ + return; + } + config->entry[index].offset = offset; + config->entry[index].rd_ptr = p->read_ptr; + config->entry[index].wr_ptr = p->write_ptr; + config->entry[index].wrap_offset = p->wrap_offset; + config->entry[index].buf_size = p->size; + } + offset += p->size; + p++; + } +} + +static int +xe_guc_log_output_lfd_init(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot, + struct guc_lic_save *config) +{ + int type, len; + size_t size = 0; + + /* FW required types */ + for (type = GUC_LFD_TYPE_FW_RANGE_FIRST; type <= GUC_LFD_TYPE_FW_RANGE_LAST; type++) + size += xe_guc_log_add_klv(p, type, config); + + /* KMD required type(s) */ + len = xe_guc_log_add_os_id(p, GUC_LFD_OS_TYPE_OSID_LIN); + size += len; + + return size; +} + +void +xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p); +void +xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p) +{ + struct guc_lfd_file_header header; + struct guc_lic_save config; + + if (!snapshot || !snapshot->size) + return; + + header.magic = GUC_LFD_DRIVER_KEY_STREAMING; + header.version = FIELD_PREP_CONST(GUC_LFD_FILE_HEADER_VERSION_MASK_MINOR, + GUC_LFD_FORMAT_VERSION_MINOR) | + FIELD_PREP_CONST(GUC_LFD_FILE_HEADER_VERSION_MASK_MAJOR, + GUC_LFD_FORMAT_VERSION_MAJOR); + + /* Output LFD file header */ + lfd_output_binary(p, (char *)&header, + offsetof(struct guc_lfd_file_header, stream)); + + /* Output LFD stream */ + xe_guc_log_load_lic(snapshot->copy[0], &config); + xe_guc_log_output_lfd_init(p, snapshot, &config); +} + /** * xe_guc_log_print_dmesg - dump a copy of the GuC log to dmesg * @log: GuC log structure From fc40e8f6fc2de469b40494df9c12fda2c11dea3f Mon Sep 17 00:00:00 2001 From: Zhanjun Dong Date: Thu, 27 Nov 2025 12:07:57 -0500 Subject: [PATCH 105/187] drm/xe/guc: Add GuC log event buffer output in LFD format Add GuC log event buffer output in LFD format. Signed-off-by: Zhanjun Dong Reviewed-by: Julia Filipchuk Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251127170759.2620994-5-zhanjun.dong@intel.com --- drivers/gpu/drm/xe/xe_guc_log.c | 77 ++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c index 6e840f1abe4e..51c54d2a49fd 100644 --- a/drivers/gpu/drm/xe/xe_guc_log.c +++ b/drivers/gpu/drm/xe/xe_guc_log.c @@ -428,6 +428,76 @@ xe_guc_log_output_lfd_init(struct drm_printer *p, struct xe_guc_log_snapshot *sn return size; } +static void +xe_guc_log_print_chunks(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot, + u32 from, u32 to) +{ + int chunk_from = from % GUC_LOG_CHUNK_SIZE; + int chunk_id = from / GUC_LOG_CHUNK_SIZE; + int to_chunk_id = to / GUC_LOG_CHUNK_SIZE; + int chunk_to = to % GUC_LOG_CHUNK_SIZE; + int pos = from; + + do { + size_t size = (to_chunk_id == chunk_id ? chunk_to : GUC_LOG_CHUNK_SIZE) - + chunk_from; + + lfd_output_binary(p, snapshot->copy[chunk_id] + chunk_from, size); + pos += size; + chunk_id++; + chunk_from = 0; + } while (pos < to); +} + +static inline int +xe_guc_log_add_log_event(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot, + struct guc_lic_save *config) +{ + size_t size; + u32 data_len, section_len; + struct guc_lfd_data lfd; + struct guc_log_buffer_entry_list *entry; + struct guc_lfd_data_log_events_buf events_buf; + + entry = &config->entry[GUC_LOG_TYPE_EVENT_DATA]; + + /* Skip empty log */ + if (entry->rd_ptr == entry->wr_ptr) + return 0; + + size = xe_guc_log_add_lfd_header(&lfd); + lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_LOG_EVENTS_BUFFER); + events_buf.log_events_format_version = config->version; + + /* Adjust to log_format_buf */ + section_len = offsetof(struct guc_lfd_data_log_events_buf, log_event); + data_len = section_len; + + /* Calculate data length */ + data_len += entry->rd_ptr < entry->wr_ptr ? (entry->wr_ptr - entry->rd_ptr) : + (entry->wr_ptr + entry->wrap_offset - entry->rd_ptr); + /* make length u32 aligned */ + lfd.data_count = DIV_ROUND_UP(data_len, sizeof(u32)); + + /* Output GUC_LFD_TYPE_LOG_EVENTS_BUFFER header */ + lfd_output_binary(p, (char *)&lfd, size); + lfd_output_binary(p, (char *)&events_buf, section_len); + + /* Output data from guc log chunks directly */ + if (entry->rd_ptr < entry->wr_ptr) { + xe_guc_log_print_chunks(p, snapshot, entry->offset + entry->rd_ptr, + entry->offset + entry->wr_ptr); + } else { + /* 1st, print from rd to wrap offset */ + xe_guc_log_print_chunks(p, snapshot, entry->offset + entry->rd_ptr, + entry->offset + entry->wrap_offset); + + /* 2nd, print from buf start to wr */ + xe_guc_log_print_chunks(p, snapshot, entry->offset, entry->offset + entry->wr_ptr); + } + return size; +} + void xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p); void @@ -435,6 +505,7 @@ xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_p { struct guc_lfd_file_header header; struct guc_lic_save config; + size_t size; if (!snapshot || !snapshot->size) return; @@ -451,7 +522,11 @@ xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_p /* Output LFD stream */ xe_guc_log_load_lic(snapshot->copy[0], &config); - xe_guc_log_output_lfd_init(p, snapshot, &config); + size = xe_guc_log_output_lfd_init(p, snapshot, &config); + if (!size) + return; + + xe_guc_log_add_log_event(p, snapshot, &config); } /** From 09fb6bccef8221544221ebeb97cffec3fb42920d Mon Sep 17 00:00:00 2001 From: Zhanjun Dong Date: Thu, 27 Nov 2025 12:07:58 -0500 Subject: [PATCH 106/187] drm/xe/guc: Only add GuC crash dump if available Add GuC crash dump data empty check. LFD will only include crash dump section when data is not empty. Signed-off-by: Zhanjun Dong Reviewed-by: Julia Filipchuk Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251127170759.2620994-6-zhanjun.dong@intel.com --- drivers/gpu/drm/xe/xe_guc_log.c | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c index 51c54d2a49fd..40c9e812e9d5 100644 --- a/drivers/gpu/drm/xe/xe_guc_log.c +++ b/drivers/gpu/drm/xe/xe_guc_log.c @@ -498,6 +498,50 @@ xe_guc_log_add_log_event(struct drm_printer *p, struct xe_guc_log_snapshot *snap return size; } +static int +xe_guc_log_add_crash_dump(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot, + struct guc_lic_save *config) +{ + struct guc_log_buffer_entry_list *entry; + int chunk_from, chunk_id; + int from, to, i; + size_t size = 0; + u32 *buf32; + + entry = &config->entry[GUC_LOG_TYPE_CRASH_DUMP]; + + /* Skip zero sized crash dump */ + if (!entry->buf_size) + return 0; + + /* Check if crash dump section are all zero */ + from = entry->offset; + to = entry->offset + entry->buf_size; + chunk_from = from % GUC_LOG_CHUNK_SIZE; + chunk_id = from / GUC_LOG_CHUNK_SIZE; + buf32 = snapshot->copy[chunk_id] + chunk_from; + + for (i = 0; i < entry->buf_size / sizeof(u32); i++) + if (buf32[i]) + break; + + /* Buffer has non-zero data? */ + if (i < entry->buf_size / sizeof(u32)) { + struct guc_lfd_data lfd; + + size = xe_guc_log_add_lfd_header(&lfd); + lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_FW_CRASH_DUMP); + /* Calculate data length */ + lfd.data_count = DIV_ROUND_UP(entry->buf_size, sizeof(u32)); + /* Output GUC_LFD_TYPE_FW_CRASH_DUMP header */ + lfd_output_binary(p, (char *)&lfd, size); + + /* rd/wr ptr is not used for crash dump */ + xe_guc_log_print_chunks(p, snapshot, from, to); + } + return size; +} + void xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p); void @@ -527,6 +571,7 @@ xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_p return; xe_guc_log_add_log_event(p, snapshot, &config); + xe_guc_log_add_crash_dump(p, snapshot, &config); } /** From c7ea291be0866ca562bb7d32d1646c08b2c3064d Mon Sep 17 00:00:00 2001 From: Zhanjun Dong Date: Thu, 27 Nov 2025 12:07:59 -0500 Subject: [PATCH 107/187] drm/xe/guc: Add new debugfs entry for lfd format output Add new debugfs entry "guc_log_lfd", prepared for output guc log in LFD(Log Format Descriptors) format. Signed-off-by: Zhanjun Dong Reviewed-by: Julia Filipchuk Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251127170759.2620994-7-zhanjun.dong@intel.com --- drivers/gpu/drm/xe/xe_guc_debugfs.c | 7 +++++++ drivers/gpu/drm/xe/xe_guc_log.c | 18 +++++++++++++++--- drivers/gpu/drm/xe/xe_guc_log.h | 1 + 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_debugfs.c b/drivers/gpu/drm/xe/xe_guc_debugfs.c index efaca259d3e8..23827e87450f 100644 --- a/drivers/gpu/drm/xe/xe_guc_debugfs.c +++ b/drivers/gpu/drm/xe/xe_guc_debugfs.c @@ -81,6 +81,12 @@ static int guc_log(struct xe_guc *guc, struct drm_printer *p) return 0; } +static int guc_log_lfd(struct xe_guc *guc, struct drm_printer *p) +{ + xe_guc_log_print_lfd(&guc->log, p); + return 0; +} + static int guc_log_dmesg(struct xe_guc *guc, struct drm_printer *p) { xe_guc_log_print_dmesg(&guc->log); @@ -117,6 +123,7 @@ static const struct drm_info_list slpc_debugfs_list[] = { /* everything else should be added here */ static const struct drm_info_list pf_only_debugfs_list[] = { { "guc_log", .show = guc_debugfs_show, .data = guc_log }, + { "guc_log_lfd", .show = guc_debugfs_show, .data = guc_log_lfd }, { "guc_log_dmesg", .show = guc_debugfs_show, .data = guc_log_dmesg }, }; diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c index 40c9e812e9d5..d7473b9673bb 100644 --- a/drivers/gpu/drm/xe/xe_guc_log.c +++ b/drivers/gpu/drm/xe/xe_guc_log.c @@ -542,9 +542,7 @@ xe_guc_log_add_crash_dump(struct drm_printer *p, struct xe_guc_log_snapshot *sna return size; } -void -xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p); -void +static void xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p) { struct guc_lfd_file_header header; @@ -609,6 +607,20 @@ void xe_guc_log_print(struct xe_guc_log *log, struct drm_printer *p) xe_guc_log_snapshot_free(snapshot); } +/** + * xe_guc_log_print_lfd - dump a copy of the GuC log in LFD format + * @log: GuC log structure + * @p: the printer object to output to + */ +void xe_guc_log_print_lfd(struct xe_guc_log *log, struct drm_printer *p) +{ + struct xe_guc_log_snapshot *snapshot; + + snapshot = xe_guc_log_snapshot_capture(log, false); + xe_guc_log_snapshot_print_lfd(snapshot, p); + xe_guc_log_snapshot_free(snapshot); +} + int xe_guc_log_init(struct xe_guc_log *log) { struct xe_device *xe = log_to_xe(log); diff --git a/drivers/gpu/drm/xe/xe_guc_log.h b/drivers/gpu/drm/xe/xe_guc_log.h index 0bd5e89d75e0..1b05bb60c1c7 100644 --- a/drivers/gpu/drm/xe/xe_guc_log.h +++ b/drivers/gpu/drm/xe/xe_guc_log.h @@ -52,6 +52,7 @@ struct xe_device; int xe_guc_log_init(struct xe_guc_log *log); void xe_guc_log_print(struct xe_guc_log *log, struct drm_printer *p); +void xe_guc_log_print_lfd(struct xe_guc_log *log, struct drm_printer *p); void xe_guc_log_print_dmesg(struct xe_guc_log *log); struct xe_guc_log_snapshot *xe_guc_log_snapshot_capture(struct xe_guc_log *log, bool atomic); void xe_guc_log_snapshot_print(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p); From 624f494ee6e9c6b1bd16289c0de34e399950baa8 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Fri, 5 Dec 2025 07:02:19 +0000 Subject: [PATCH 108/187] drm/xe: Refactor PAT dump to use shared helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the PAT entry formatting into shared helper functions to ensure consistency and enable code reuse. This preparation is necessary for a follow-up patch that introduces a software-based PAT dump, which is required for debugging on VFs where hardware access is limited. V2: (Matt) - Xe3p XPC doesn’t define COMP_EN; omit it to match bspec and avoid confusion. Suggested-by: Matt Roper Signed-off-by: Xin Wang Reviewed-by: Matt Roper Signed-off-by: Matt Roper Link: https://patch.msgid.link/20251205070220.27859-1-x.wang@intel.com --- drivers/gpu/drm/xe/xe_pat.c | 108 +++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 44 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c index 717425dd0475..beff250c7fa0 100644 --- a/drivers/gpu/drm/xe/xe_pat.c +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -50,8 +50,37 @@ #define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1) #define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0) +#define PAT_LABEL_LEN 20 + static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" }; +static void xe_pat_index_label(char *label, size_t len, int index) +{ + snprintf(label, len, "PAT[%2d] ", index); +} + +static void xelp_pat_entry_dump(struct drm_printer *p, int index, u32 pat) +{ + u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); + + drm_printf(p, "PAT[%2d] = %s (%#8x)\n", index, + XELP_MEM_TYPE_STR_MAP[mem_type], pat); +} + +static void xehpc_pat_entry_dump(struct drm_printer *p, int index, u32 pat) +{ + drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, + REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat), + REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat); +} + +static void xelpg_pat_entry_dump(struct drm_printer *p, int index, u32 pat) +{ + drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, + REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat), + REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat); +} + struct xe_pat_ops { void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[], int n_entries); @@ -249,10 +278,8 @@ static int xelp_dump(struct xe_gt *gt, struct drm_printer *p) for (i = 0; i < xe->pat.n_entries; i++) { u32 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); - u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); - drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i, - XELP_MEM_TYPE_STR_MAP[mem_type], pat); + xelp_pat_entry_dump(p, i, pat); } return 0; @@ -276,12 +303,8 @@ static int xehp_dump(struct xe_gt *gt, struct drm_printer *p) for (i = 0; i < xe->pat.n_entries; i++) { u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); - u8 mem_type; - mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); - - drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i, - XELP_MEM_TYPE_STR_MAP[mem_type], pat); + xelp_pat_entry_dump(p, i, pat); } return 0; @@ -306,9 +329,7 @@ static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p) for (i = 0; i < xe->pat.n_entries; i++) { u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); - drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i, - REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat), - REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat); + xehpc_pat_entry_dump(p, i, pat); } return 0; @@ -338,9 +359,7 @@ static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p) else pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); - drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i, - REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat), - REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat); + xelpg_pat_entry_dump(p, i, pat); } return 0; @@ -356,11 +375,35 @@ static const struct xe_pat_ops xelpg_pat_ops = { .dump = xelpg_dump, }; +static void xe2_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) +{ + drm_printf(p, "%s= [ %u, %u, %u, %u, %u, %u ] (%#8x)%s\n", label, + !!(pat & XE2_NO_PROMOTE), + !!(pat & XE2_COMP_EN), + REG_FIELD_GET(XE2_L3_CLOS, pat), + REG_FIELD_GET(XE2_L3_POLICY, pat), + REG_FIELD_GET(XE2_L4_POLICY, pat), + REG_FIELD_GET(XE2_COH_MODE, pat), + pat, rsvd ? " *" : ""); +} + +static void xe3p_xpc_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) +{ + drm_printf(p, "%s= [ %u, %u, %u, %u, %u ] (%#8x)%s\n", label, + !!(pat & XE2_NO_PROMOTE), + REG_FIELD_GET(XE2_L3_CLOS, pat), + REG_FIELD_GET(XE2_L3_POLICY, pat), + REG_FIELD_GET(XE2_L4_POLICY, pat), + REG_FIELD_GET(XE2_COH_MODE, pat), + pat, rsvd ? " *" : ""); +} + static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) { struct xe_device *xe = gt_to_xe(gt); u32 pat; int i; + char label[PAT_LABEL_LEN]; CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); if (!fw_ref.domains) @@ -374,14 +417,8 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) else pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); - drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u, %u ] (%#8x)%s\n", i, - !!(pat & XE2_NO_PROMOTE), - !!(pat & XE2_COMP_EN), - REG_FIELD_GET(XE2_L3_CLOS, pat), - REG_FIELD_GET(XE2_L3_POLICY, pat), - REG_FIELD_GET(XE2_L4_POLICY, pat), - REG_FIELD_GET(XE2_COH_MODE, pat), - pat, xe->pat.table[i].valid ? "" : " *"); + xe_pat_index_label(label, sizeof(label), i); + xe2_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid); } /* @@ -394,14 +431,7 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA)); drm_printf(p, "Page Table Access:\n"); - drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u, %u ] (%#8x)\n", - !!(pat & XE2_NO_PROMOTE), - !!(pat & XE2_COMP_EN), - REG_FIELD_GET(XE2_L3_CLOS, pat), - REG_FIELD_GET(XE2_L3_POLICY, pat), - REG_FIELD_GET(XE2_L4_POLICY, pat), - REG_FIELD_GET(XE2_COH_MODE, pat), - pat); + xe2_pat_entry_dump(p, "PTA_MODE", pat, false); return 0; } @@ -417,6 +447,7 @@ static int xe3p_xpc_dump(struct xe_gt *gt, struct drm_printer *p) struct xe_device *xe = gt_to_xe(gt); u32 pat; int i; + char label[PAT_LABEL_LEN]; CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); if (!fw_ref.domains) @@ -427,13 +458,8 @@ static int xe3p_xpc_dump(struct xe_gt *gt, struct drm_printer *p) for (i = 0; i < xe->pat.n_entries; i++) { pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); - drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u ] (%#8x)%s\n", i, - !!(pat & XE2_NO_PROMOTE), - REG_FIELD_GET(XE2_L3_CLOS, pat), - REG_FIELD_GET(XE2_L3_POLICY, pat), - REG_FIELD_GET(XE2_L4_POLICY, pat), - REG_FIELD_GET(XE2_COH_MODE, pat), - pat, xe->pat.table[i].valid ? "" : " *"); + xe_pat_index_label(label, sizeof(label), i); + xe3p_xpc_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid); } /* @@ -443,13 +469,7 @@ static int xe3p_xpc_dump(struct xe_gt *gt, struct drm_printer *p) pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA)); drm_printf(p, "Page Table Access:\n"); - drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u ] (%#8x)\n", - !!(pat & XE2_NO_PROMOTE), - REG_FIELD_GET(XE2_L3_CLOS, pat), - REG_FIELD_GET(XE2_L3_POLICY, pat), - REG_FIELD_GET(XE2_L4_POLICY, pat), - REG_FIELD_GET(XE2_COH_MODE, pat), - pat); + xe3p_xpc_pat_entry_dump(p, "PTA_MODE", pat, false); return 0; } From 944a8313a7ebbd4cfbb0a579d4f6283e125edc08 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Fri, 5 Dec 2025 07:06:33 +0000 Subject: [PATCH 109/187] drm/xe: expose PAT software config to debugfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing "pat" debugfs node dumps the live PAT registers. Under SR-IOV the VF cannot touch those registers, so the file vanishes and users lose all PAT visibility. Add a VF-safe "pat_sw_config" entry to the VF-safe debugfs list. It prints the cached PAT table the driver programmed, rather than poking HW, so PF and VF instances present the same view. This lets IGT and other tools query the PAT configuration without carrying platform-specific tables or mirroring kernel logic. v2: (Jonathan) - Only append "(* = reserved entry)" to the PAT table header on Xe2+ platforms where it actually applies. - Deduplicate the PTA/ATS mode printing by introducing the small drm_printf_pat_mode() helper macro. v3: (Matt) - Print IDX[XE_CACHE_NONE_COMPRESSION] on every Xe2+ platform so the dump always reflects the value the driver might use (even if it defaults to 0) and future IP revisions don’t need extra condition tweaks. v4: - Drop the drm_printf_pat_mode macro and introduce a real helper xe2_pat_entry_dump(). (Jani) - Reuse the helper across all PTA/ATS/PAT dumps for xe2+ entries to keep output format identical. v5: (Matt) - Split the original patch into two: one for refactoring helpers, one for the new debugfs entry. CC: Jani Nikula Suggested-by: Matt Roper Signed-off-by: Xin Wang Reviewed-by: Jonathan Cavitt Signed-off-by: Matt Roper Link: https://patch.msgid.link/20251205070633.28072-1-x.wang@intel.com --- drivers/gpu/drm/xe/xe_gt_debugfs.c | 1 + drivers/gpu/drm/xe/xe_pat.c | 62 ++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_pat.h | 1 + 3 files changed, 64 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c index 7c3de6539044..e4f38b5150fc 100644 --- a/drivers/gpu/drm/xe/xe_gt_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -209,6 +209,7 @@ static const struct drm_info_list vf_safe_debugfs_list[] = { { "default_lrc_vcs", .show = xe_gt_debugfs_show_with_rpm, .data = vcs_default_lrc }, { "default_lrc_vecs", .show = xe_gt_debugfs_show_with_rpm, .data = vecs_default_lrc }, { "hwconfig", .show = xe_gt_debugfs_show_with_rpm, .data = hwconfig }, + { "pat_sw_config", .show = xe_gt_debugfs_simple_show, .data = xe_pat_dump_sw_config }, }; /* everything else should be added here */ diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c index beff250c7fa0..6f48d34711a6 100644 --- a/drivers/gpu/drm/xe/xe_pat.c +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -592,3 +592,65 @@ int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p) return xe->pat.ops->dump(gt, p); } + +/** + * xe_pat_dump_sw_config() - Dump the software-configured GT PAT table into a drm printer. + * @gt: the &xe_gt + * @p: the &drm_printer + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_device *xe = gt_to_xe(gt); + char label[PAT_LABEL_LEN]; + + if (!xe->pat.table || !xe->pat.n_entries) + return -EOPNOTSUPP; + + drm_printf(p, "PAT table:%s\n", GRAPHICS_VER(xe) >= 20 ? " (* = reserved entry)" : ""); + for (u32 i = 0; i < xe->pat.n_entries; i++) { + u32 pat = xe->pat.table[i].value; + + if (GRAPHICS_VERx100(xe) == 3511) { + xe_pat_index_label(label, sizeof(label), i); + xe3p_xpc_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid); + } else if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) { + xe_pat_index_label(label, sizeof(label), i); + xe2_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid); + } else if (xe->info.platform == XE_METEORLAKE) { + xelpg_pat_entry_dump(p, i, pat); + } else if (xe->info.platform == XE_PVC) { + xehpc_pat_entry_dump(p, i, pat); + } else if (xe->info.platform == XE_DG2 || GRAPHICS_VERx100(xe) <= 1210) { + xelp_pat_entry_dump(p, i, pat); + } else { + return -EOPNOTSUPP; + } + } + + if (xe->pat.pat_pta) { + u32 pat = xe->pat.pat_pta->value; + + drm_printf(p, "Page Table Access:\n"); + xe2_pat_entry_dump(p, "PTA_MODE", pat, false); + } + + if (xe->pat.pat_ats) { + u32 pat = xe->pat.pat_ats->value; + + drm_printf(p, "PCIe ATS/PASID:\n"); + xe2_pat_entry_dump(p, "PAT_ATS ", pat, false); + } + + drm_printf(p, "Cache Level:\n"); + drm_printf(p, "IDX[XE_CACHE_NONE] = %d\n", xe->pat.idx[XE_CACHE_NONE]); + drm_printf(p, "IDX[XE_CACHE_WT] = %d\n", xe->pat.idx[XE_CACHE_WT]); + drm_printf(p, "IDX[XE_CACHE_WB] = %d\n", xe->pat.idx[XE_CACHE_WB]); + if (GRAPHICS_VER(xe) >= 20) { + drm_printf(p, "IDX[XE_CACHE_NONE_COMPRESSION] = %d\n", + xe->pat.idx[XE_CACHE_NONE_COMPRESSION]); + } + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_pat.h b/drivers/gpu/drm/xe/xe_pat.h index b8559120989e..5749a488d9a9 100644 --- a/drivers/gpu/drm/xe/xe_pat.h +++ b/drivers/gpu/drm/xe/xe_pat.h @@ -49,6 +49,7 @@ void xe_pat_init_early(struct xe_device *xe); void xe_pat_init(struct xe_gt *gt); int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p); +int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p); /** * xe_pat_index_get_coh_mode - Extract the coherency mode for the given From da33e457f6803dc48b2b74f8ade75c14624d85ca Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Tue, 2 Dec 2025 14:03:34 +0530 Subject: [PATCH 110/187] drm/xe/guc_ct: Assert on credits mismatch during runtime suspend G2H credits should be in fully idle state when runtime suspending GuC CT. Assert on mismatch. Suggested-by: Matthew Brost Signed-off-by: Raag Jadav Reviewed-by: Matthew Brost Signed-off-by: Daniele Ceraolo Spurio Link: https://patch.msgid.link/20251202083334.554045-1-raag.jadav@intel.com --- drivers/gpu/drm/xe/xe_guc_ct.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 2a24d3fb6881..648f0f523abb 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -253,8 +253,11 @@ static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) #define CTB_H2G_BUFFER_OFFSET (CTB_DESC_SIZE * 2) #define CTB_H2G_BUFFER_SIZE (SZ_4K) +#define CTB_H2G_BUFFER_DWORDS (CTB_H2G_BUFFER_SIZE / sizeof(u32)) #define CTB_G2H_BUFFER_SIZE (SZ_128K) +#define CTB_G2H_BUFFER_DWORDS (CTB_G2H_BUFFER_SIZE / sizeof(u32)) #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2) +#define G2H_ROOM_BUFFER_DWORDS (CTB_G2H_BUFFER_DWORDS / 2) /** * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full @@ -403,7 +406,7 @@ int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct) static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, struct iosys_map *map) { - h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32); + h2g->info.size = CTB_H2G_BUFFER_DWORDS; h2g->info.resv_space = 0; h2g->info.tail = 0; h2g->info.head = 0; @@ -421,8 +424,8 @@ static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, struct iosys_map *map) { - g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32); - g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32); + g2h->info.size = CTB_G2H_BUFFER_DWORDS; + g2h->info.resv_space = G2H_ROOM_BUFFER_DWORDS; g2h->info.head = 0; g2h->info.tail = 0; g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head, @@ -725,6 +728,12 @@ void xe_guc_ct_stop(struct xe_guc_ct *ct) */ void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct) { + struct guc_ctb *g2h = &ct->ctbs.g2h; + u32 credits = CIRC_SPACE(0, 0, CTB_G2H_BUFFER_DWORDS) - G2H_ROOM_BUFFER_DWORDS; + + /* We should be back to guc_ct_ctb_g2h_init() values */ + xe_gt_assert(ct_to_gt(ct), g2h->info.space == credits); + /* * Since we're already in runtime suspend path, we shouldn't have pending * messages. But if there happen to be any, we'd probably want them to be From b64a14334ef3ebbcf70d11bc67d0934bdc0e390d Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Wed, 3 Dec 2025 18:03:55 +0530 Subject: [PATCH 111/187] drm/xe/throttle: Skip reason prefix while emitting array The newly introduced "reasons" attribute already signifies possible reasons for throttling and makes the prefix in individual attribute names redundant while emitting them as an array. Skip the prefix. Fixes: 83ccde67a3f7 ("drm/xe/gt_throttle: Avoid TOCTOU when monitoring reasons") Signed-off-by: Raag Jadav Reviewed-by: Sk Anirban Link: https://patch.msgid.link/20251203123355.571606-1-raag.jadav@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_gt_throttle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_gt_throttle.c b/drivers/gpu/drm/xe/xe_gt_throttle.c index 096a6187ff12..570358310e97 100644 --- a/drivers/gpu/drm/xe/xe_gt_throttle.c +++ b/drivers/gpu/drm/xe/xe_gt_throttle.c @@ -137,7 +137,7 @@ static ssize_t reasons_show(struct kobject *kobj, struct throttle_attribute *other_ta = kobj_attribute_to_throttle(kattr); if (other_ta->mask != U32_MAX && reasons & other_ta->mask) - ret += sysfs_emit_at(buff, ret, "%s ", (*pother)->name); + ret += sysfs_emit_at(buff, ret, "%s ", (*pother)->name + strlen("reason_")); } if (drm_WARN_ONCE(&xe->drm, !ret, "Unknown reason: %#x\n", reasons)) From 81e77c028848b7131d7653dd443a4049334a920b Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 3 Dec 2025 22:34:52 -0800 Subject: [PATCH 112/187] drm/xe/xe3p_xpc: Enable Indirect Ring State for xe3p_xpc The xe3p_xpc platform supports Indirect Ring State and it is required for the upcoming multi-queue feature. Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Reviewed-by: Matt Roper Link: https://patch.msgid.link/20251204063451.1180387-2-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 8dd0ea9d1d56..c8188a5b0f76 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -108,6 +108,7 @@ static const struct xe_graphics_desc graphics_xe2 = { static const struct xe_graphics_desc graphics_xe3p_xpc = { XE2_GFX_FEATURES, + .has_indirect_ring_state = 1, .hw_engine_mask = GENMASK(XE_HW_ENGINE_BCS8, XE_HW_ENGINE_BCS1) | GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0), From a00e305fba02a915cf2745bf6ef3f55537e65d57 Mon Sep 17 00:00:00 2001 From: Tomasz Lis Date: Thu, 4 Dec 2025 21:08:20 +0100 Subject: [PATCH 113/187] drm/xe/vf: Stop waiting for ring space on VF post migration recovery If wait for ring space started just before migration, it can delay the recovery process, by waiting without bailout path for up to 2 seconds. Two second wait for recovery is not acceptable, and if the ring was completely filled even without the migration temporarily stopping execution, then such a wait will result in up to a thousand new jobs (assuming constant flow) being added while the wait is happening. While this will not cause data corruption, it will lead to warning messages getting logged due to reset being scheduled on a GT under recovery. Also several seconds of unresponsiveness, as the backlog of jobs gets progressively executed. Add a bailout condition, to make sure the recovery starts without much delay. The recovery is expected to finish in about 100 ms when under moderate stress, so the condition verification period needs to be below that - settling at 64 ms. The theoretical max time which the recovery can take depends on how many requests can be emitted to engine rings and be pending execution. While stress testing, it was possible to reach 10k pending requests on rings when a platform with two GTs was used. This resulted in max recovery time of 5 seconds. But in real life situations, it is very unlikely that the amount of pending requests will ever exceed 100, and for that the recovery time will be around 50 ms - well within our claimed limit of 100ms. Fixes: a4dae94aad6a ("drm/xe/vf: Wakeup in GuC backend on VF post migration recovery") Signed-off-by: Tomasz Lis Reviewed-by: Matthew Brost Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251204200820.2206168-1-tomasz.lis@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index f3f2c8556a66..ff6fda84bf0f 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -722,21 +722,23 @@ static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); - unsigned int sleep_period_ms = 1; + unsigned int sleep_period_ms = 1, sleep_total_ms = 0; #define AVAILABLE_SPACE \ CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE) if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) { try_again: q->guc->wqi_head = parallel_read(xe, map, wq_desc.head); - if (wqi_size > AVAILABLE_SPACE) { - if (sleep_period_ms == 1024) { + if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) { + if (sleep_total_ms > 2000) { xe_gt_reset_async(q->gt); return -ENODEV; } msleep(sleep_period_ms); - sleep_period_ms <<= 1; + sleep_total_ms += sleep_period_ms; + if (sleep_period_ms < 64) + sleep_period_ms <<= 1; goto try_again; } } From f4e9fc967afdb53b1203f894fb4b68451a7fe202 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Mon, 8 Dec 2025 14:15:41 +0530 Subject: [PATCH 114/187] drm/xe/xe_survivability: Redesign survivability mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Redesign survivability mode to have only one value per file. 1) Retain the survivability_mode sysfs to indicate the type cat /sys/bus/pci/devices/0000\:03\:00.0/survivability_mode (Boot / Runtime) 2) Add survivability_info directory to expose boot breadcrumbs. Entries in survivability mode sysfs are only visible when boot breadcrumb registers are populated. /sys/bus/pci/devices/0000:03:00.0/survivability_info ├── aux_info0 ├── aux_info1 ├── aux_info2 ├── aux_info3 ├── aux_info4 ├── capability_info ├── postcode_trace └── postcode_trace_overflow Capability Info: Provides data about boot status and has bits that indicate the support for the other breadcrumbs Postcode Trace / Postcode Trace Overflow : Each postcode is represented as an 8-bit value and represents a boot failure event. When a new failure event is logged by Pcode the existing postcodes are shifted left. These entries provide a history of 8 postcodes. Auxiliary Info: Some failures have additional debug information. Signed-off-by: Riana Tauro Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20251208084539.3652902-5-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_survivability_mode.c | 228 +++++++++++------- .../gpu/drm/xe/xe_survivability_mode_types.h | 22 +- 2 files changed, 157 insertions(+), 93 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index 1662bfddd4bc..b6ff5da86a4d 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -19,8 +19,6 @@ #include "xe_pcode_api.h" #include "xe_vsec.h" -#define MAX_SCRATCH_MMIO 8 - /** * DOC: Survivability Mode * @@ -48,19 +46,38 @@ * * Refer :ref:`xe_configfs` for more details on how to use configfs * - * Survivability mode is indicated by the below admin-only readable sysfs which provides additional - * debug information:: + * Survivability mode is indicated by the below admin-only readable sysfs entry. It + * provides information about the type of survivability mode (Boot/Runtime). * - * /sys/bus/pci/devices//survivability_mode + * .. code-block:: shell * - * Capability Information: - * Provides boot status - * Postcode Information: - * Provides information about the failure - * Overflow Information - * Provides history of previous failures - * Auxiliary Information - * Certain failures may have information in addition to postcode information + * # cat /sys/bus/pci/devices//survivability_mode + * Boot + * + * + * Any additional debug information if present will be visible under the directory + * ``survivability_info``:: + * + * /sys/bus/pci/devices//survivability_info/ + * ├── aux_info0 + * ├── aux_info1 + * ├── aux_info2 + * ├── aux_info3 + * ├── aux_info4 + * ├── capability_info + * ├── fdo_mode + * ├── postcode_trace + * └── postcode_trace_overflow + * + * This directory has the following attributes + * + * - ``capability_info`` : Indicates Boot status and support for additional information + * + * - ``postcode_trace``, ``postcode_trace_overflow`` : Each postcode is a 8bit value and + * represents a boot failure event. When a new failure event is logged by PCODE the + * existing postcodes are shifted left. These entries provide a history of 8 postcodes. + * + * - ``aux_info`` : Some failures have additional debug information * * Runtime Survivability * ===================== @@ -68,60 +85,76 @@ * Certain runtime firmware errors can cause the device to enter a wedged state * (:ref:`xe-device-wedging`) requiring a firmware flash to restore normal operation. * Runtime Survivability Mode indicates that a firmware flash is necessary to recover the device and - * is indicated by the presence of survivability mode sysfs:: - * - * /sys/bus/pci/devices//survivability_mode - * + * is indicated by the presence of survivability mode sysfs. * Survivability mode sysfs provides information about the type of survivability mode. * + * .. code-block:: shell + * + * # cat /sys/bus/pci/devices//survivability_mode + * Runtime + * * When such errors occur, userspace is notified with the drm device wedged uevent and runtime * survivability mode. User can then initiate a firmware flash using userspace tools like fwupd * to restore device to normal operation. */ +static const char * const reg_map[] = { + [CAPABILITY_INFO] = "Capability Info", + [POSTCODE_TRACE] = "Postcode trace", + [POSTCODE_TRACE_OVERFLOW] = "Postcode trace overflow", + [AUX_INFO0] = "Auxiliary Info 0", + [AUX_INFO1] = "Auxiliary Info 1", + [AUX_INFO2] = "Auxiliary Info 2", + [AUX_INFO3] = "Auxiliary Info 3", + [AUX_INFO4] = "Auxiliary Info 4", +}; + +struct xe_survivability_attribute { + struct device_attribute attr; + u8 index; +}; + +static struct +xe_survivability_attribute *dev_attr_to_survivability_attr(struct device_attribute *attr) +{ + return container_of(attr, struct xe_survivability_attribute, attr); +} + static u32 aux_history_offset(u32 reg_value) { return REG_FIELD_GET(AUXINFO_HISTORY_OFFSET, reg_value); } -static void set_survivability_info(struct xe_mmio *mmio, struct xe_survivability_info *info, - int id, char *name) +static void set_survivability_info(struct xe_mmio *mmio, u32 *info, int id) { - strscpy(info[id].name, name, sizeof(info[id].name)); - info[id].reg = PCODE_SCRATCH(id).raw; - info[id].value = xe_mmio_read32(mmio, PCODE_SCRATCH(id)); + info[id] = xe_mmio_read32(mmio, PCODE_SCRATCH(id)); } static void populate_survivability_info(struct xe_device *xe) { struct xe_survivability *survivability = &xe->survivability; - struct xe_survivability_info *info = survivability->info; + u32 *info = survivability->info; struct xe_mmio *mmio; u32 id = 0, reg_value; - char name[NAME_MAX]; int index; mmio = xe_root_tile_mmio(xe); - set_survivability_info(mmio, info, id, "Capability Info"); - reg_value = info[id].value; + set_survivability_info(mmio, info, CAPABILITY_INFO); + reg_value = info[CAPABILITY_INFO]; if (reg_value & HISTORY_TRACKING) { - id++; - set_survivability_info(mmio, info, id, "Postcode Info"); + set_survivability_info(mmio, info, POSTCODE_TRACE); - if (reg_value & OVERFLOW_SUPPORT) { - id = REG_FIELD_GET(OVERFLOW_REG_OFFSET, reg_value); - set_survivability_info(mmio, info, id, "Overflow Info"); - } + if (reg_value & OVERFLOW_SUPPORT) + set_survivability_info(mmio, info, POSTCODE_TRACE_OVERFLOW); } if (reg_value & AUXINFO_SUPPORT) { id = REG_FIELD_GET(AUXINFO_REG_OFFSET, reg_value); - for (index = 0; id && reg_value; index++, reg_value = info[id].value, - id = aux_history_offset(reg_value)) { - snprintf(name, NAME_MAX, "Auxiliary Info %d", index); - set_survivability_info(mmio, info, id, name); + for (index = 0; id >= AUX_INFO0 && id < MAX_SCRATCH_REG; index++) { + set_survivability_info(mmio, info, id); + id = aux_history_offset(info[id]); } } } @@ -130,15 +163,14 @@ static void log_survivability_info(struct pci_dev *pdev) { struct xe_device *xe = pdev_to_xe_device(pdev); struct xe_survivability *survivability = &xe->survivability; - struct xe_survivability_info *info = survivability->info; + u32 *info = survivability->info; int id; dev_info(&pdev->dev, "Survivability Boot Status : Critical Failure (%d)\n", survivability->boot_status); - for (id = 0; id < MAX_SCRATCH_MMIO; id++) { - if (info[id].reg) - dev_info(&pdev->dev, "%s: 0x%x - 0x%x\n", info[id].name, - info[id].reg, info[id].value); + for (id = 0; id < MAX_SCRATCH_REG; id++) { + if (info[id]) + dev_info(&pdev->dev, "%s: 0x%x\n", reg_map[id], info[id]); } } @@ -156,43 +188,87 @@ static ssize_t survivability_mode_show(struct device *dev, struct pci_dev *pdev = to_pci_dev(dev); struct xe_device *xe = pdev_to_xe_device(pdev); struct xe_survivability *survivability = &xe->survivability; - struct xe_survivability_info *info = survivability->info; - int index = 0, count = 0; - count += sysfs_emit_at(buff, count, "Survivability mode type: %s\n", - survivability->type ? "Runtime" : "Boot"); - - if (!check_boot_failure(xe)) - return count; - - for (index = 0; index < MAX_SCRATCH_MMIO; index++) { - if (info[index].reg) - count += sysfs_emit_at(buff, count, "%s: 0x%x - 0x%x\n", info[index].name, - info[index].reg, info[index].value); - } - - return count; + return sysfs_emit(buff, "%s\n", survivability->type ? "Runtime" : "Boot"); } static DEVICE_ATTR_ADMIN_RO(survivability_mode); +static ssize_t survivability_info_show(struct device *dev, + struct device_attribute *attr, char *buff) +{ + struct xe_survivability_attribute *sa = dev_attr_to_survivability_attr(attr); + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + struct xe_survivability *survivability = &xe->survivability; + u32 *info = survivability->info; + + return sysfs_emit(buff, "0x%x\n", info[sa->index]); +} + +#define SURVIVABILITY_ATTR_RO(name, _index) \ + struct xe_survivability_attribute attr_##name = { \ + .attr = __ATTR(name, 0400, survivability_info_show, NULL), \ + .index = _index, \ + } + +SURVIVABILITY_ATTR_RO(capability_info, CAPABILITY_INFO); +SURVIVABILITY_ATTR_RO(postcode_trace, POSTCODE_TRACE); +SURVIVABILITY_ATTR_RO(postcode_trace_overflow, POSTCODE_TRACE_OVERFLOW); +SURVIVABILITY_ATTR_RO(aux_info0, AUX_INFO0); +SURVIVABILITY_ATTR_RO(aux_info1, AUX_INFO1); +SURVIVABILITY_ATTR_RO(aux_info2, AUX_INFO2); +SURVIVABILITY_ATTR_RO(aux_info3, AUX_INFO3); +SURVIVABILITY_ATTR_RO(aux_info4, AUX_INFO4); + static void xe_survivability_mode_fini(void *arg) { struct xe_device *xe = arg; struct pci_dev *pdev = to_pci_dev(xe->drm.dev); struct device *dev = &pdev->dev; - sysfs_remove_file(&dev->kobj, &dev_attr_survivability_mode.attr); + device_remove_file(dev, &dev_attr_survivability_mode); } +static umode_t survivability_info_attrs_visible(struct kobject *kobj, struct attribute *attr, + int idx) +{ + struct xe_device *xe = kdev_to_xe_device(kobj_to_dev(kobj)); + struct xe_survivability *survivability = &xe->survivability; + u32 *info = survivability->info; + + if (info[idx]) + return 0400; + + return 0; +} + +/* Attributes are ordered according to enum scratch_reg */ +static struct attribute *survivability_info_attrs[] = { + &attr_capability_info.attr.attr, + &attr_postcode_trace.attr.attr, + &attr_postcode_trace_overflow.attr.attr, + &attr_aux_info0.attr.attr, + &attr_aux_info1.attr.attr, + &attr_aux_info2.attr.attr, + &attr_aux_info3.attr.attr, + &attr_aux_info4.attr.attr, + NULL, +}; + +static const struct attribute_group survivability_info_group = { + .name = "survivability_info", + .attrs = survivability_info_attrs, + .is_visible = survivability_info_attrs_visible, +}; + static int create_survivability_sysfs(struct pci_dev *pdev) { struct device *dev = &pdev->dev; struct xe_device *xe = pdev_to_xe_device(pdev); int ret; - /* create survivability mode sysfs */ - ret = sysfs_create_file(&dev->kobj, &dev_attr_survivability_mode.attr); + ret = device_create_file(dev, &dev_attr_survivability_mode); if (ret) { dev_warn(dev, "Failed to create survivability sysfs files\n"); return ret; @@ -203,6 +279,12 @@ static int create_survivability_sysfs(struct pci_dev *pdev) if (ret) return ret; + if (check_boot_failure(xe)) { + ret = devm_device_add_group(dev, &survivability_info_group); + if (ret) + return ret; + } + return 0; } @@ -239,25 +321,6 @@ static int enable_boot_survivability_mode(struct pci_dev *pdev) return ret; } -static int init_survivability_mode(struct xe_device *xe) -{ - struct xe_survivability *survivability = &xe->survivability; - struct xe_survivability_info *info; - - survivability->size = MAX_SCRATCH_MMIO; - - info = devm_kcalloc(xe->drm.dev, survivability->size, sizeof(*info), - GFP_KERNEL); - if (!info) - return -ENOMEM; - - survivability->info = info; - - populate_survivability_info(xe); - - return 0; -} - /** * xe_survivability_mode_is_boot_enabled- check if boot survivability mode is enabled * @xe: xe device instance @@ -325,9 +388,7 @@ int xe_survivability_mode_runtime_enable(struct xe_device *xe) return -EINVAL; } - ret = init_survivability_mode(xe); - if (ret) - return ret; + populate_survivability_info(xe); ret = create_survivability_sysfs(pdev); if (ret) @@ -356,14 +417,11 @@ int xe_survivability_mode_boot_enable(struct xe_device *xe) { struct xe_survivability *survivability = &xe->survivability; struct pci_dev *pdev = to_pci_dev(xe->drm.dev); - int ret; if (!xe_survivability_mode_is_requested(xe)) return 0; - ret = init_survivability_mode(xe); - if (ret) - return ret; + populate_survivability_info(xe); /* Log breadcrumbs but do not enter survivability mode for Critical boot errors */ if (survivability->boot_status == CRITICAL_FAILURE) { diff --git a/drivers/gpu/drm/xe/xe_survivability_mode_types.h b/drivers/gpu/drm/xe/xe_survivability_mode_types.h index cd65a5d167c9..f31b3907d933 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode_types.h +++ b/drivers/gpu/drm/xe/xe_survivability_mode_types.h @@ -9,23 +9,29 @@ #include #include +enum scratch_reg { + CAPABILITY_INFO, + POSTCODE_TRACE, + POSTCODE_TRACE_OVERFLOW, + AUX_INFO0, + AUX_INFO1, + AUX_INFO2, + AUX_INFO3, + AUX_INFO4, + MAX_SCRATCH_REG, +}; + enum xe_survivability_type { XE_SURVIVABILITY_TYPE_BOOT, XE_SURVIVABILITY_TYPE_RUNTIME, }; -struct xe_survivability_info { - char name[NAME_MAX]; - u32 reg; - u32 value; -}; - /** * struct xe_survivability: Contains survivability mode information */ struct xe_survivability { - /** @info: struct that holds survivability info from scratch registers */ - struct xe_survivability_info *info; + /** @info: survivability debug info */ + u32 info[MAX_SCRATCH_REG]; /** @size: number of scratch registers */ u32 size; From 1987ea95ac37ea3f9299d220974676207d5262f6 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Mon, 8 Dec 2025 14:15:42 +0530 Subject: [PATCH 115/187] drm/xe/xe_survivability: Add support for survivability mode v2 v2 survivability breadcrumbs introduces a new mode called SPI Flash Descriptor Override mode (FDO). This is enabled by PCODE when MEI itself fails and firmware cannot be updated via MEI using igsc. This mode provides the ability to update the firmware directly via SPI driver. Xe KMD initializes the nvm aux driver if FDO mode is enabled. Userspace should check FDO mode entry in survivability info sysfs before using the SPI driver to update firmware. /sys/bus/pci/devices//survivability_info/fdo_mode v2 also supports survivability mode for critical boot errors. Signed-off-by: Riana Tauro Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20251208084539.3652902-6-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_pcode_api.h | 2 + drivers/gpu/drm/xe/xe_survivability_mode.c | 44 ++++++++++++++++--- .../gpu/drm/xe/xe_survivability_mode_types.h | 6 +++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pcode_api.h b/drivers/gpu/drm/xe/xe_pcode_api.h index 70dcd6625680..975892d6b230 100644 --- a/drivers/gpu/drm/xe/xe_pcode_api.h +++ b/drivers/gpu/drm/xe/xe_pcode_api.h @@ -77,11 +77,13 @@ #define PCODE_SCRATCH(x) XE_REG(0x138320 + ((x) * 4)) /* PCODE_SCRATCH0 */ +#define BREADCRUMB_VERSION REG_GENMASK(31, 29) #define AUXINFO_REG_OFFSET REG_GENMASK(17, 15) #define OVERFLOW_REG_OFFSET REG_GENMASK(14, 12) #define HISTORY_TRACKING REG_BIT(11) #define OVERFLOW_SUPPORT REG_BIT(10) #define AUXINFO_SUPPORT REG_BIT(9) +#define FDO_MODE REG_BIT(4) #define BOOT_STATUS REG_GENMASK(3, 1) #define CRITICAL_FAILURE 4 #define NON_CRITICAL_FAILURE 7 diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index b6ff5da86a4d..7520d7b7f9b8 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -16,6 +16,7 @@ #include "xe_heci_gsc.h" #include "xe_i2c.h" #include "xe_mmio.h" +#include "xe_nvm.h" #include "xe_pcode_api.h" #include "xe_vsec.h" @@ -79,6 +80,11 @@ * * - ``aux_info`` : Some failures have additional debug information * + * - ``fdo_mode`` : To allow recovery in scenarios where MEI itself fails, a new SPI Flash + * Descriptor Override (FDO) mode is added in v2 survivability breadcrumbs. This mode is enabled + * by PCODE and provides the ability to directly update the firmware via SPI Driver without + * any dependency on MEI. Xe KMD initializes the nvm aux driver if FDO mode is enabled. + * * Runtime Survivability * ===================== * @@ -109,6 +115,8 @@ static const char * const reg_map[] = { [AUX_INFO4] = "Auxiliary Info 4", }; +#define FDO_INFO (MAX_SCRATCH_REG + 1) + struct xe_survivability_attribute { struct device_attribute attr; u8 index; @@ -142,6 +150,11 @@ static void populate_survivability_info(struct xe_device *xe) set_survivability_info(mmio, info, CAPABILITY_INFO); reg_value = info[CAPABILITY_INFO]; + survivability->version = REG_FIELD_GET(BREADCRUMB_VERSION, reg_value); + /* FDO mode is exposed only from version 2 */ + if (survivability->version >= 2) + survivability->fdo_mode = REG_FIELD_GET(FDO_MODE, reg_value); + if (reg_value & HISTORY_TRACKING) { set_survivability_info(mmio, info, POSTCODE_TRACE); @@ -203,6 +216,9 @@ static ssize_t survivability_info_show(struct device *dev, struct xe_survivability *survivability = &xe->survivability; u32 *info = survivability->info; + if (sa->index == FDO_INFO) + return sysfs_emit(buff, "%s\n", str_enabled_disabled(survivability->fdo_mode)); + return sysfs_emit(buff, "0x%x\n", info[sa->index]); } @@ -220,13 +236,18 @@ SURVIVABILITY_ATTR_RO(aux_info1, AUX_INFO1); SURVIVABILITY_ATTR_RO(aux_info2, AUX_INFO2); SURVIVABILITY_ATTR_RO(aux_info3, AUX_INFO3); SURVIVABILITY_ATTR_RO(aux_info4, AUX_INFO4); +SURVIVABILITY_ATTR_RO(fdo_mode, FDO_INFO); static void xe_survivability_mode_fini(void *arg) { struct xe_device *xe = arg; + struct xe_survivability *survivability = &xe->survivability; struct pci_dev *pdev = to_pci_dev(xe->drm.dev); struct device *dev = &pdev->dev; + if (survivability->fdo_mode) + xe_nvm_fini(xe); + device_remove_file(dev, &dev_attr_survivability_mode); } @@ -237,7 +258,10 @@ static umode_t survivability_info_attrs_visible(struct kobject *kobj, struct att struct xe_survivability *survivability = &xe->survivability; u32 *info = survivability->info; - if (info[idx]) + /* FDO mode is visible only when supported */ + if (idx >= MAX_SCRATCH_REG && survivability->version >= 2) + return 0400; + else if (info[idx]) return 0400; return 0; @@ -253,6 +277,7 @@ static struct attribute *survivability_info_attrs[] = { &attr_aux_info2.attr.attr, &attr_aux_info3.attr.attr, &attr_aux_info4.attr.attr, + &attr_fdo_mode.attr.attr, NULL, }; @@ -302,12 +327,16 @@ static int enable_boot_survivability_mode(struct pci_dev *pdev) /* Make sure xe_heci_gsc_init() knows about survivability mode */ survivability->mode = true; - ret = xe_heci_gsc_init(xe); - if (ret) - goto err; + xe_heci_gsc_init(xe); xe_vsec_init(xe); + if (survivability->fdo_mode) { + ret = xe_nvm_init(xe); + if (ret) + goto err; + } + ret = xe_i2c_probe(xe); if (ret) goto err; @@ -317,6 +346,7 @@ static int enable_boot_survivability_mode(struct pci_dev *pdev) return 0; err: + dev_err(dev, "Failed to enable Survivability Mode\n"); survivability->mode = false; return ret; } @@ -423,8 +453,10 @@ int xe_survivability_mode_boot_enable(struct xe_device *xe) populate_survivability_info(xe); - /* Log breadcrumbs but do not enter survivability mode for Critical boot errors */ - if (survivability->boot_status == CRITICAL_FAILURE) { + /* + * v2 supports survivability mode for critical errors + */ + if (survivability->version < 2 && survivability->boot_status == CRITICAL_FAILURE) { log_survivability_info(pdev); return -ENXIO; } diff --git a/drivers/gpu/drm/xe/xe_survivability_mode_types.h b/drivers/gpu/drm/xe/xe_survivability_mode_types.h index f31b3907d933..bd5dc1c955ff 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode_types.h +++ b/drivers/gpu/drm/xe/xe_survivability_mode_types.h @@ -44,6 +44,12 @@ struct xe_survivability { /** @type: survivability type */ enum xe_survivability_type type; + + /** @fdo_mode: indicates if FDO mode is enabled */ + bool fdo_mode; + + /** @version: breadcrumb version of survivability mode */ + u8 version; }; #endif /* _XE_SURVIVABILITY_MODE_TYPES_H_ */ From 74a658229bb027964713522706254b727373aa3b Mon Sep 17 00:00:00 2001 From: Julia Filipchuk Date: Mon, 24 Nov 2025 17:41:39 -0800 Subject: [PATCH 116/187] drm/xe/guc: Recommend GuC v70.53.0 for MTL, DG2, LNL UAPI compatibility version 1.26.0 Update recommended GuC version for MTL, DG2, LNL. Signed-off-by: Julia Filipchuk Reviewed-by: Daniele Ceraolo Spurio Signed-off-by: Vinay Belgaumkar Link: https://patch.msgid.link/20251125014134.2075988-14-julia.filipchuk@intel.com --- drivers/gpu/drm/xe/xe_uc_fw.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c index 622b76078567..810709f106e7 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.c +++ b/drivers/gpu/drm/xe/xe_uc_fw.c @@ -117,9 +117,9 @@ struct fw_blobs_by_type { #define XE_GUC_FIRMWARE_DEFS(fw_def, mmp_ver, major_ver) \ fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, guc, ptl, 70, 49, 4)) \ fw_def(BATTLEMAGE, GT_TYPE_ANY, major_ver(xe, guc, bmg, 70, 49, 4)) \ - fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, guc, lnl, 70, 45, 2)) \ - fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, guc, mtl, 70, 44, 1)) \ - fw_def(DG2, GT_TYPE_ANY, major_ver(i915, guc, dg2, 70, 45, 2)) \ + fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, guc, lnl, 70, 53, 0)) \ + fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, guc, mtl, 70, 53, 0)) \ + fw_def(DG2, GT_TYPE_ANY, major_ver(i915, guc, dg2, 70, 53, 0)) \ fw_def(DG1, GT_TYPE_ANY, major_ver(i915, guc, dg1, 70, 44, 1)) \ fw_def(ALDERLAKE_N, GT_TYPE_ANY, major_ver(i915, guc, tgl, 70, 44, 1)) \ fw_def(ALDERLAKE_P, GT_TYPE_ANY, major_ver(i915, guc, adlp, 70, 44, 1)) \ From 1898840d6c4405092243bb7dfcc399ccb1177498 Mon Sep 17 00:00:00 2001 From: Julia Filipchuk Date: Mon, 24 Nov 2025 17:41:40 -0800 Subject: [PATCH 117/187] drm/xe/guc: Recommend GuC v70.54.0 for BMG, PTL UAPI compatibility version 1.27.0 Update recommended GuC version for BMG, PTL. Signed-off-by: Julia Filipchuk Reviewed-by: Daniele Ceraolo Spurio Signed-off-by: Vinay Belgaumkar Link: https://patch.msgid.link/20251125014134.2075988-15-julia.filipchuk@intel.com --- drivers/gpu/drm/xe/xe_uc_fw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c index 810709f106e7..dcb4a32e7a64 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.c +++ b/drivers/gpu/drm/xe/xe_uc_fw.c @@ -115,8 +115,8 @@ struct fw_blobs_by_type { #define XE_GT_TYPE_ANY XE_GT_TYPE_UNINITIALIZED #define XE_GUC_FIRMWARE_DEFS(fw_def, mmp_ver, major_ver) \ - fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, guc, ptl, 70, 49, 4)) \ - fw_def(BATTLEMAGE, GT_TYPE_ANY, major_ver(xe, guc, bmg, 70, 49, 4)) \ + fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, guc, ptl, 70, 54, 0)) \ + fw_def(BATTLEMAGE, GT_TYPE_ANY, major_ver(xe, guc, bmg, 70, 54, 0)) \ fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, guc, lnl, 70, 53, 0)) \ fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, guc, mtl, 70, 53, 0)) \ fw_def(DG2, GT_TYPE_ANY, major_ver(i915, guc, dg2, 70, 53, 0)) \ From 3efadf028783a49ab2941294187c8b6dd86bf7da Mon Sep 17 00:00:00 2001 From: Junxiao Chang Date: Fri, 7 Nov 2025 11:31:52 +0800 Subject: [PATCH 118/187] drm/me/gsc: mei interrupt top half should be in irq disabled context MEI GSC interrupt comes from i915 or xe driver. It has top half and bottom half. Top half is called from i915/xe interrupt handler. It should be in irq disabled context. With RT kernel(PREEMPT_RT enabled), by default IRQ handler is in threaded IRQ. MEI GSC top half might be in threaded IRQ context. generic_handle_irq_safe API could be called from either IRQ or process context, it disables local IRQ then calls MEI GSC interrupt top half. This change fixes B580 GPU boot issue with RT enabled. Fixes: e02cea83d32d ("drm/xe/gsc: add Battlemage support") Tested-by: Baoli Zhang Signed-off-by: Junxiao Chang Reviewed-by: Sebastian Andrzej Siewior Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251107033152.834960-1-junxiao.chang@intel.com Signed-off-by: Maarten Lankhorst --- drivers/gpu/drm/xe/xe_heci_gsc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_heci_gsc.c b/drivers/gpu/drm/xe/xe_heci_gsc.c index 2b3d49dd394c..495cdd4f948d 100644 --- a/drivers/gpu/drm/xe/xe_heci_gsc.c +++ b/drivers/gpu/drm/xe/xe_heci_gsc.c @@ -223,7 +223,7 @@ void xe_heci_gsc_irq_handler(struct xe_device *xe, u32 iir) if (xe->heci_gsc.irq < 0) return; - ret = generic_handle_irq(xe->heci_gsc.irq); + ret = generic_handle_irq_safe(xe->heci_gsc.irq); if (ret) drm_err_ratelimited(&xe->drm, "error handling GSC irq: %d\n", ret); } @@ -243,7 +243,7 @@ void xe_heci_csc_irq_handler(struct xe_device *xe, u32 iir) if (xe->heci_gsc.irq < 0) return; - ret = generic_handle_irq(xe->heci_gsc.irq); + ret = generic_handle_irq_safe(xe->heci_gsc.irq); if (ret) drm_err_ratelimited(&xe->drm, "error handling GSC irq: %d\n", ret); } From 8e011f6d08abcd740a9c5157b2f79512a579bced Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Wed, 10 Dec 2025 17:50:00 +0100 Subject: [PATCH 119/187] drm/xe/hw_engine_group: Add stats for mode switching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GT stats interface is extended to include counters of how many queues are either interrupted or waited on in the hardware engine groups. This can help application debugging. v2: Rename to queue as those operations are queue-based (Matthew Brost) Cc: Matthew Brost Cc: Thomas Hellström Reviewed-by: Matthew Brost Link: https://lore.kernel.org/r/20251210165000.60789-1-francois.dugast@intel.com Signed-off-by: Francois Dugast --- drivers/gpu/drm/xe/xe_gt_stats.c | 4 ++++ drivers/gpu/drm/xe/xe_gt_stats_types.h | 2 ++ drivers/gpu/drm/xe/xe_hw_engine_group.c | 3 +++ 3 files changed, 9 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_gt_stats.c b/drivers/gpu/drm/xe/xe_gt_stats.c index 5f74706bab81..714045ad9354 100644 --- a/drivers/gpu/drm/xe/xe_gt_stats.c +++ b/drivers/gpu/drm/xe/xe_gt_stats.c @@ -66,6 +66,10 @@ static const char *const stat_description[__XE_GT_STATS_NUM_IDS] = { DEF_STAT_STR(SVM_4K_BIND_US, "svm_4K_bind_us"), DEF_STAT_STR(SVM_64K_BIND_US, "svm_64K_bind_us"), DEF_STAT_STR(SVM_2M_BIND_US, "svm_2M_bind_us"), + DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, + "hw_engine_group_suspend_lr_queue_count"), + DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, + "hw_engine_group_wait_dma_queue_count"), }; /** diff --git a/drivers/gpu/drm/xe/xe_gt_stats_types.h b/drivers/gpu/drm/xe/xe_gt_stats_types.h index d8348a8de2e1..aada5df421e5 100644 --- a/drivers/gpu/drm/xe/xe_gt_stats_types.h +++ b/drivers/gpu/drm/xe/xe_gt_stats_types.h @@ -44,6 +44,8 @@ enum xe_gt_stats_id { XE_GT_STATS_ID_SVM_4K_BIND_US, XE_GT_STATS_ID_SVM_64K_BIND_US, XE_GT_STATS_ID_SVM_2M_BIND_US, + XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, + XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, /* must be the last entry */ __XE_GT_STATS_NUM_IDS, }; diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c index fa4db5f23342..290205a266b8 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_group.c +++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c @@ -9,6 +9,7 @@ #include "xe_device.h" #include "xe_exec_queue.h" #include "xe_gt.h" +#include "xe_gt_stats.h" #include "xe_hw_engine_group.h" #include "xe_vm.h" @@ -203,6 +204,7 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group if (!xe_vm_in_fault_mode(q->vm)) continue; + xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1); need_resume = true; q->ops->suspend(q); } @@ -244,6 +246,7 @@ static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group if (xe_vm_in_lr_mode(q->vm)) continue; + xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1); fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm); timeout = dma_fence_wait(fence, false); dma_fence_put(fence); From a4ebfb9d95d78a12512b435a698ee6886d712571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Tue, 9 Dec 2025 21:49:20 +0100 Subject: [PATCH 120/187] drm/xe/bo: Don't include the CCS metadata in the dma-buf sg-table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some Xe bos are allocated with extra backing-store for the CCS metadata. It's never been the intention to share the CCS metadata when exporting such bos as dma-buf. Don't include it in the dma-buf sg-table. Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Cc: Rodrigo Vivi Cc: Matthew Brost Cc: Maarten Lankhorst Cc: # v6.8+ Signed-off-by: Thomas Hellström Reviewed-by: Matthew Brost Reviewed-by: Karol Wachowski Link: https://patch.msgid.link/20251209204920.224374-1-thomas.hellstrom@linux.intel.com --- drivers/gpu/drm/xe/xe_dma_buf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index 54e42960daad..7c74a31d4486 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -124,7 +124,7 @@ static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach, case XE_PL_TT: sgt = drm_prime_pages_to_sg(obj->dev, bo->ttm.ttm->pages, - bo->ttm.ttm->num_pages); + obj->size >> PAGE_SHIFT); if (IS_ERR(sgt)) return sgt; From 8d8cf42b03f149dcb545b547906306f3b474565e Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Wed, 10 Dec 2025 05:25:48 +0000 Subject: [PATCH 121/187] drm/xe/vf: Fix queuing of recovery work Ensure VF migration recovery work is only queued when no recovery is already queued and teardown is not in progress. Fixes: b47c0c07c350 ("drm/xe/vf: Teardown VF post migration worker on driver unload") Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Tomasz Lis Reviewed-by: Michal Wajdeczko Reviewed-by: Matthew Brost Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251210052546.622809-5-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_vf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index 3c806c8e5f3e..47612e67d462 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -781,7 +781,7 @@ static void vf_start_migration_recovery(struct xe_gt *gt) spin_lock(>->sriov.vf.migration.lock); - if (!gt->sriov.vf.migration.recovery_queued || + if (!gt->sriov.vf.migration.recovery_queued && !gt->sriov.vf.migration.recovery_teardown) { gt->sriov.vf.migration.recovery_queued = true; WRITE_ONCE(gt->sriov.vf.migration.recovery_inprogress, true); From 7319c2ceb2d74d54f24b7f5409b97aed406f0189 Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Wed, 10 Dec 2025 05:25:49 +0000 Subject: [PATCH 122/187] drm/xe/vf: Reset recovery_queued after issuing RESFIX_START During VF_RESTORE or VF_RESUME, the GuC sends a migration interrupt and clears the RESFIX_START marker. If migration or resume occurs before the VF issues its own RESFIX_START, VF KMD may receive two back-to-back migration interrupts. VF then sends RESFIX_START to indicate the beginning of fixups and RESFIX_DONE to mark completion. However, the second RESFIX_START fails because the GuC is already in the RUNNING state. Clear the recovery_queued flag after sending a RESFIX_START message to ignore duplicated IRQs seen before we start actual recovery. This ensures the state is reset only after the fixup process begins, avoiding redundant work item queuing. Fixes: b5fbb94341a2 ("drm/xe/vf: Introduce RESFIX start marker support") Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Cc: Tomasz Lis Reviewed-by: Matthew Brost Reviewed-by: Michal Wajdeczko Signed-off-by: Michal Wajdeczko Link: https://patch.msgid.link/20251210052546.622809-6-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_vf.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index 47612e67d462..b8b391cfc8eb 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -1171,10 +1171,6 @@ static bool vf_post_migration_shutdown(struct xe_gt *gt) return true; } - spin_lock_irq(>->sriov.vf.migration.lock); - gt->sriov.vf.migration.recovery_queued = false; - spin_unlock_irq(>->sriov.vf.migration.lock); - xe_guc_ct_flush_and_stop(>->uc.guc.ct); xe_guc_submit_pause_vf(>->uc.guc); xe_tlb_inval_reset(>->tlb_inval); @@ -1258,7 +1254,14 @@ static int vf_post_migration_resfix_done(struct xe_gt *gt, u16 marker) static int vf_post_migration_resfix_start(struct xe_gt *gt, u16 marker) { - return vf_resfix_start(gt, marker); + int err; + + err = vf_resfix_start(gt, marker); + + guard(spinlock_irq) (>->sriov.vf.migration.lock); + gt->sriov.vf.migration.recovery_queued = false; + + return err; } static u16 vf_post_migration_next_resfix_marker(struct xe_gt *gt) From ee26c3b2765a34d1055660cbfd4f00766dd25412 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:49 -0800 Subject: [PATCH 123/187] drm/xe/multi_queue: Add multi_queue_enable_mask to gt information Add multi_queue_enable_mask field to the gt information structure which is bitmask of all engine classes with multi queue support enabled. v2: Rename multi_queue_enable_mask to multi_queue_engine_class_mask (Matt Brost) Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-20-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_debugfs.c | 2 ++ drivers/gpu/drm/xe/xe_gt_types.h | 5 +++++ drivers/gpu/drm/xe/xe_pci.c | 1 + drivers/gpu/drm/xe/xe_pci_types.h | 1 + 4 files changed, 9 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index 0f8a96a05a8e..4fa423a82bea 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -93,6 +93,8 @@ static int info(struct seq_file *m, void *data) xe_force_wake_ref(gt_to_fw(gt), XE_FW_GT)); drm_printf(&p, "gt%d engine_mask 0x%llx\n", id, gt->info.engine_mask); + drm_printf(&p, "gt%d multi_queue_engine_class_mask 0x%x\n", id, + gt->info.multi_queue_engine_class_mask); } return 0; diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index 0a728180b6fe..5318d92fd473 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -140,6 +140,11 @@ struct xe_gt { u64 engine_mask; /** @info.gmdid: raw GMD_ID value from hardware */ u32 gmdid; + /** + * @multi_queue_engine_class_mask: Bitmask of engine classes with + * multi queue support enabled. + */ + u16 multi_queue_engine_class_mask; /** @info.id: Unique ID of this GT within the PCI Device */ u8 id; /** @info.has_indirect_ring_state: GT has indirect ring state support */ diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index c8188a5b0f76..16b3eb247439 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -764,6 +764,7 @@ static struct xe_gt *alloc_primary_gt(struct xe_tile *tile, gt->info.type = XE_GT_TYPE_MAIN; gt->info.id = tile->id * xe->info.max_gt_per_tile; gt->info.has_indirect_ring_state = graphics_desc->has_indirect_ring_state; + gt->info.multi_queue_engine_class_mask = graphics_desc->multi_queue_engine_class_mask; gt->info.engine_mask = graphics_desc->hw_engine_mask; /* diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index f19f35359696..b06c108e25e6 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -60,6 +60,7 @@ struct xe_device_desc { struct xe_graphics_desc { u64 hw_engine_mask; /* hardware engines provided by graphics IP */ + u16 multi_queue_engine_class_mask; /* bitmask of engine classes which support multi queue */ u8 has_asid:1; u8 has_atomic_enable_pte_bit:1; From d9ec63474648a258094704ce223c9249fa7bb279 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:50 -0800 Subject: [PATCH 124/187] drm/xe/multi_queue: Add user interface for multi queue support Multi Queue is a new mode of execution supported by the compute and blitter copy command streamers (CCS and BCS, respectively). It is an enhancement of the existing hardware architecture and leverages the same submission model. It enables support for efficient, parallel execution of multiple queues within a single context. All the queues of a group must use the same address space (VM). The new DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE execution queue property supports creating a multi queue group and adding queues to a queue group. All queues of a multi queue group share the same context. A exec queue create ioctl call with above property specified with value DRM_XE_SUPER_GROUP_CREATE will create a new multi queue group with the queue being created as the primary queue (aka q0) of the group. To add secondary queues to the group, they need to be created with the above property with id of the primary queue as the value. The properties of the primary queue (like priority, timeslice) applies to the whole group. So, these properties can't be set for secondary queues of a group. Once destroyed, the secondary queues of a multi queue group can't be replaced. However, they can be dynamically added to the group up to a total of 64 queues per group. Once the primary queue is destroyed, secondary queues can't be added to the queue group. v2: Remove group->lock, fix xe_exec_queue_group_add()/delete() function semantics, add additional comments, remove unused group->list_lock, add XE_BO_FLAG_GGTT_INVALIDATE for cgp bo, Assert LRC is valid, update uapi kernel doc. (Matt Brost) v3: Use XE_BO_FLAG_PINNED_LATE_RESTORE/USER_VRAM/GGTT_INVALIDATE flags for cgp bo (Matt) v4: Ensure queue is not a vm_bind queue uapi change due to rebase Signed-off-by: Stuart Summers Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-21-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.c | 197 ++++++++++++++++++++++- drivers/gpu/drm/xe/xe_exec_queue.h | 47 ++++++ drivers/gpu/drm/xe/xe_exec_queue_types.h | 26 +++ include/uapi/drm/xe_drm.h | 10 ++ 4 files changed, 278 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 02b75652d497..f76ec277c5af 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -13,6 +13,7 @@ #include #include +#include "xe_bo.h" #include "xe_dep_scheduler.h" #include "xe_device.h" #include "xe_gt.h" @@ -63,6 +64,33 @@ enum xe_exec_queue_sched_prop { static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, u64 extensions, int ext_number); +static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q) +{ + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_lrc *lrc; + unsigned long idx; + + if (xe_exec_queue_is_multi_queue_secondary(q)) { + /* + * Put pairs with get from xe_exec_queue_lookup() call + * in xe_exec_queue_group_validate(). + */ + xe_exec_queue_put(xe_exec_queue_multi_queue_primary(q)); + return; + } + + if (!group) + return; + + /* Primary queue cleanup */ + xa_for_each(&group->xa, idx, lrc) + xe_lrc_put(lrc); + + xa_destroy(&group->xa); + xe_bo_unpin_map_no_vm(group->cgp_bo); + kfree(group); +} + static void __xe_exec_queue_free(struct xe_exec_queue *q) { int i; @@ -73,6 +101,10 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q) if (xe_exec_queue_uses_pxp(q)) xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); + + if (xe_exec_queue_is_multi_queue(q)) + xe_exec_queue_group_cleanup(q); + if (q->vm) xe_vm_put(q->vm); @@ -588,6 +620,150 @@ static int exec_queue_set_hang_replay_state(struct xe_device *xe, return 0; } +static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue *q) +{ + struct xe_tile *tile = gt_to_tile(q->gt); + struct xe_exec_queue_group *group; + struct xe_bo *bo; + + group = kzalloc(sizeof(*group), GFP_KERNEL); + if (!group) + return -ENOMEM; + + bo = xe_bo_create_pin_map_novm(xe, tile, SZ_4K, ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_PINNED_LATE_RESTORE | + XE_BO_FLAG_FORCE_USER_VRAM | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_GGTT, false); + if (IS_ERR(bo)) { + drm_err(&xe->drm, "CGP bo allocation for queue group failed: %ld\n", + PTR_ERR(bo)); + kfree(group); + return PTR_ERR(bo); + } + + xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K); + + group->primary = q; + group->cgp_bo = bo; + xa_init_flags(&group->xa, XA_FLAGS_ALLOC1); + q->multi_queue.group = group; + + return 0; +} + +static inline bool xe_exec_queue_supports_multi_queue(struct xe_exec_queue *q) +{ + return q->gt->info.multi_queue_engine_class_mask & BIT(q->class); +} + +static int xe_exec_queue_group_validate(struct xe_device *xe, struct xe_exec_queue *q, + u32 primary_id) +{ + struct xe_exec_queue_group *group; + struct xe_exec_queue *primary; + int ret; + + /* + * Get from below xe_exec_queue_lookup() pairs with put + * in xe_exec_queue_group_cleanup(). + */ + primary = xe_exec_queue_lookup(q->vm->xef, primary_id); + if (XE_IOCTL_DBG(xe, !primary)) + return -ENOENT; + + if (XE_IOCTL_DBG(xe, !xe_exec_queue_is_multi_queue_primary(primary)) || + XE_IOCTL_DBG(xe, q->vm != primary->vm) || + XE_IOCTL_DBG(xe, q->logical_mask != primary->logical_mask)) { + ret = -EINVAL; + goto put_primary; + } + + group = primary->multi_queue.group; + q->multi_queue.valid = true; + q->multi_queue.group = group; + + return 0; +put_primary: + xe_exec_queue_put(primary); + return ret; +} + +#define XE_MAX_GROUP_SIZE 64 +static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q) +{ + struct xe_exec_queue_group *group = q->multi_queue.group; + u32 pos; + int err; + + xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q)); + + /* Primary queue holds a reference to LRCs of all secondary queues */ + err = xa_alloc(&group->xa, &pos, xe_lrc_get(q->lrc[0]), + XA_LIMIT(1, XE_MAX_GROUP_SIZE - 1), GFP_KERNEL); + if (XE_IOCTL_DBG(xe, err)) { + xe_lrc_put(q->lrc[0]); + + /* It is invalid if queue group limit is exceeded */ + if (err == -EBUSY) + err = -EINVAL; + + return err; + } + + q->multi_queue.pos = pos; + + return 0; +} + +static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queue *q) +{ + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_lrc *lrc; + + xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q)); + + lrc = xa_erase(&group->xa, q->multi_queue.pos); + xe_assert(xe, lrc); + xe_lrc_put(lrc); +} + +static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q, + u64 value) +{ + if (XE_IOCTL_DBG(xe, !xe_exec_queue_supports_multi_queue(q))) + return -ENODEV; + + if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe))) + return -EOPNOTSUPP; + + if (XE_IOCTL_DBG(xe, !q->vm->xef)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, xe_exec_queue_is_parallel(q))) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, xe_exec_queue_is_multi_queue(q))) + return -EINVAL; + + if (value & DRM_XE_MULTI_GROUP_CREATE) { + if (XE_IOCTL_DBG(xe, value & ~DRM_XE_MULTI_GROUP_CREATE)) + return -EINVAL; + + q->multi_queue.valid = true; + q->multi_queue.is_primary = true; + q->multi_queue.pos = 0; + return 0; + } + + /* While adding secondary queues, the upper 32 bits must be 0 */ + if (XE_IOCTL_DBG(xe, value & (~0ull << 32))) + return -EINVAL; + + return xe_exec_queue_group_validate(xe, q, value); +} + typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, struct xe_exec_queue *q, u64 value); @@ -597,6 +773,7 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type, [DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE] = exec_queue_set_hang_replay_state, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP] = exec_queue_set_multi_group, }; static int exec_queue_user_ext_set_property(struct xe_device *xe, @@ -618,7 +795,8 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe, XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE && ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE && - ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE)) + ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE && + ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP)) return -EINVAL; idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); @@ -667,6 +845,12 @@ static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue return exec_queue_user_extensions(xe, q, ext.next_extension, ++ext_number); + if (xe_exec_queue_is_multi_queue_primary(q)) { + err = xe_exec_queue_group_init(xe, q); + if (XE_IOCTL_DBG(xe, err)) + return err; + } + return 0; } @@ -821,12 +1005,18 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, if (IS_ERR(q)) return PTR_ERR(q); + if (xe_exec_queue_is_multi_queue_secondary(q)) { + err = xe_exec_queue_group_add(xe, q); + if (XE_IOCTL_DBG(xe, err)) + goto put_exec_queue; + } + if (xe_vm_in_preempt_fence_mode(vm)) { q->lr.context = dma_fence_context_alloc(1); err = xe_vm_add_compute_exec_queue(vm, q); if (XE_IOCTL_DBG(xe, err)) - goto put_exec_queue; + goto delete_queue_group; } if (q->vm && q->hwe->hw_engine_group) { @@ -849,6 +1039,9 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, kill_exec_queue: xe_exec_queue_kill(q); +delete_queue_group: + if (xe_exec_queue_is_multi_queue_secondary(q)) + xe_exec_queue_group_delete(xe, q); put_exec_queue: xe_exec_queue_put(q); return err; diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index fda4d4f9bda8..e6daa40003f2 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -66,6 +66,53 @@ static inline bool xe_exec_queue_uses_pxp(struct xe_exec_queue *q) return q->pxp.type; } +/** + * xe_exec_queue_is_multi_queue() - Whether an exec_queue is part of a queue group. + * @q: The exec_queue + * + * Return: True if the exec_queue is part of a queue group, false otherwise. + */ +static inline bool xe_exec_queue_is_multi_queue(struct xe_exec_queue *q) +{ + return q->multi_queue.valid; +} + +/** + * xe_exec_queue_is_multi_queue_primary() - Whether an exec_queue is primary queue + * of a multi queue group. + * @q: The exec_queue + * + * Return: True if @q is primary queue of a queue group, false otherwise. + */ +static inline bool xe_exec_queue_is_multi_queue_primary(struct xe_exec_queue *q) +{ + return q->multi_queue.is_primary; +} + +/** + * xe_exec_queue_is_multi_queue_secondary() - Whether an exec_queue is secondary queue + * of a multi queue group. + * @q: The exec_queue + * + * Return: True if @q is secondary queue of a queue group, false otherwise. + */ +static inline bool xe_exec_queue_is_multi_queue_secondary(struct xe_exec_queue *q) +{ + return xe_exec_queue_is_multi_queue(q) && !xe_exec_queue_is_multi_queue_primary(q); +} + +/** + * xe_exec_queue_multi_queue_primary() - Get multi queue group's primary queue + * @q: The exec_queue + * + * If @q belongs to a multi queue group, then the primary queue of the group will + * be returned. Otherwise, @q will be returned. + */ +static inline struct xe_exec_queue *xe_exec_queue_multi_queue_primary(struct xe_exec_queue *q) +{ + return xe_exec_queue_is_multi_queue(q) ? q->multi_queue.group->primary : q; +} + bool xe_exec_queue_is_lr(struct xe_exec_queue *q); bool xe_exec_queue_is_idle(struct xe_exec_queue *q); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 3ba10632dcd6..29feafb42e0a 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -32,6 +32,20 @@ enum xe_exec_queue_priority { XE_EXEC_QUEUE_PRIORITY_COUNT }; +/** + * struct xe_exec_queue_group - Execution multi queue group + * + * Contains multi queue group information. + */ +struct xe_exec_queue_group { + /** @primary: Primary queue of this group */ + struct xe_exec_queue *primary; + /** @cgp_bo: BO for the Context Group Page */ + struct xe_bo *cgp_bo; + /** @xa: xarray to store LRCs */ + struct xarray xa; +}; + /** * struct xe_exec_queue - Execution queue * @@ -111,6 +125,18 @@ struct xe_exec_queue { struct xe_guc_exec_queue *guc; }; + /** @multi_queue: Multi queue information */ + struct { + /** @multi_queue.group: Queue group information */ + struct xe_exec_queue_group *group; + /** @multi_queue.pos: Position of queue within the multi-queue group */ + u8 pos; + /** @multi_queue.valid: Queue belongs to a multi queue group */ + u8 valid:1; + /** @multi_queue.is_primary: Is primary queue (Q0) of the group */ + u8 is_primary:1; + } multi_queue; + /** @sched_props: scheduling properties */ struct { /** @sched_props.timeslice_us: timeslice period in micro-seconds */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 876a076fa6c0..19a8ae856a17 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1272,6 +1272,14 @@ struct drm_xe_vm_bind { * Given that going into a power-saving state kills PXP HWDRM sessions, * runtime PM will be blocked while queues of this type are alive. * All PXP queues will be killed if a PXP invalidation event occurs. + * - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP - Create a multi-queue group + * or add secondary queues to a multi-queue group. + * If the extension's 'value' field has %DRM_XE_MULTI_GROUP_CREATE flag set, + * then a new multi-queue group is created with this queue as the primary queue + * (Q0). Otherwise, the queue gets added to the multi-queue group whose primary + * queue's exec_queue_id is specified in the lower 32 bits of the 'value' field. + * All the other non-relevant bits of extension's 'value' field while adding the + * primary or the secondary queues of the group must be set to 0. * * The example below shows how to use @drm_xe_exec_queue_create to create * a simple exec_queue (no parallel submission) of class @@ -1313,6 +1321,8 @@ struct drm_xe_exec_queue_create { #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE 1 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE 2 #define DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE 3 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP 4 +#define DRM_XE_MULTI_GROUP_CREATE (1ull << 63) /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; From bc5775c59258e1da6fe393845ee5cdd25ca618f3 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:51 -0800 Subject: [PATCH 125/187] drm/xe/multi_queue: Add GuC interface for multi queue support Implement GuC commands and response along with the Context Group Page (CGP) interface for multi queue support. Ensure that only primary queue (q0) of a multi queue group communicate with GuC. The secondary queues of the group only need to maintain LRCA and interface with drm scheduler. Use primary queue's submit_wq for all secondary queues of a multi queue group. This serialization avoids any locking around CGP synchronization with GuC. v2: Fix G2H_LEN_DW_MULTI_QUEUE_CONTEXT value, add more comments (Matt Brost) v3: Minor code refactro, use xe_gt_assert v4: Use xe_guc_ct_wake_waiters(), remove vf recovery support (Matt Brost) Signed-off-by: Stuart Summers Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-22-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/abi/guc_actions_abi.h | 3 + drivers/gpu/drm/xe/xe_exec_queue_types.h | 2 + drivers/gpu/drm/xe/xe_guc_ct.c | 4 + drivers/gpu/drm/xe/xe_guc_fwif.h | 3 + drivers/gpu/drm/xe/xe_guc_submit.c | 278 +++++++++++++++++++++-- drivers/gpu/drm/xe/xe_guc_submit.h | 1 + 6 files changed, 269 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/xe/abi/guc_actions_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_abi.h index 47756e4674a1..3e9fbed9cda6 100644 --- a/drivers/gpu/drm/xe/abi/guc_actions_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_actions_abi.h @@ -139,6 +139,9 @@ enum xe_guc_action { XE_GUC_ACTION_DEREGISTER_G2G = 0x4508, XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE = 0x4600, XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC = 0x4601, + XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE = 0x4602, + XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC = 0x4603, + XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE = 0x4604, XE_GUC_ACTION_CLIENT_SOFT_RESET = 0x5507, XE_GUC_ACTION_SET_ENG_UTIL_BUFF = 0x550A, XE_GUC_ACTION_SET_DEVICE_ENGINE_ACTIVITY_BUFFER = 0x550C, diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 29feafb42e0a..06fb518b8533 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -44,6 +44,8 @@ struct xe_exec_queue_group { struct xe_bo *cgp_bo; /** @xa: xarray to store LRCs */ struct xarray xa; + /** @sync_pending: CGP_SYNC_DONE g2h response pending */ + bool sync_pending; }; /** diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 648f0f523abb..4d5b4ed357cc 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -1401,6 +1401,7 @@ static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) lockdep_assert_held(&ct->lock); switch (action) { + case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: @@ -1614,6 +1615,9 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) ret = xe_guc_g2g_test_notification(guc, payload, adj_len); break; #endif + case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: + ret = xe_guc_exec_queue_cgp_sync_done_handler(guc, payload, adj_len); + break; default: xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); } diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h index 7d93c2749485..e27f0088f24f 100644 --- a/drivers/gpu/drm/xe/xe_guc_fwif.h +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -16,6 +16,7 @@ #define G2H_LEN_DW_DEREGISTER_CONTEXT 3 #define G2H_LEN_DW_TLB_INVALIDATE 3 #define G2H_LEN_DW_G2G_NOTIFY_MIN 3 +#define G2H_LEN_DW_MULTI_QUEUE_CONTEXT 3 #define GUC_ID_MAX 65535 #define GUC_ID_UNKNOWN 0xffffffff @@ -62,6 +63,8 @@ struct guc_ctxt_registration_info { u32 wq_base_lo; u32 wq_base_hi; u32 wq_size; + u32 cgp_lo; + u32 cgp_hi; u32 hwlrca_lo; u32 hwlrca_hi; }; diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index ff6fda84bf0f..bafe42393d22 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -19,6 +19,7 @@ #include "abi/guc_klvs_abi.h" #include "regs/xe_lrc_layout.h" #include "xe_assert.h" +#include "xe_bo.h" #include "xe_devcoredump.h" #include "xe_device.h" #include "xe_exec_queue.h" @@ -541,7 +542,8 @@ static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) u32 slpc_exec_queue_freq_req = 0; u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); + xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q) && + !xe_exec_queue_is_multi_queue_secondary(q)); if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY) slpc_exec_queue_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE; @@ -561,6 +563,8 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue { struct exec_queue_policy policy; + xe_assert(guc_to_xe(guc), !xe_exec_queue_is_multi_queue_secondary(q)); + __guc_exec_queue_policy_start_klv(&policy, q->guc->id); __guc_exec_queue_policy_add_preemption_timeout(&policy, 1); @@ -568,6 +572,11 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue __guc_exec_queue_policy_action_size(&policy), 0, 0); } +static bool vf_recovery(struct xe_guc *guc) +{ + return xe_gt_recovery_pending(guc_to_gt(guc)); +} + #define parallel_read(xe_, map_, field_) \ xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ field_) @@ -575,6 +584,119 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ field_, val_) +#define CGP_VERSION_MAJOR_SHIFT 8 + +static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe, + struct xe_exec_queue *q) +{ + struct xe_exec_queue_group *group = q->multi_queue.group; + u32 guc_id = group->primary->guc->id; + + /* Currently implementing CGP version 1.0 */ + xe_map_wr(xe, &group->cgp_bo->vmap, 0, u32, + 1 << CGP_VERSION_MAJOR_SHIFT); + + xe_map_wr(xe, &group->cgp_bo->vmap, + (32 + q->multi_queue.pos * 2) * sizeof(u32), + u32, lower_32_bits(xe_lrc_descriptor(q->lrc[0]))); + + xe_map_wr(xe, &group->cgp_bo->vmap, + (33 + q->multi_queue.pos * 2) * sizeof(u32), + u32, guc_id); + + if (q->multi_queue.pos / 32) { + xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), + u32, BIT(q->multi_queue.pos % 32)); + xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), u32, 0); + } else { + xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), + u32, BIT(q->multi_queue.pos)); + xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), u32, 0); + } +} + +static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc, + struct xe_exec_queue *q, + const u32 *action, u32 len) +{ + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_device *xe = guc_to_xe(guc); + long ret; + + /* + * As all queues of a multi queue group use single drm scheduler + * submit workqueue, CGP synchronization with GuC are serialized. + * Hence, no locking is required here. + * Wait for any pending CGP_SYNC_DONE response before updating the + * CGP page and sending CGP_SYNC message. + * + * FIXME: Support VF migration + */ + ret = wait_event_timeout(guc->ct.wq, + !READ_ONCE(group->sync_pending) || + xe_guc_read_stopped(guc), HZ); + if (!ret || xe_guc_read_stopped(guc)) { + xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n"); + return; + } + + xe_guc_exec_queue_group_cgp_update(xe, q); + + WRITE_ONCE(group->sync_pending, true); + xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1); +} + +static void __register_exec_queue_group(struct xe_guc *guc, + struct xe_exec_queue *q, + struct guc_ctxt_registration_info *info) +{ +#define MAX_MULTI_QUEUE_REG_SIZE (8) + u32 action[MAX_MULTI_QUEUE_REG_SIZE]; + int len = 0; + + action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE; + action[len++] = info->flags; + action[len++] = info->context_idx; + action[len++] = info->engine_class; + action[len++] = info->engine_submit_mask; + action[len++] = 0; /* Reserved */ + action[len++] = info->cgp_lo; + action[len++] = info->cgp_hi; + + xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_REG_SIZE); +#undef MAX_MULTI_QUEUE_REG_SIZE + + /* + * The above XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE do expect a + * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response + * from guc. + */ + xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); +} + +static void xe_guc_exec_queue_group_add(struct xe_guc *guc, + struct xe_exec_queue *q) +{ +#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) + u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; + int len = 0; + + xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_multi_queue_secondary(q)); + + action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; + action[len++] = q->multi_queue.group->primary->guc->id; + + xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); +#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE + + /* + * The above XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC do expect a + * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response + * from guc. + */ + xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); +} + static void __register_mlrc_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q, struct guc_ctxt_registration_info *info) @@ -670,6 +792,13 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) info.flags = CONTEXT_REGISTRATION_FLAG_KMD | FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type); + if (xe_exec_queue_is_multi_queue(q)) { + struct xe_exec_queue_group *group = q->multi_queue.group; + + info.cgp_lo = xe_bo_ggtt_addr(group->cgp_bo); + info.cgp_hi = 0; + } + if (xe_exec_queue_is_parallel(q)) { u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); struct iosys_map map = xe_lrc_parallel_map(lrc); @@ -700,11 +829,18 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) set_exec_queue_registered(q); trace_xe_exec_queue_register(q); - if (xe_exec_queue_is_parallel(q)) + if (xe_exec_queue_is_multi_queue_primary(q)) + __register_exec_queue_group(guc, q, &info); + else if (xe_exec_queue_is_parallel(q)) __register_mlrc_exec_queue(guc, q, &info); - else + else if (!xe_exec_queue_is_multi_queue_secondary(q)) __register_exec_queue(guc, &info); - init_policies(guc, q); + + if (!xe_exec_queue_is_multi_queue_secondary(q)) + init_policies(guc, q); + + if (xe_exec_queue_is_multi_queue_secondary(q)) + xe_guc_exec_queue_group_add(guc, q); } static u32 wq_space_until_wrap(struct xe_exec_queue *q) @@ -712,11 +848,6 @@ static u32 wq_space_until_wrap(struct xe_exec_queue *q) return (WQ_SIZE - q->guc->wqi_tail); } -static bool vf_recovery(struct xe_guc *guc) -{ - return xe_gt_recovery_pending(guc_to_gt(guc)); -} - static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) { struct xe_guc *guc = exec_queue_to_guc(q); @@ -835,6 +966,12 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q)) return; + /* + * All queues in a multi-queue group will use the primary queue + * of the group to interface with GuC. + */ + q = xe_exec_queue_multi_queue_primary(q); + if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) { action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; action[len++] = q->guc->id; @@ -881,6 +1018,18 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) trace_xe_sched_job_run(job); if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) { + if (xe_exec_queue_is_multi_queue_secondary(q)) { + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + + if (exec_queue_killed_or_banned_or_wedged(primary)) { + killed_or_banned_or_wedged = true; + goto run_job_out; + } + + if (!exec_queue_registered(primary)) + register_exec_queue(primary, GUC_CONTEXT_NORMAL); + } + if (!exec_queue_registered(q)) register_exec_queue(q, GUC_CONTEXT_NORMAL); if (!job->restore_replay) @@ -889,6 +1038,7 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) job->restore_replay = false; } +run_job_out: /* * We don't care about job-fence ordering in LR VMs because these fences * are never exported; they are used solely to keep jobs on the pending @@ -914,6 +1064,11 @@ int xe_guc_read_stopped(struct xe_guc *guc) return atomic_read(&guc->submission_state.stopped); } +static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc, + struct xe_exec_queue *q, + u32 runnable_state); +static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q); + #define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \ u32 action[] = { \ XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ @@ -927,7 +1082,9 @@ static void disable_scheduling_deregister(struct xe_guc *guc, MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); int ret; - set_min_preemption_timeout(guc, q); + if (!xe_exec_queue_is_multi_queue_secondary(q)) + set_min_preemption_timeout(guc, q); + smp_rmb(); ret = wait_event_timeout(guc->ct.wq, (!exec_queue_pending_enable(q) && @@ -955,9 +1112,12 @@ static void disable_scheduling_deregister(struct xe_guc *guc, * Reserve space for both G2H here as the 2nd G2H is sent from a G2H * handler and we are not allowed to reserved G2H space in handlers. */ - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + - G2H_LEN_DW_DEREGISTER_CONTEXT, 2); + if (xe_exec_queue_is_multi_queue_secondary(q)) + handle_multi_queue_secondary_sched_done(guc, q, 0); + else + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + + G2H_LEN_DW_DEREGISTER_CONTEXT, 2); } static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) @@ -1163,8 +1323,11 @@ static void enable_scheduling(struct xe_exec_queue *q) set_exec_queue_enabled(q); trace_xe_exec_queue_scheduling_enable(q); - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); + if (xe_exec_queue_is_multi_queue_secondary(q)) + handle_multi_queue_secondary_sched_done(guc, q, 1); + else + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); ret = wait_event_timeout(guc->ct.wq, !exec_queue_pending_enable(q) || @@ -1188,14 +1351,17 @@ static void disable_scheduling(struct xe_exec_queue *q, bool immediate) xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - if (immediate) + if (immediate && !xe_exec_queue_is_multi_queue_secondary(q)) set_min_preemption_timeout(guc, q); clear_exec_queue_enabled(q); set_exec_queue_pending_disable(q); trace_xe_exec_queue_scheduling_disable(q); - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); + if (xe_exec_queue_is_multi_queue_secondary(q)) + handle_multi_queue_secondary_sched_done(guc, q, 0); + else + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); } static void __deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) @@ -1213,8 +1379,11 @@ static void __deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) set_exec_queue_destroyed(q); trace_xe_exec_queue_deregister(q); - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), - G2H_LEN_DW_DEREGISTER_CONTEXT, 1); + if (xe_exec_queue_is_multi_queue_secondary(q)) + handle_deregister_done(guc, q); + else + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_DEREGISTER_CONTEXT, 1); } static enum drm_gpu_sched_stat @@ -1657,6 +1826,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched; struct xe_guc *guc = exec_queue_to_guc(q); + struct workqueue_struct *submit_wq = NULL; struct xe_guc_exec_queue *ge; long timeout; int err, i; @@ -1677,8 +1847,20 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies(q->sched_props.job_timeout_ms); + + /* + * Use primary queue's submit_wq for all secondary queues of a + * multi queue group. This serialization avoids any locking around + * CGP synchronization with GuC. + */ + if (xe_exec_queue_is_multi_queue_secondary(q)) { + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + + submit_wq = primary->guc->sched.base.submit_wq; + } + err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, - NULL, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64, + submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64, timeout, guc_to_gt(guc)->ordered_wq, NULL, q->name, gt_to_xe(q->gt)->drm.dev); if (err) @@ -2463,7 +2645,11 @@ static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) trace_xe_exec_queue_deregister(q); - xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); + if (xe_exec_queue_is_multi_queue_secondary(q)) + handle_deregister_done(guc, q); + else + xe_guc_ct_send_g2h_handler(&guc->ct, action, + ARRAY_SIZE(action)); } static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, @@ -2513,6 +2699,16 @@ static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, } } +static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc, + struct xe_exec_queue *q, + u32 runnable_state) +{ + /* Take CT lock here as handle_sched_done() do send a h2g message */ + mutex_lock(&guc->ct.lock); + handle_sched_done(guc, q, runnable_state); + mutex_unlock(&guc->ct.lock); +} + int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_exec_queue *q; @@ -2717,6 +2913,44 @@ int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 le return 0; } +/** + * xe_guc_exec_queue_cgp_sync_done_handler - CGP synchronization done handler + * @guc: guc + * @msg: message indicating CGP sync done + * @len: length of message + * + * Set multi queue group's sync_pending flag to false and wakeup anyone waiting + * for CGP synchronization to complete. + * + * Return: 0 on success, -EPROTO for malformed messages. + */ +int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_exec_queue *q; + u32 guc_id = msg[0]; + + if (unlikely(len < 1)) { + drm_err(&xe->drm, "Invalid CGP_SYNC_DONE length %u", len); + return -EPROTO; + } + + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) + return -EPROTO; + + if (!xe_exec_queue_is_multi_queue_primary(q)) { + drm_err(&xe->drm, "Unexpected CGP_SYNC_DONE response"); + return -EPROTO; + } + + /* Wakeup the serialized cgp update wait */ + WRITE_ONCE(q->multi_queue.group->sync_pending, false); + xe_guc_ct_wake_waiters(&guc->ct); + + return 0; +} + static void guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q, struct xe_guc_submit_exec_queue_snapshot *snapshot) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index 100a7891b918..ad8c0e8e0415 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -36,6 +36,7 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, u32 len); int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len); int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len); struct xe_guc_submit_exec_queue_snapshot * xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q); From 898a00f4b43311adfd4da1711ed2b72adc8c98a5 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:52 -0800 Subject: [PATCH 126/187] drm/xe/multi_queue: Add multi queue priority property Add support for queues of a multi queue group to set their priority within the queue group by adding property DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY. This is the only other property supported by secondary queues of a multi queue group, other than DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE. v2: Add kernel doc for enum xe_multi_queue_priority, Add assert for priority values, fix includes and declarations (Matt Brost) v3: update uapi kernel-doc (Matt Brost) v4: uapi change due to rebase Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-23-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.c | 17 +++++++++++++- drivers/gpu/drm/xe/xe_exec_queue_types.h | 16 +++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.c | 1 + drivers/gpu/drm/xe/xe_lrc.c | 29 ++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_lrc.h | 3 +++ include/uapi/drm/xe_drm.h | 4 ++++ 6 files changed, 69 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index f76ec277c5af..aa46d154d04a 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -180,6 +180,7 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, INIT_LIST_HEAD(&q->multi_gt_link); INIT_LIST_HEAD(&q->hw_engine_group_link); INIT_LIST_HEAD(&q->pxp.link); + q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL; q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; q->sched_props.preempt_timeout_us = @@ -764,6 +765,17 @@ static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue return xe_exec_queue_group_validate(xe, q, value); } +static int exec_queue_set_multi_queue_priority(struct xe_device *xe, struct xe_exec_queue *q, + u64 value) +{ + if (XE_IOCTL_DBG(xe, value > XE_MULTI_QUEUE_PRIORITY_HIGH)) + return -EINVAL; + + q->multi_queue.priority = value; + + return 0; +} + typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, struct xe_exec_queue *q, u64 value); @@ -774,6 +786,8 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type, [DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE] = exec_queue_set_hang_replay_state, [DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP] = exec_queue_set_multi_group, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY] = + exec_queue_set_multi_queue_priority, }; static int exec_queue_user_ext_set_property(struct xe_device *xe, @@ -796,7 +810,8 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE && ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE && ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE && - ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP)) + ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP && + ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY)) return -EINVAL; idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 06fb518b8533..46e5f4715a0d 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -32,6 +32,20 @@ enum xe_exec_queue_priority { XE_EXEC_QUEUE_PRIORITY_COUNT }; +/** + * enum xe_multi_queue_priority - Multi Queue priority values + * + * The priority values of the queues within the multi queue group. + */ +enum xe_multi_queue_priority { + /** @XE_MULTI_QUEUE_PRIORITY_LOW: Priority low */ + XE_MULTI_QUEUE_PRIORITY_LOW = 0, + /** @XE_MULTI_QUEUE_PRIORITY_NORMAL: Priority normal */ + XE_MULTI_QUEUE_PRIORITY_NORMAL, + /** @XE_MULTI_QUEUE_PRIORITY_HIGH: Priority high */ + XE_MULTI_QUEUE_PRIORITY_HIGH, +}; + /** * struct xe_exec_queue_group - Execution multi queue group * @@ -131,6 +145,8 @@ struct xe_exec_queue { struct { /** @multi_queue.group: Queue group information */ struct xe_exec_queue_group *group; + /** @multi_queue.priority: Queue priority within the multi-queue group */ + enum xe_multi_queue_priority priority; /** @multi_queue.pos: Position of queue within the multi-queue group */ u8 pos; /** @multi_queue.valid: Queue belongs to a multi queue group */ diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index bafe42393d22..7cca03d4296c 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -640,6 +640,7 @@ static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc, return; } + xe_lrc_set_multi_queue_priority(q->lrc[0], q->multi_queue.priority); xe_guc_exec_queue_group_cgp_update(xe, q); WRITE_ONCE(group->sync_pending, true); diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index a05060f75e7e..70eae7d03a27 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -44,6 +44,11 @@ #define LRC_INDIRECT_CTX_BO_SIZE SZ_4K #define LRC_INDIRECT_RING_STATE_SIZE SZ_4K +#define LRC_PRIORITY GENMASK_ULL(10, 9) +#define LRC_PRIORITY_LOW 0 +#define LRC_PRIORITY_NORMAL 1 +#define LRC_PRIORITY_HIGH 2 + /* * Layout of the LRC and associated data allocated as * lrc->bo: @@ -1399,6 +1404,30 @@ setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) return 0; } +static u8 xe_multi_queue_prio_to_lrc(struct xe_lrc *lrc, enum xe_multi_queue_priority priority) +{ + struct xe_device *xe = gt_to_xe(lrc->gt); + + xe_assert(xe, (priority >= XE_MULTI_QUEUE_PRIORITY_LOW && + priority <= XE_MULTI_QUEUE_PRIORITY_HIGH)); + + /* xe_multi_queue_priority is directly mapped to LRC priority values */ + return priority; +} + +/** + * xe_lrc_set_multi_queue_priority() - Set multi queue priority in LRC + * @lrc: Logical Ring Context + * @priority: Multi queue priority of the exec queue + * + * Convert @priority to LRC multi queue priority and update the @lrc descriptor + */ +void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority) +{ + lrc->desc &= ~LRC_PRIORITY; + lrc->desc |= FIELD_PREP(LRC_PRIORITY, xe_multi_queue_prio_to_lrc(lrc, priority)); +} + static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm, void *replay_state, u32 ring_size, u16 msix_vec, diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index a32472b92242..8acf85273c1a 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -13,6 +13,7 @@ struct drm_printer; struct xe_bb; struct xe_device; struct xe_exec_queue; +enum xe_multi_queue_priority; enum xe_engine_class; struct xe_gt; struct xe_hw_engine; @@ -135,6 +136,8 @@ void xe_lrc_dump_default(struct drm_printer *p, u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs); +void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority); + struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc); void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot); void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 19a8ae856a17..fd79d78de2e9 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1280,6 +1280,9 @@ struct drm_xe_vm_bind { * queue's exec_queue_id is specified in the lower 32 bits of the 'value' field. * All the other non-relevant bits of extension's 'value' field while adding the * primary or the secondary queues of the group must be set to 0. + * - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY - Set the queue + * priority within the multi-queue group. Current valid priority values are 0–2 + * (default is 1), with higher values indicating higher priority. * * The example below shows how to use @drm_xe_exec_queue_create to create * a simple exec_queue (no parallel submission) of class @@ -1323,6 +1326,7 @@ struct drm_xe_exec_queue_create { #define DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE 3 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP 4 #define DRM_XE_MULTI_GROUP_CREATE (1ull << 63) +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY 5 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; From d630abd6042dc186a3c113f68709818ad8a07a79 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:53 -0800 Subject: [PATCH 127/187] drm/xe/multi_queue: Handle invalid exec queue property setting Only MULTI_QUEUE_PRIORITY property is valid for secondary queues of a multi queue group. MULTI_QUEUE_PRIORITY only applies to multi queue group queues. Detect invalid user queue property setting and return error. Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-24-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.c | 66 ++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index aa46d154d04a..d0082eb45a4a 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -62,7 +62,7 @@ enum xe_exec_queue_sched_prop { }; static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, - u64 extensions, int ext_number); + u64 extensions); static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q) { @@ -209,7 +209,7 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, * may set q->usm, must come before xe_lrc_create(), * may overwrite q->sched_props, must come before q->ops->init() */ - err = exec_queue_user_extensions(xe, q, extensions, 0); + err = exec_queue_user_extensions(xe, q, extensions); if (err) { __xe_exec_queue_free(q); return ERR_PTR(err); @@ -790,9 +790,35 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { exec_queue_set_multi_queue_priority, }; +static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties) +{ + u64 secondary_queue_valid_props = BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP) | + BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY); + + /* + * Only MULTI_QUEUE_PRIORITY property is valid for secondary queues of a + * multi-queue group. + */ + if (xe_exec_queue_is_multi_queue_secondary(q) && + properties & ~secondary_queue_valid_props) + return -EINVAL; + + return 0; +} + +static int exec_queue_user_ext_check_final(struct xe_exec_queue *q, u64 properties) +{ + /* MULTI_QUEUE_PRIORITY only applies to multi-queue group queues */ + if ((properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY)) && + !(properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP))) + return -EINVAL; + + return 0; +} + static int exec_queue_user_ext_set_property(struct xe_device *xe, struct xe_exec_queue *q, - u64 extension) + u64 extension, u64 *properties) { u64 __user *address = u64_to_user_ptr(extension); struct drm_xe_ext_set_property ext; @@ -818,20 +844,25 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe, if (!exec_queue_set_property_funcs[idx]) return -EINVAL; + *properties |= BIT_ULL(idx); + err = exec_queue_user_ext_check(q, *properties); + if (XE_IOCTL_DBG(xe, err)) + return err; + return exec_queue_set_property_funcs[idx](xe, q, ext.value); } typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, struct xe_exec_queue *q, - u64 extension); + u64 extension, u64 *properties); static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, }; #define MAX_USER_EXTENSIONS 16 -static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, - u64 extensions, int ext_number) +static int __exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, + u64 extensions, int ext_number, u64 *properties) { u64 __user *address = u64_to_user_ptr(extensions); struct drm_xe_user_extension ext; @@ -852,13 +883,30 @@ static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue idx = array_index_nospec(ext.name, ARRAY_SIZE(exec_queue_user_extension_funcs)); - err = exec_queue_user_extension_funcs[idx](xe, q, extensions); + err = exec_queue_user_extension_funcs[idx](xe, q, extensions, properties); if (XE_IOCTL_DBG(xe, err)) return err; if (ext.next_extension) - return exec_queue_user_extensions(xe, q, ext.next_extension, - ++ext_number); + return __exec_queue_user_extensions(xe, q, ext.next_extension, + ++ext_number, properties); + + return 0; +} + +static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, + u64 extensions) +{ + u64 properties = 0; + int err; + + err = __exec_queue_user_extensions(xe, q, extensions, 0, &properties); + if (XE_IOCTL_DBG(xe, err)) + return err; + + err = exec_queue_user_ext_check_final(q, properties); + if (XE_IOCTL_DBG(xe, err)) + return err; if (xe_exec_queue_is_multi_queue_primary(q)) { err = xe_exec_queue_group_init(xe, q); From 2a31ea17d5c69e51ea454485edd40e4aeff467c1 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:54 -0800 Subject: [PATCH 128/187] drm/xe/multi_queue: Add exec_queue set_property ioctl support This patch adds support for exec_queue set_property ioctl. It is derived from the original work which is part of https://patchwork.freedesktop.org/series/112188/ Currently only DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY property can be dynamically set. v2: Check for and update kernel-doc which property this ioctl supports (Matt Brost) Signed-off-by: Matthew Brost Signed-off-by: Pallavi Mishra Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-25-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_device.c | 2 ++ drivers/gpu/drm/xe/xe_exec_queue.c | 35 ++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_exec_queue.h | 2 ++ include/uapi/drm/xe_drm.h | 26 ++++++++++++++++++++++ 4 files changed, 65 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 1197f914ef77..7a498c8db7b1 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -207,6 +207,8 @@ static const struct drm_ioctl_desc xe_ioctls[] = { DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl, + DRM_RENDER_ALLOW), }; static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index d0082eb45a4a..d738a9fea1e1 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -790,6 +790,41 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { exec_queue_set_multi_queue_priority, }; +int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_exec_queue_set_property *args = data; + struct xe_exec_queue *q; + int ret; + u32 idx; + + if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->property != + DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY)) + return -EINVAL; + + q = xe_exec_queue_lookup(xef, args->exec_queue_id); + if (XE_IOCTL_DBG(xe, !q)) + return -ENOENT; + + idx = array_index_nospec(args->property, + ARRAY_SIZE(exec_queue_set_property_funcs)); + ret = exec_queue_set_property_funcs[idx](xe, q, args->value); + if (XE_IOCTL_DBG(xe, ret)) + goto err_post_lookup; + + xe_exec_queue_put(q); + return 0; + + err_post_lookup: + xe_exec_queue_put(q); + return ret; +} + static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties) { u64 secondary_queue_valid_props = BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP) | diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index e6daa40003f2..ffcc1feb879e 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -125,6 +125,8 @@ int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); enum xe_exec_queue_priority xe_exec_queue_device_get_max_priority(struct xe_device *xe); void xe_exec_queue_last_fence_put(struct xe_exec_queue *e, struct xe_vm *vm); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index fd79d78de2e9..705081bf0d81 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -106,6 +106,7 @@ extern "C" { #define DRM_XE_OBSERVATION 0x0b #define DRM_XE_MADVISE 0x0c #define DRM_XE_VM_QUERY_MEM_RANGE_ATTRS 0x0d +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x0e /* Must be kept compact -- no holes */ @@ -123,6 +124,7 @@ extern "C" { #define DRM_IOCTL_XE_OBSERVATION DRM_IOW(DRM_COMMAND_BASE + DRM_XE_OBSERVATION, struct drm_xe_observation_param) #define DRM_IOCTL_XE_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_MADVISE, struct drm_xe_madvise) #define DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_QUERY_MEM_RANGE_ATTRS, struct drm_xe_vm_query_mem_range_attr) +#define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property) /** * DOC: Xe IOCTL Extensions @@ -2315,6 +2317,30 @@ struct drm_xe_vm_query_mem_range_attr { }; +/** + * struct drm_xe_exec_queue_set_property - exec queue set property + * + * Sets execution queue properties dynamically. + * Currently only %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY + * property can be dynamically set. + */ +struct drm_xe_exec_queue_set_property { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @exec_queue_id: Exec queue ID */ + __u32 exec_queue_id; + + /** @property: property to set */ + __u32 property; + + /** @value: property value */ + __u64 value; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + #if defined(__cplusplus) } #endif From 71e7d7e81d6a08c2abb1bb1ff01107280db62abb Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:55 -0800 Subject: [PATCH 129/187] drm/xe/multi_queue: Add support for multi queue dynamic priority change Support dynamic priority change for multi queue group queues via exec queue set_property ioctl. Issue CGP_SYNC command to GuC through the drm scheduler message interface for priority to take effect. v2: Move is_multi_queue check to exec_queue layer and assert is_multi_queue being set in guc submission layer (Matt Brost) v3: Assert CGP_SYNC message length is valid (Matt Brost) Signed-off-by: Pallavi Mishra Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-26-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.c | 11 ++++- drivers/gpu/drm/xe/xe_exec_queue_types.h | 3 ++ drivers/gpu/drm/xe/xe_guc_submit.c | 57 ++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index d738a9fea1e1..256e2ce1fe69 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -771,9 +771,16 @@ static int exec_queue_set_multi_queue_priority(struct xe_device *xe, struct xe_e if (XE_IOCTL_DBG(xe, value > XE_MULTI_QUEUE_PRIORITY_HIGH)) return -EINVAL; - q->multi_queue.priority = value; + /* For queue creation time (!q->xef) setting, just store the priority value */ + if (!q->xef) { + q->multi_queue.priority = value; + return 0; + } - return 0; + if (!xe_exec_queue_is_multi_queue(q)) + return -EINVAL; + + return q->ops->set_multi_queue_priority(q, value); } typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 46e5f4715a0d..1c285ac12868 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -260,6 +260,9 @@ struct xe_exec_queue_ops { int (*set_timeslice)(struct xe_exec_queue *q, u32 timeslice_us); /** @set_preempt_timeout: Set preemption timeout for exec queue */ int (*set_preempt_timeout)(struct xe_exec_queue *q, u32 preempt_timeout_us); + /** @set_multi_queue_priority: Set multi queue priority */ + int (*set_multi_queue_priority)(struct xe_exec_queue *q, + enum xe_multi_queue_priority priority); /** * @suspend: Suspend exec queue from executing, allowed to be called * multiple times in a row before resume with the caveat that diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 7cca03d4296c..2f467cc1929f 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -1779,10 +1779,34 @@ static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) } } -#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ -#define SET_SCHED_PROPS 2 -#define SUSPEND 3 -#define RESUME 4 +static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg) +{ + struct xe_exec_queue *q = msg->private_data; + + if (guc_exec_queue_allowed_to_change_state(q)) { +#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; + int len = 0; + + action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; + action[len++] = group->primary->guc->id; + + xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); +#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE + + xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); + } + + kfree(msg); +} + +#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ +#define SET_SCHED_PROPS 2 +#define SUSPEND 3 +#define RESUME 4 +#define SET_MULTI_QUEUE_PRIORITY 5 #define OPCODE_MASK 0xf #define MSG_LOCKED BIT(8) #define MSG_HEAD BIT(9) @@ -1806,6 +1830,9 @@ static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) case RESUME: __guc_exec_queue_process_msg_resume(msg); break; + case SET_MULTI_QUEUE_PRIORITY: + __guc_exec_queue_process_msg_set_multi_queue_priority(msg); + break; default: XE_WARN_ON("Unknown message type"); } @@ -2022,6 +2049,27 @@ static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, return 0; } +static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q, + enum xe_multi_queue_priority priority) +{ + struct xe_sched_msg *msg; + + xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q)); + + if (q->multi_queue.priority == priority || + exec_queue_killed_or_banned_or_wedged(q)) + return 0; + + msg = kmalloc(sizeof(*msg), GFP_KERNEL); + if (!msg) + return -ENOMEM; + + q->multi_queue.priority = priority; + guc_exec_queue_add_msg(q, msg, SET_MULTI_QUEUE_PRIORITY); + + return 0; +} + static int guc_exec_queue_suspend(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; @@ -2113,6 +2161,7 @@ static const struct xe_exec_queue_ops guc_exec_queue_ops = { .set_priority = guc_exec_queue_set_priority, .set_timeslice = guc_exec_queue_set_timeslice, .set_preempt_timeout = guc_exec_queue_set_preempt_timeout, + .set_multi_queue_priority = guc_exec_queue_set_multi_queue_priority, .suspend = guc_exec_queue_suspend, .suspend_wait = guc_exec_queue_suspend_wait, .resume = guc_exec_queue_resume, From 464a0bc0235f8333f77131433bb389a24efaf287 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:56 -0800 Subject: [PATCH 130/187] drm/xe/multi_queue: Add multi queue information to guc_info dump Dump multi queue specific information in the guc exec queue dump. v2: Move multi queue related fields inside the multi_queue sub-structure (Matt Brost) Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-27-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 10 ++++++++++ drivers/gpu/drm/xe/xe_guc_submit_types.h | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 2f467cc1929f..d52b7b9bcedf 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -3100,6 +3100,11 @@ xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) if (snapshot->parallel_execution) guc_exec_queue_wq_snapshot_capture(q, snapshot); + if (xe_exec_queue_is_multi_queue(q)) { + snapshot->multi_queue.valid = true; + snapshot->multi_queue.primary = xe_exec_queue_multi_queue_primary(q)->guc->id; + snapshot->multi_queue.pos = q->multi_queue.pos; + } spin_lock(&sched->base.job_list_lock); snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list); snapshot->pending_list = kmalloc_array(snapshot->pending_list_size, @@ -3182,6 +3187,11 @@ xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps if (snapshot->parallel_execution) guc_exec_queue_wq_snapshot_print(snapshot, p); + if (snapshot->multi_queue.valid) { + drm_printf(p, "\tMulti queue primary GuC ID: %d\n", snapshot->multi_queue.primary); + drm_printf(p, "\tMulti queue position: %d\n", snapshot->multi_queue.pos); + } + for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size; i++) drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n", diff --git a/drivers/gpu/drm/xe/xe_guc_submit_types.h b/drivers/gpu/drm/xe/xe_guc_submit_types.h index dc7456c34583..25e29e85502c 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit_types.h +++ b/drivers/gpu/drm/xe/xe_guc_submit_types.h @@ -135,6 +135,19 @@ struct xe_guc_submit_exec_queue_snapshot { u32 wq[WQ_SIZE / sizeof(u32)]; } parallel; + /** @multi_queue: snapshot of the multi queue information */ + struct { + /** + * @multi_queue.primary: GuC id of the primary exec queue + * of the multi queue group. + */ + u32 primary; + /** @multi_queue.pos: Position of the exec queue within the multi queue group */ + u8 pos; + /** @valid: The exec queue is part of a multi queue group */ + bool valid; + } multi_queue; + /** @pending_list_size: Size of the pending list snapshot array */ int pending_list_size; /** @pending_list: snapshot of the pending list info */ From d716a5088c88391daea7a3bd2b26589060309a79 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:57 -0800 Subject: [PATCH 131/187] drm/xe/multi_queue: Handle tearing down of a multi queue As all queues of a multi queue group use the primary queue of the group to interface with GuC. Hence there is a dependency between the queues of the group. So, when primary queue of a multi queue group is cleaned up, also trigger a cleanup of the secondary queues also. During cleanup, stop and re-start submission for all queues of a multi queue group to avoid any submission happening in parallel when a queue is being cleaned up. v2: Initialize group->list_lock, add fs_reclaim dependency, remove unwanted secondary queues cleanup (Matt Brost) v3: Properly handle cleanup of multi-queue group (Matt Brost) v4: Fix IS_ENABLED(CONFIG_LOCKDEP) check (Matt Brost) Revert stopping/restarting of submissions on queues of the group in TDR as it is not needed. Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-28-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.c | 10 +++ drivers/gpu/drm/xe/xe_exec_queue_types.h | 6 ++ drivers/gpu/drm/xe/xe_guc_submit.c | 86 ++++++++++++++++++------ 3 files changed, 82 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 256e2ce1fe69..d337b7bc2b80 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -87,6 +87,7 @@ static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q) xe_lrc_put(lrc); xa_destroy(&group->xa); + mutex_destroy(&group->list_lock); xe_bo_unpin_map_no_vm(group->cgp_bo); kfree(group); } @@ -648,9 +649,18 @@ static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue * group->primary = q; group->cgp_bo = bo; + INIT_LIST_HEAD(&group->list); xa_init_flags(&group->xa, XA_FLAGS_ALLOC1); + mutex_init(&group->list_lock); q->multi_queue.group = group; + /* group->list_lock is used in submission backend */ + if (IS_ENABLED(CONFIG_LOCKDEP)) { + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&group->list_lock); + fs_reclaim_release(GFP_KERNEL); + } + return 0; } diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 1c285ac12868..8a954ee62505 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -58,6 +58,10 @@ struct xe_exec_queue_group { struct xe_bo *cgp_bo; /** @xa: xarray to store LRCs */ struct xarray xa; + /** @list: List of all secondary queues in the group */ + struct list_head list; + /** @list_lock: Secondary queue list lock */ + struct mutex list_lock; /** @sync_pending: CGP_SYNC_DONE g2h response pending */ bool sync_pending; }; @@ -145,6 +149,8 @@ struct xe_exec_queue { struct { /** @multi_queue.group: Queue group information */ struct xe_exec_queue_group *group; + /** @multi_queue.link: Link into group's secondary queues list */ + struct list_head link; /** @multi_queue.priority: Queue priority within the multi-queue group */ enum xe_multi_queue_priority priority; /** @multi_queue.pos: Position of queue within the multi-queue group */ diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index d52b7b9bcedf..d38f5aab0a99 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -577,6 +577,45 @@ static bool vf_recovery(struct xe_guc *guc) return xe_gt_recovery_pending(guc_to_gt(guc)); } +static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + + /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */ + wake_up_all(&xe->ufence_wq); + + if (xe_exec_queue_is_lr(q)) + queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr); + else + xe_sched_tdr_queue_imm(&q->guc->sched); +} + +static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q) +{ + if (xe_exec_queue_is_multi_queue(q)) { + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_exec_queue *eq; + + set_exec_queue_reset(primary); + if (!exec_queue_banned(primary) && !exec_queue_check_timeout(primary)) + xe_guc_exec_queue_trigger_cleanup(primary); + + mutex_lock(&group->list_lock); + list_for_each_entry(eq, &group->list, multi_queue.link) { + set_exec_queue_reset(eq); + if (!exec_queue_banned(eq) && !exec_queue_check_timeout(eq)) + xe_guc_exec_queue_trigger_cleanup(eq); + } + mutex_unlock(&group->list_lock); + } else { + set_exec_queue_reset(q); + if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) + xe_guc_exec_queue_trigger_cleanup(q); + } +} + #define parallel_read(xe_, map_, field_) \ xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ field_) @@ -1121,20 +1160,6 @@ static void disable_scheduling_deregister(struct xe_guc *guc, G2H_LEN_DW_DEREGISTER_CONTEXT, 2); } -static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); - - /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */ - wake_up_all(&xe->ufence_wq); - - if (xe_exec_queue_is_lr(q)) - queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr); - else - xe_sched_tdr_queue_imm(&q->guc->sched); -} - /** * xe_guc_submit_wedge() - Wedge GuC submission * @guc: the GuC object @@ -1627,6 +1652,14 @@ static void __guc_exec_queue_destroy_async(struct work_struct *w) guard(xe_pm_runtime)(guc_to_xe(guc)); trace_xe_exec_queue_destroy(q); + if (xe_exec_queue_is_multi_queue_secondary(q)) { + struct xe_exec_queue_group *group = q->multi_queue.group; + + mutex_lock(&group->list_lock); + list_del(&q->multi_queue.link); + mutex_unlock(&group->list_lock); + } + if (xe_exec_queue_is_lr(q)) cancel_work_sync(&ge->lr_tdr); /* Confirm no work left behind accessing device structures */ @@ -1917,6 +1950,19 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) xe_exec_queue_assign_name(q, q->guc->id); + /* + * Maintain secondary queues of the multi queue group in a list + * for handling dependencies across the queues in the group. + */ + if (xe_exec_queue_is_multi_queue_secondary(q)) { + struct xe_exec_queue_group *group = q->multi_queue.group; + + INIT_LIST_HEAD(&q->multi_queue.link); + mutex_lock(&group->list_lock); + list_add_tail(&q->multi_queue.link, &group->list); + mutex_unlock(&group->list_lock); + } + trace_xe_exec_queue_create(q); return 0; @@ -2144,6 +2190,10 @@ static void guc_exec_queue_resume(struct xe_exec_queue *q) static bool guc_exec_queue_reset_status(struct xe_exec_queue *q) { + if (xe_exec_queue_is_multi_queue_secondary(q) && + guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q))) + return true; + return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q); } @@ -2853,9 +2903,7 @@ int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) * jobs by setting timeout of the job to the minimum value kicking * guc_exec_queue_timedout_job. */ - set_exec_queue_reset(q); - if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) - xe_guc_exec_queue_trigger_cleanup(q); + xe_guc_exec_queue_reset_trigger_cleanup(q); return 0; } @@ -2934,9 +2982,7 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, trace_xe_exec_queue_memory_cat_error(q); /* Treat the same as engine reset */ - set_exec_queue_reset(q); - if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) - xe_guc_exec_queue_trigger_cleanup(q); + xe_guc_exec_queue_reset_trigger_cleanup(q); return 0; } From 1b5d39e6672fdee158c3306f5cb2df8975c77e5a Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:58 -0800 Subject: [PATCH 132/187] drm/xe/multi_queue: Set QUEUE_DRAIN_MODE for Multi Queue batches To properly support soft light restore between batches being arbitrated at the CFEG, PIPE_CONTROL instructions have a new bit in the first DW, QUEUE_DRAIN_MODE. When set, this indicates to the CFEG that it should only drain the current queue. Additionally we no longer want to set the CS_STALL bit for these multi queue queues as this causes the entire pipeline to stall waiting for completion of the prior batch, preventing this soft light restore from occurring between queues in a queue group. v4: Assert !multi_queue where applicable (Matt Roper) Bspec: 56551 Signed-off-by: Stuart Summers Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matt Roper Link: https://patch.msgid.link/20251211010249.1647839-29-niranjana.vishwanathapura@intel.com --- .../gpu/drm/xe/instructions/xe_gpu_commands.h | 1 + drivers/gpu/drm/xe/xe_ring_ops.c | 64 ++++++++++++------- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h index 5d41ca297447..885fcf211e6d 100644 --- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h @@ -47,6 +47,7 @@ #define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2)) +#define PIPE_CONTROL0_QUEUE_DRAIN_MODE BIT(12) #define PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE BIT(10) /* gen12 */ #define PIPE_CONTROL0_HDC_PIPELINE_FLUSH BIT(9) /* gen12 */ diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index ac0c6dcffe15..96a14fb74507 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -12,7 +12,7 @@ #include "regs/xe_engine_regs.h" #include "regs/xe_gt_regs.h" #include "regs/xe_lrc_layout.h" -#include "xe_exec_queue_types.h" +#include "xe_exec_queue.h" #include "xe_gt.h" #include "xe_lrc.h" #include "xe_macros.h" @@ -135,12 +135,11 @@ emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, return i; } -static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw, - int i) +static int emit_pipe_invalidate(struct xe_exec_queue *q, u32 mask_flags, + bool invalidate_tlb, u32 *dw, int i) { u32 flags0 = 0; - u32 flags1 = PIPE_CONTROL_CS_STALL | - PIPE_CONTROL_COMMAND_CACHE_INVALIDATE | + u32 flags1 = PIPE_CONTROL_COMMAND_CACHE_INVALIDATE | PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE | PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | PIPE_CONTROL_VF_CACHE_INVALIDATE | @@ -152,6 +151,11 @@ static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw, if (invalidate_tlb) flags1 |= PIPE_CONTROL_TLB_INVALIDATE; + if (xe_exec_queue_is_multi_queue(q)) + flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; + else + flags1 |= PIPE_CONTROL_CS_STALL; + flags1 &= ~mask_flags; if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE) @@ -175,37 +179,47 @@ static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) { - struct xe_gt *gt = job->q->gt; + struct xe_exec_queue *q = job->q; + struct xe_gt *gt = q->gt; bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); - u32 flags; + u32 flags0, flags1; if (XE_GT_WA(gt, 14016712196)) i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH, LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); - flags = (PIPE_CONTROL_CS_STALL | - PIPE_CONTROL_TILE_CACHE_FLUSH | + flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH; + flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH | PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | PIPE_CONTROL_DEPTH_CACHE_FLUSH | PIPE_CONTROL_DC_FLUSH_ENABLE | PIPE_CONTROL_FLUSH_ENABLE); if (XE_GT_WA(gt, 1409600907)) - flags |= PIPE_CONTROL_DEPTH_STALL; + flags1 |= PIPE_CONTROL_DEPTH_STALL; if (lacks_render) - flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS; + flags1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS; else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) - flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; + flags1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; - return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0); + if (xe_exec_queue_is_multi_queue(q)) + flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; + else + flags1 |= PIPE_CONTROL_CS_STALL; + + return emit_pipe_control(dw, i, flags0, flags1, 0, 0); } -static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i) +static int emit_pipe_control_to_ring_end(struct xe_exec_queue *q, u32 *dw, int i) { + struct xe_hw_engine *hwe = q->hwe; + if (hwe->class != XE_ENGINE_CLASS_RENDER) return i; + xe_gt_assert(q->gt, !xe_exec_queue_is_multi_queue(q)); + if (XE_GT_WA(hwe->gt, 16020292621)) i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC, RING_NOPID(hwe->mmio_base).addr, 0); @@ -213,16 +227,20 @@ static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int return i; } -static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw, - int i) +static int emit_pipe_imm_ggtt(struct xe_exec_queue *q, u32 addr, u32 value, + bool stall_only, u32 *dw, int i) { - u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB | - PIPE_CONTROL_QW_WRITE; + u32 flags0 = 0, flags1 = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE; if (!stall_only) - flags |= PIPE_CONTROL_FLUSH_ENABLE; + flags1 |= PIPE_CONTROL_FLUSH_ENABLE; - return emit_pipe_control(dw, i, 0, flags, addr, value); + if (xe_exec_queue_is_multi_queue(q)) + flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; + else + flags1 |= PIPE_CONTROL_CS_STALL; + + return emit_pipe_control(dw, i, flags0, flags1, addr, value); } static u32 get_ppgtt_flag(struct xe_sched_job *job) @@ -371,7 +389,7 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS; /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */ - i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i); + i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i); /* hsdes: 1809175790 */ if (has_aux_ccs(xe)) @@ -391,11 +409,11 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, job->user_fence.value, dw, i); - i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i); + i = emit_pipe_imm_ggtt(job->q, xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i); i = emit_user_interrupt(dw, i); - i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i); + i = emit_pipe_control_to_ring_end(job->q, dw, i); xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); From c85285b32cc697d7612ee28a9ea9ded5e53d2b57 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:02:59 -0800 Subject: [PATCH 133/187] drm/xe/multi_queue: Handle CGP context error Trigger multi-queue context cleanup upon CGP context error notification from GuC. v4: Fix error message Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-30-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/abi/guc_actions_abi.h | 1 + drivers/gpu/drm/xe/xe_guc_ct.c | 4 +++ drivers/gpu/drm/xe/xe_guc_submit.c | 31 ++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.h | 2 ++ drivers/gpu/drm/xe/xe_trace.h | 5 ++++ 5 files changed, 43 insertions(+) diff --git a/drivers/gpu/drm/xe/abi/guc_actions_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_abi.h index 3e9fbed9cda6..8af3691626bf 100644 --- a/drivers/gpu/drm/xe/abi/guc_actions_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_actions_abi.h @@ -142,6 +142,7 @@ enum xe_guc_action { XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE = 0x4602, XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC = 0x4603, XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE = 0x4604, + XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR = 0x4605, XE_GUC_ACTION_CLIENT_SOFT_RESET = 0x5507, XE_GUC_ACTION_SET_ENG_UTIL_BUFF = 0x550A, XE_GUC_ACTION_SET_DEVICE_ENGINE_ACTIVITY_BUFFER = 0x550C, diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 4d5b4ed357cc..3e49e7fd0031 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -1618,6 +1618,10 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: ret = xe_guc_exec_queue_cgp_sync_done_handler(guc, payload, adj_len); break; + case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR: + ret = xe_guc_exec_queue_cgp_context_error_handler(guc, payload, + adj_len); + break; default: xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); } diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index d38f5aab0a99..3be5e78485c7 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -48,6 +48,8 @@ #include "xe_uc_fw.h" #include "xe_vm.h" +#define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN 6 + static struct xe_guc * exec_queue_to_guc(struct xe_exec_queue *q) { @@ -3009,6 +3011,35 @@ int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 le return 0; } +int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg, + u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = guc_to_xe(guc); + struct xe_exec_queue *q; + u32 guc_id = msg[2]; + + if (unlikely(len != XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN)) { + drm_err(&xe->drm, "Invalid length %u", len); + return -EPROTO; + } + + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) + return -EPROTO; + + xe_gt_dbg(gt, + "CGP context error: [%s] err=0x%x, q0_id=0x%x LRCA=0x%x guc_id=0x%x", + msg[0] & 1 ? "uc" : "kmd", msg[1], msg[2], msg[3], msg[4]); + + trace_xe_exec_queue_cgp_context_error(q); + + /* Treat the same as engine reset */ + xe_guc_exec_queue_reset_trigger_cleanup(q); + + return 0; +} + /** * xe_guc_exec_queue_cgp_sync_done_handler - CGP synchronization done handler * @guc: guc diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index ad8c0e8e0415..4d89b2975fe9 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -37,6 +37,8 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len); int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len); int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg, + u32 len); struct xe_guc_submit_exec_queue_snapshot * xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q); diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h index 79a97b086cb2..c9d0748dae9d 100644 --- a/drivers/gpu/drm/xe/xe_trace.h +++ b/drivers/gpu/drm/xe/xe_trace.h @@ -172,6 +172,11 @@ DEFINE_EVENT(xe_exec_queue, xe_exec_queue_memory_cat_error, TP_ARGS(q) ); +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_cgp_context_error, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + DEFINE_EVENT(xe_exec_queue, xe_exec_queue_stop, TP_PROTO(struct xe_exec_queue *q), TP_ARGS(q) From bb9343f122add786c57a6e8865209a9c6671bc9b Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:03:00 -0800 Subject: [PATCH 134/187] drm/xe/multi_queue: Reset GT upon CGP_SYNC failure If GuC doesn't response to CGP_SYNC message, trigger GT reset and cleanup of all the queues of the multi queue group. Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-31-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 3be5e78485c7..e8bde976e4c8 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -593,6 +593,23 @@ static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) xe_sched_tdr_queue_imm(&q->guc->sched); } +static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q) +{ + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_exec_queue *eq; + + xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), + xe_exec_queue_is_multi_queue(q)); + + xe_guc_exec_queue_trigger_cleanup(primary); + + mutex_lock(&group->list_lock); + list_for_each_entry(eq, &group->list, multi_queue.link) + xe_guc_exec_queue_trigger_cleanup(eq); + mutex_unlock(&group->list_lock); +} + static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q) { if (xe_exec_queue_is_multi_queue(q)) { @@ -618,6 +635,23 @@ static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q) } } +static void set_exec_queue_group_banned(struct xe_exec_queue *q) +{ + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_exec_queue *eq; + + /* Ban all queues of the multi-queue group */ + xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), + xe_exec_queue_is_multi_queue(q)); + set_exec_queue_banned(primary); + + mutex_lock(&group->list_lock); + list_for_each_entry(eq, &group->list, multi_queue.link) + set_exec_queue_banned(eq); + mutex_unlock(&group->list_lock); +} + #define parallel_read(xe_, map_, field_) \ xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ field_) @@ -677,7 +711,11 @@ static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc, !READ_ONCE(group->sync_pending) || xe_guc_read_stopped(guc), HZ); if (!ret || xe_guc_read_stopped(guc)) { + /* CGP_SYNC failed. Reset gt, cleanup the group */ xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n"); + set_exec_queue_group_banned(q); + xe_gt_reset_async(q->gt); + xe_guc_exec_queue_group_trigger_cleanup(q); return; } From 8b81c76885e8f61681cf4c7d6d0ce816809e3b2f Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:03:01 -0800 Subject: [PATCH 135/187] drm/xe/multi_queue: Teardown group upon job timeout Upon a job timeout, teardown the multi-queue group by triggering TDR on all queues of the multi-queue group and by skipping timeout checks in them. v5: Ban the group while triggering TDR for the guc reported errors Add FIXME in TDR to take multi-queue group off HW (Matt Brost) v6: Trigger cleanup of group only for multi-queue case Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-32-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_exec_queue_types.h | 2 ++ drivers/gpu/drm/xe/xe_guc_submit.c | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 8a954ee62505..5fc516b0bb77 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -64,6 +64,8 @@ struct xe_exec_queue_group { struct mutex list_lock; /** @sync_pending: CGP_SYNC_DONE g2h response pending */ bool sync_pending; + /** @banned: Group banned */ + bool banned; }; /** diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index e8bde976e4c8..f678b806acaa 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -602,6 +602,8 @@ static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q) xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q)); + /* Group banned, skip timeout check in TDR */ + WRITE_ONCE(group->banned, true); xe_guc_exec_queue_trigger_cleanup(primary); mutex_lock(&group->list_lock); @@ -617,6 +619,9 @@ static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q) struct xe_exec_queue_group *group = q->multi_queue.group; struct xe_exec_queue *eq; + /* Group banned, skip timeout check in TDR */ + WRITE_ONCE(group->banned, true); + set_exec_queue_reset(primary); if (!exec_queue_banned(primary) && !exec_queue_check_timeout(primary)) xe_guc_exec_queue_trigger_cleanup(primary); @@ -1487,6 +1492,19 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) exec_queue_killed_or_banned_or_wedged(q) || exec_queue_destroyed(q); + /* Skip timeout check if multi-queue group is banned */ + if (xe_exec_queue_is_multi_queue(q) && + READ_ONCE(q->multi_queue.group->banned)) + skip_timeout_check = true; + + /* + * FIXME: In multi-queue scenario, the TDR must ensure that the whole + * multi-queue group is off the HW before signaling the fences to avoid + * possible memory corruptions. This means disabling scheduling on the + * primary queue before or during the secondary queue's TDR. Need to + * implement this in least obtrusive way. + */ + /* * If devcoredump not captured and GuC capture for the job is not ready * do manual capture first and decide later if we need to use it @@ -1639,7 +1657,10 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) xe_sched_add_pending_job(sched, job); xe_sched_submission_start(sched); - xe_guc_exec_queue_trigger_cleanup(q); + if (xe_exec_queue_is_multi_queue(q)) + xe_guc_exec_queue_group_trigger_cleanup(q); + else + xe_guc_exec_queue_trigger_cleanup(q); /* Mark all outstanding jobs as bad, thus completing them */ spin_lock(&sched->base.job_list_lock); From 91abe57c27084819848c4dad6bfd1a2065b24521 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:03:02 -0800 Subject: [PATCH 136/187] drm/xe/multi_queue: Tracepoint support Add xe_exec_queue_create_multi_queue event with multi-queue information. Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-33-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 5 +++- drivers/gpu/drm/xe/xe_trace.h | 41 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index f678b806acaa..778cab377f84 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -2024,7 +2024,10 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) mutex_unlock(&group->list_lock); } - trace_xe_exec_queue_create(q); + if (xe_exec_queue_is_multi_queue(q)) + trace_xe_exec_queue_create_multi_queue(q); + else + trace_xe_exec_queue_create(q); return 0; diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h index c9d0748dae9d..6d12fcc13f43 100644 --- a/drivers/gpu/drm/xe/xe_trace.h +++ b/drivers/gpu/drm/xe/xe_trace.h @@ -13,6 +13,7 @@ #include #include "xe_exec_queue_types.h" +#include "xe_exec_queue.h" #include "xe_gpu_scheduler_types.h" #include "xe_gt_types.h" #include "xe_guc_exec_queue_types.h" @@ -97,11 +98,51 @@ DECLARE_EVENT_CLASS(xe_exec_queue, __entry->guc_state, __entry->flags) ); +DECLARE_EVENT_CLASS(xe_exec_queue_multi_queue, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q), + + TP_STRUCT__entry( + __string(dev, __dev_name_eq(q)) + __field(enum xe_engine_class, class) + __field(u32, logical_mask) + __field(u8, gt_id) + __field(u16, width) + __field(u32, guc_id) + __field(u32, guc_state) + __field(u32, flags) + __field(u32, primary) + ), + + TP_fast_assign( + __assign_str(dev); + __entry->class = q->class; + __entry->logical_mask = q->logical_mask; + __entry->gt_id = q->gt->info.id; + __entry->width = q->width; + __entry->guc_id = q->guc->id; + __entry->guc_state = atomic_read(&q->guc->state); + __entry->flags = q->flags; + __entry->primary = xe_exec_queue_multi_queue_primary(q)->guc->id; + ), + + TP_printk("dev=%s, %d:0x%x, gt=%d, width=%d guc_id=%d, guc_state=0x%x, flags=0x%x, primary=%d", + __get_str(dev), __entry->class, __entry->logical_mask, + __entry->gt_id, __entry->width, __entry->guc_id, + __entry->guc_state, __entry->flags, + __entry->primary) +); + DEFINE_EVENT(xe_exec_queue, xe_exec_queue_create, TP_PROTO(struct xe_exec_queue *q), TP_ARGS(q) ); +DEFINE_EVENT(xe_exec_queue_multi_queue, xe_exec_queue_create_multi_queue, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) +); + DEFINE_EVENT(xe_exec_queue, xe_exec_queue_supress_resume, TP_PROTO(struct xe_exec_queue *q), TP_ARGS(q) From 3131a43ecb346ae3b5287ee195779fc38c6fcd11 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:03:03 -0800 Subject: [PATCH 137/187] drm/xe/multi_queue: Support active group after primary is destroyed Add support to keep the group active after the primary queue is destroyed. Instead of killing the primary queue during exec_queue destroy ioctl, kill it when all the secondary queues of the group are killed. Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-34-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_device.c | 7 ++- drivers/gpu/drm/xe/xe_exec_queue.c | 55 +++++++++++++++++++++++- drivers/gpu/drm/xe/xe_exec_queue.h | 2 + drivers/gpu/drm/xe/xe_exec_queue_types.h | 4 ++ include/uapi/drm/xe_drm.h | 4 ++ 5 files changed, 69 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 7a498c8db7b1..24efb6a3e0ea 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -177,7 +177,12 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file) xa_for_each(&xef->exec_queue.xa, idx, q) { if (q->vm && q->hwe->hw_engine_group) xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); - xe_exec_queue_kill(q); + + if (xe_exec_queue_is_multi_queue_primary(q)) + xe_exec_queue_group_kill_put(q->multi_queue.group); + else + xe_exec_queue_kill(q); + xe_exec_queue_put(q); } xa_for_each(&xef->vm.xa, idx, vm) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index d337b7bc2b80..3f4840d135a0 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -418,6 +418,26 @@ struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, } ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); +static void xe_exec_queue_group_kill(struct kref *ref) +{ + struct xe_exec_queue_group *group = container_of(ref, struct xe_exec_queue_group, + kill_refcount); + xe_exec_queue_kill(group->primary); +} + +static inline void xe_exec_queue_group_kill_get(struct xe_exec_queue_group *group) +{ + kref_get(&group->kill_refcount); +} + +void xe_exec_queue_group_kill_put(struct xe_exec_queue_group *group) +{ + if (!group) + return; + + kref_put(&group->kill_refcount, xe_exec_queue_group_kill); +} + void xe_exec_queue_destroy(struct kref *ref) { struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); @@ -650,6 +670,7 @@ static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue * group->primary = q; group->cgp_bo = bo; INIT_LIST_HEAD(&group->list); + kref_init(&group->kill_refcount); xa_init_flags(&group->xa, XA_FLAGS_ALLOC1); mutex_init(&group->list_lock); q->multi_queue.group = group; @@ -725,6 +746,11 @@ static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q q->multi_queue.pos = pos; + if (group->primary->multi_queue.keep_active) { + xe_exec_queue_group_kill_get(group); + q->multi_queue.keep_active = true; + } + return 0; } @@ -738,6 +764,11 @@ static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queu lrc = xa_erase(&group->xa, q->multi_queue.pos); xe_assert(xe, lrc); xe_lrc_put(lrc); + + if (q->multi_queue.keep_active) { + xe_exec_queue_group_kill_put(group); + q->multi_queue.keep_active = false; + } } static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q, @@ -759,12 +790,24 @@ static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue return -EINVAL; if (value & DRM_XE_MULTI_GROUP_CREATE) { - if (XE_IOCTL_DBG(xe, value & ~DRM_XE_MULTI_GROUP_CREATE)) + if (XE_IOCTL_DBG(xe, value & ~(DRM_XE_MULTI_GROUP_CREATE | + DRM_XE_MULTI_GROUP_KEEP_ACTIVE))) + return -EINVAL; + + /* + * KEEP_ACTIVE is not supported in preempt fence mode as in that mode, + * VM_DESTROY ioctl expects all exec queues of that VM are already killed. + */ + if (XE_IOCTL_DBG(xe, (value & DRM_XE_MULTI_GROUP_KEEP_ACTIVE) && + xe_vm_in_preempt_fence_mode(q->vm))) return -EINVAL; q->multi_queue.valid = true; q->multi_queue.is_primary = true; q->multi_queue.pos = 0; + if (value & DRM_XE_MULTI_GROUP_KEEP_ACTIVE) + q->multi_queue.keep_active = true; + return 0; } @@ -1312,6 +1355,11 @@ void xe_exec_queue_kill(struct xe_exec_queue *q) q->ops->kill(q); xe_vm_remove_compute_exec_queue(q->vm, q); + + if (!xe_exec_queue_is_multi_queue_primary(q) && q->multi_queue.keep_active) { + xe_exec_queue_group_kill_put(q->multi_queue.group); + q->multi_queue.keep_active = false; + } } int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, @@ -1338,7 +1386,10 @@ int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, if (q->vm && q->hwe->hw_engine_group) xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); - xe_exec_queue_kill(q); + if (xe_exec_queue_is_multi_queue_primary(q)) + xe_exec_queue_group_kill_put(q->multi_queue.group); + else + xe_exec_queue_kill(q); trace_xe_exec_queue_close(q); xe_exec_queue_put(q); diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index ffcc1feb879e..10abed98fb6b 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -113,6 +113,8 @@ static inline struct xe_exec_queue *xe_exec_queue_multi_queue_primary(struct xe_ return xe_exec_queue_is_multi_queue(q) ? q->multi_queue.group->primary : q; } +void xe_exec_queue_group_kill_put(struct xe_exec_queue_group *group); + bool xe_exec_queue_is_lr(struct xe_exec_queue *q); bool xe_exec_queue_is_idle(struct xe_exec_queue *q); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 5fc516b0bb77..67ea5eebf70b 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -62,6 +62,8 @@ struct xe_exec_queue_group { struct list_head list; /** @list_lock: Secondary queue list lock */ struct mutex list_lock; + /** @kill_refcount: ref count to kill primary queue */ + struct kref kill_refcount; /** @sync_pending: CGP_SYNC_DONE g2h response pending */ bool sync_pending; /** @banned: Group banned */ @@ -161,6 +163,8 @@ struct xe_exec_queue { u8 valid:1; /** @multi_queue.is_primary: Is primary queue (Q0) of the group */ u8 is_primary:1; + /** @multi_queue.keep_active: Keep the group active after primary is destroyed */ + u8 keep_active:1; } multi_queue; /** @sched_props: scheduling properties */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 705081bf0d81..bd6154e3b728 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1280,6 +1280,9 @@ struct drm_xe_vm_bind { * then a new multi-queue group is created with this queue as the primary queue * (Q0). Otherwise, the queue gets added to the multi-queue group whose primary * queue's exec_queue_id is specified in the lower 32 bits of the 'value' field. + * If the extension's 'value' field has %DRM_XE_MULTI_GROUP_KEEP_ACTIVE flag + * set, then the multi-queue group is kept active after the primary queue is + * destroyed. * All the other non-relevant bits of extension's 'value' field while adding the * primary or the secondary queues of the group must be set to 0. * - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY - Set the queue @@ -1328,6 +1331,7 @@ struct drm_xe_exec_queue_create { #define DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE 3 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP 4 #define DRM_XE_MULTI_GROUP_CREATE (1ull << 63) +#define DRM_XE_MULTI_GROUP_KEEP_ACTIVE (1ull << 62) #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY 5 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; From 9a3e975d6619c6fb8997ca59361768b4ec853565 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:03:04 -0800 Subject: [PATCH 138/187] drm/xe/doc: Add documentation for Multi Queue Group Add kernel documentation for Multi Queue group and update the corresponding rst. Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-35-niranjana.vishwanathapura@intel.com --- Documentation/gpu/xe/xe_exec_queue.rst | 6 ++++ drivers/gpu/drm/xe/xe_exec_queue.c | 45 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/Documentation/gpu/xe/xe_exec_queue.rst b/Documentation/gpu/xe/xe_exec_queue.rst index 6076569e311c..732af4741df4 100644 --- a/Documentation/gpu/xe/xe_exec_queue.rst +++ b/Documentation/gpu/xe/xe_exec_queue.rst @@ -7,6 +7,12 @@ Execution Queue .. kernel-doc:: drivers/gpu/drm/xe/xe_exec_queue.c :doc: Execution Queue +Multi Queue Group +================= + +.. kernel-doc:: drivers/gpu/drm/xe/xe_exec_queue.c + :doc: Multi Queue Group + Internal API ============ diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 3f4840d135a0..e16e4d2d4053 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -54,6 +54,51 @@ * the ring operations the different engine classes support. */ +/** + * DOC: Multi Queue Group + * + * Multi Queue Group is another mode of execution supported by the compute + * and blitter copy command streamers (CCS and BCS, respectively). It is + * an enhancement of the existing hardware architecture and leverages the + * same submission model. It enables support for efficient, parallel + * execution of multiple queues within a single shared context. The multi + * queue group functionality is only supported with GuC submission backend. + * All the queues of a group must use the same address space (VM). + * + * The DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE execution queue property + * supports creating a multi queue group and adding queues to a queue group. + * + * The XE_EXEC_QUEUE_CREATE ioctl call with above property with value field + * set to DRM_XE_MULTI_GROUP_CREATE, will create a new multi queue group with + * the queue being created as the primary queue (aka q0) of the group. To add + * secondary queues to the group, they need to be created with the above + * property with id of the primary queue as the value. The properties of + * the primary queue (like priority, time slice) applies to the whole group. + * So, these properties can't be set for secondary queues of a group. + * + * The hardware does not support removing a queue from a multi-queue group. + * However, queues can be dynamically added to the group. A group can have + * up to 64 queues. To support this, XeKMD holds references to LRCs of the + * queues even after the queues are destroyed by the user until the whole + * group is destroyed. The secondary queues hold a reference to the primary + * queue thus preventing the group from being destroyed when user destroys + * the primary queue. Once the primary queue is destroyed, secondary queues + * can't be added to the queue group, but they can continue to submit the + * jobs if the DRM_XE_MULTI_GROUP_KEEP_ACTIVE flag is set during the multi + * queue group creation. + * + * The queues of a multi queue group can set their priority within the group + * through the DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY property. + * This multi queue priority can also be set dynamically through the + * XE_EXEC_QUEUE_SET_PROPERTY ioctl. This is the only other property + * supported by the secondary queues of a multi queue group, other than + * DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE. + * + * When GuC reports an error on any of the queues of a multi queue group, + * the queue cleanup mechanism is invoked for all the queues of the group + * as hardware cannot make progress on the multi queue context. + */ + enum xe_exec_queue_sched_prop { XE_EXEC_QUEUE_JOB_TIMEOUT = 0, XE_EXEC_QUEUE_TIMESLICE = 1, From 6601e0714bf08e6dfce04611796167255f63f222 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 10 Dec 2025 17:03:05 -0800 Subject: [PATCH 139/187] drm/xe/doc: Add documentation for Multi Queue Group GuC interface Add kernel documentation for Multi Queue group GuC interface. Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251211010249.1647839-36-niranjana.vishwanathapura@intel.com --- Documentation/gpu/xe/xe_exec_queue.rst | 8 ++++ drivers/gpu/drm/xe/xe_exec_queue.c | 3 ++ drivers/gpu/drm/xe/xe_guc_submit.c | 57 ++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/Documentation/gpu/xe/xe_exec_queue.rst b/Documentation/gpu/xe/xe_exec_queue.rst index 732af4741df4..8707806211c9 100644 --- a/Documentation/gpu/xe/xe_exec_queue.rst +++ b/Documentation/gpu/xe/xe_exec_queue.rst @@ -13,6 +13,14 @@ Multi Queue Group .. kernel-doc:: drivers/gpu/drm/xe/xe_exec_queue.c :doc: Multi Queue Group +.. _multi-queue-group-guc-interface: + +Multi Queue Group GuC interface +=============================== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_guc_submit.c + :doc: Multi Queue Group GuC interface + Internal API ============ diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index e16e4d2d4053..cb45962be14c 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -97,6 +97,9 @@ * When GuC reports an error on any of the queues of a multi queue group, * the queue cleanup mechanism is invoked for all the queues of the group * as hardware cannot make progress on the multi queue context. + * + * Refer :ref:`multi-queue-group-guc-interface` for multi queue group GuC + * interface. */ enum xe_exec_queue_sched_prop { diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 778cab377f84..21a8bd2ec672 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -664,6 +664,63 @@ static void set_exec_queue_group_banned(struct xe_exec_queue *q) xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ field_, val_) +/** + * DOC: Multi Queue Group GuC interface + * + * The multi queue group coordination between KMD and GuC is through a software + * construct called Context Group Page (CGP). The CGP is a KMD managed 4KB page + * allocated in the global GTT. + * + * CGP format: + * + * +-----------+---------------------------+---------------------------------------------+ + * | DWORD | Name | Description | + * +-----------+---------------------------+---------------------------------------------+ + * | 0 | Version | Bits [15:8]=Major ver, [7:0]=Minor ver | + * +-----------+---------------------------+---------------------------------------------+ + * | 1..15 | RESERVED | MBZ | + * +-----------+---------------------------+---------------------------------------------+ + * | 16 | KMD_QUEUE_UPDATE_MASK_DW0 | KMD queue mask for queues 31..0 | + * +-----------+---------------------------+---------------------------------------------+ + * | 17 | KMD_QUEUE_UPDATE_MASK_DW1 | KMD queue mask for queues 63..32 | + * +-----------+---------------------------+---------------------------------------------+ + * | 18..31 | RESERVED | MBZ | + * +-----------+---------------------------+---------------------------------------------+ + * | 32 | Q0CD_DW0 | Queue 0 context LRC descriptor lower DWORD | + * +-----------+---------------------------+---------------------------------------------+ + * | 33 | Q0ContextIndex | Context ID for Queue 0 | + * +-----------+---------------------------+---------------------------------------------+ + * | 34 | Q1CD_DW0 | Queue 1 context LRC descriptor lower DWORD | + * +-----------+---------------------------+---------------------------------------------+ + * | 35 | Q1ContextIndex | Context ID for Queue 1 | + * +-----------+---------------------------+---------------------------------------------+ + * | ... |... | ... | + * +-----------+---------------------------+---------------------------------------------+ + * | 158 | Q63CD_DW0 | Queue 63 context LRC descriptor lower DWORD | + * +-----------+---------------------------+---------------------------------------------+ + * | 159 | Q63ContextIndex | Context ID for Queue 63 | + * +-----------+---------------------------+---------------------------------------------+ + * | 160..1024 | RESERVED | MBZ | + * +-----------+---------------------------+---------------------------------------------+ + * + * While registering Q0 with GuC, CGP is updated with Q0 entry and GuC is notified + * through XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE H2G message which specifies + * the CGP address. When the secondary queues are added to the group, the CGP is + * updated with entry for that queue and GuC is notified through the H2G interface + * XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC. GuC responds to these H2G messages + * with a XE_GUC_ACTION_NOTIFY_MULTIQ_CONTEXT_CGP_SYNC_DONE G2H message. GuC also + * sends a XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR notification for any + * error in the CGP. Only one of these CGP update messages can be outstanding + * (waiting for GuC response) at any time. The bits in KMD_QUEUE_UPDATE_MASK_DW* + * fields indicate which queue entry is being updated in the CGP. + * + * The primary queue (Q0) represents the multi queue group context in GuC and + * submission on any queue of the group must be through Q0 GuC interface only. + * + * As it is not required to register secondary queues with GuC, the secondary queue + * context ids in the CGP are populated with Q0 context id. + */ + #define CGP_VERSION_MAJOR_SHIFT 8 static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe, From bd5840819aa12d1fc2831be1ceafb42237141be7 Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Fri, 28 Nov 2025 14:14:14 +0530 Subject: [PATCH 140/187] drm/xe/cri: Enable I2C controller Enable I2C controller for Crescent Island and while at it, rely on has_i2c flag instead of manual platform checks. Signed-off-by: Raag Jadav Reviewed-by: Heikki Krogerus Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20251128084414.306265-1-raag.jadav@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_device_types.h | 2 ++ drivers/gpu/drm/xe/xe_i2c.c | 2 +- drivers/gpu/drm/xe/xe_pci.c | 3 +++ drivers/gpu/drm/xe/xe_pci_types.h | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 842b33444944..ffce1dcca982 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -304,6 +304,8 @@ struct xe_device { u8 has_heci_cscfi:1; /** @info.has_heci_gscfi: device has heci gscfi */ u8 has_heci_gscfi:1; + /** @info.has_i2c: Device has I2C controller */ + u8 has_i2c:1; /** @info.has_late_bind: Device has firmware late binding support */ u8 has_late_bind:1; /** @info.has_llc: Device has a shared CPU+GPU last level cache */ diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c index 0b5452be0c87..8eccbae05705 100644 --- a/drivers/gpu/drm/xe/xe_i2c.c +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -319,7 +319,7 @@ int xe_i2c_probe(struct xe_device *xe) struct xe_i2c *i2c; int ret; - if (xe->info.platform != XE_BATTLEMAGE) + if (!xe->info.has_i2c) return 0; if (IS_SRIOV_VF(xe)) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 16b3eb247439..0a4e9d59859e 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -367,6 +367,7 @@ static const struct xe_device_desc bmg_desc = { .has_mbx_power_limits = true, .has_gsc_nvm = 1, .has_heci_cscfi = 1, + .has_i2c = true, .has_late_bind = true, .has_sriov = true, .has_mem_copy_instr = true, @@ -412,6 +413,7 @@ static const struct xe_device_desc cri_desc = { .dma_mask_size = 52, .has_display = false, .has_flat_ccs = false, + .has_i2c = true, .has_mbx_power_limits = true, .has_mert = true, .has_sriov = true, @@ -678,6 +680,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.has_gsc_nvm = desc->has_gsc_nvm; xe->info.has_heci_gscfi = desc->has_heci_gscfi; xe->info.has_heci_cscfi = desc->has_heci_cscfi; + xe->info.has_i2c = desc->has_i2c; xe->info.has_late_bind = desc->has_late_bind; xe->info.has_llc = desc->has_llc; xe->info.has_mert = desc->has_mert; diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index b06c108e25e6..bfac64d04dee 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -44,6 +44,7 @@ struct xe_device_desc { u8 has_gsc_nvm:1; u8 has_heci_gscfi:1; u8 has_heci_cscfi:1; + u8 has_i2c:1; u8 has_late_bind:1; u8 has_llc:1; u8 has_mbx_power_limits:1; From d69d3636f5f7a84bae7cd43473b3701ad9b7d544 Mon Sep 17 00:00:00 2001 From: Jagmeet Randhawa Date: Fri, 12 Dec 2025 05:21:46 +0800 Subject: [PATCH 141/187] drm/xe: Increase TDF timeout There are some corner cases where flushing transient data may take slightly longer than the 150us timeout we currently allow. Update the driver to use a 300us timeout instead based on the latest guidance from the hardware team. An update to the bspec to formally document this is expected to arrive soon. Fixes: c01c6066e6fa ("drm/xe/device: implement transient flush") Signed-off-by: Jagmeet Randhawa Reviewed-by: Jonathan Cavitt Reviewed-by: Matt Roper Link: https://patch.msgid.link/0201b1d6ec64d3651fcbff1ea21026efa915126a.1765487866.git.jagmeet.randhawa@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 24efb6a3e0ea..339b9aef9499 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -1057,7 +1057,7 @@ static void tdf_request_sync(struct xe_device *xe) * transient and need to be flushed.. */ if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, - 150, NULL, false)) + 300, NULL, false)) xe_gt_err_once(gt, "TD flush timeout\n"); } } From 825aed0328588b2837636c1c5a0c48795d724617 Mon Sep 17 00:00:00 2001 From: Jan Maslak Date: Wed, 10 Dec 2025 15:56:18 +0100 Subject: [PATCH 142/187] drm/xe: Restore engine registers before restarting schedulers after GT reset During GT reset recovery in do_gt_restart(), xe_uc_start() was called before xe_reg_sr_apply_mmio() restored engine-specific registers. This created a race window where the scheduler could run jobs before hardware state was fully restored. This caused failures in eudebug tests (xe_exec_sip_eudebug@breakpoint- waitsip-*) where TD_CTL register (containing TD_CTL_GLOBAL_DEBUG_ENABLE) wasn't restored before jobs started executing. Breakpoints would fail to trigger SIP entry because the debug enable bit wasn't set yet. Fix by moving xe_uc_start() after all MMIO register restoration, including engine registers and CCS mode configuration, ensuring all hardware state is fully restored before any jobs can be scheduled. Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Signed-off-by: Jan Maslak Reviewed-by: Jonathan Cavitt Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251210145618.169625-2-jan.maslak@intel.com --- drivers/gpu/drm/xe/xe_gt.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 7caf781ba9e8..313ce83ab0e5 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -771,9 +771,6 @@ static int do_gt_restart(struct xe_gt *gt) xe_gt_sriov_pf_init_hw(gt); xe_mocs_init(gt); - err = xe_uc_start(>->uc); - if (err) - return err; for_each_hw_engine(hwe, gt, id) xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); @@ -781,6 +778,10 @@ static int do_gt_restart(struct xe_gt *gt) /* Get CCS mode in sync between sw/hw */ xe_gt_apply_ccs_mode(gt); + err = xe_uc_start(>->uc); + if (err) + return err; + /* Restore GT freq to expected values */ xe_gt_sanitize_freq(gt); From 44ece22518594ec9ffd9ab8c4c500b522278289e Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Sat, 13 Dec 2025 05:32:27 +0800 Subject: [PATCH 143/187] drm/xe: Do not forward invalid TLB invalidation seqnos to upper layers Certain TLB invalidation operations send multiple H2G messages per seqno with only the final H2G containing the valid seqno - the others carry an invalid seqno. The G2H handler drops these invalid seqno to aovid prematurely signaling a TLB invalidation fence. With TLB_INVALIDATION_SEQNO_INVALID used to indicate in progress multi-step TLB invalidations, reset tdr to ensure that timeout won't prematurely trigger when G2H actions are still ongoing. v2: Remove lock from xe_tlb_inval_reset_timeout. (Matthew B) v3: Squash with dependent patch from Matthew Brost' series. Signed-off-by: Brian Nguyen Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251212213225.3564537-13-brian3.nguyen@intel.com --- drivers/gpu/drm/xe/xe_tlb_inval.c | 20 ++++++++++++++++++++ drivers/gpu/drm/xe/xe_tlb_inval_types.h | 1 + 2 files changed, 21 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c index 918a59e686ea..a122fbb9fc4a 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -199,6 +199,20 @@ void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval) mutex_unlock(&tlb_inval->seqno_lock); } +/** + * xe_tlb_inval_reset_timeout() - Reset TLB inval fence timeout + * @tlb_inval: TLB invalidation client + * + * Reset the TLB invalidation timeout timer. + */ +static void xe_tlb_inval_reset_timeout(struct xe_tlb_inval *tlb_inval) +{ + lockdep_assert_held(&tlb_inval->pending_lock); + + mod_delayed_work(system_wq, &tlb_inval->fence_tdr, + tlb_inval->ops->timeout_delay(tlb_inval)); +} + static bool xe_tlb_inval_seqno_past(struct xe_tlb_inval *tlb_inval, int seqno) { int seqno_recv = READ_ONCE(tlb_inval->seqno_recv); @@ -360,6 +374,12 @@ void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno) * process_g2h_msg(). */ spin_lock_irqsave(&tlb_inval->pending_lock, flags); + if (seqno == TLB_INVALIDATION_SEQNO_INVALID) { + xe_tlb_inval_reset_timeout(tlb_inval); + spin_unlock_irqrestore(&tlb_inval->pending_lock, flags); + return; + } + if (xe_tlb_inval_seqno_past(tlb_inval, seqno)) { spin_unlock_irqrestore(&tlb_inval->pending_lock, flags); return; diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h index 8f8b060e9005..7a6967ce3b76 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval_types.h +++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h @@ -80,6 +80,7 @@ struct xe_tlb_inval { const struct xe_tlb_inval_ops *ops; /** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */ #define TLB_INVALIDATION_SEQNO_MAX 0x100000 +#define TLB_INVALIDATION_SEQNO_INVALID TLB_INVALIDATION_SEQNO_MAX int seqno; /** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */ struct mutex seqno_lock; From b4abe06d6d82df6521f449357ca6b7c6ce9c0903 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sat, 13 Dec 2025 05:32:28 +0800 Subject: [PATCH 144/187] drm/xe/xe_tlb_inval: Modify fence interface to support PPC flush Allow tlb_invalidation to control when driver wants to flush the Private Physical Cache (PPC) as a process of the tlb invalidation process. Default behavior is still to always flush the PPC but driver now has the option to disable it. v2: - Revise commit/kernel doc descriptions. (Shuicheng) - Remove unused function. (Shuicheng) - Remove bool flush_cache parameter from fence, and various function inputs. (Matthew B) Signed-off-by: Brian Nguyen Reviewed-by: Matthew Brost Reviewed-by: Tejas Upadhyay Cc: Shuicheng Lin Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251212213225.3564537-14-brian3.nguyen@intel.com --- drivers/gpu/drm/xe/xe_guc_tlb_inval.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c index 848d3493df10..37ac943cb10f 100644 --- a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c @@ -34,9 +34,12 @@ static int send_tlb_inval(struct xe_guc *guc, const u32 *action, int len) G2H_LEN_DW_TLB_INVALIDATE, 1); } -#define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \ +#define MAKE_INVAL_OP_FLUSH(type, flush_cache) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \ XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \ - XE_GUC_TLB_INVAL_FLUSH_CACHE) + (flush_cache ? \ + XE_GUC_TLB_INVAL_FLUSH_CACHE : 0)) + +#define MAKE_INVAL_OP(type) MAKE_INVAL_OP_FLUSH(type, true) static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, u32 seqno) { @@ -152,7 +155,7 @@ static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, ilog2(SZ_2M) + 1))); xe_gt_assert(gt, IS_ALIGNED(start, length)); - action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE); + action[len++] = MAKE_INVAL_OP_FLUSH(XE_GUC_TLB_INVAL_PAGE_SELECTIVE, true); action[len++] = asid; action[len++] = lower_32_bits(start); action[len++] = upper_32_bits(start); From 9b1a0e0a15c97987fdf56a615f3d13995bafd042 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Sat, 13 Dec 2025 05:32:29 +0800 Subject: [PATCH 145/187] drm/xe: Add page reclamation info to device info Starting from Xe3p, HW adds a feature assisting range based page reclamation. Introduce a bit in device info to indicate whether device has such capability. Signed-off-by: Oak Zeng Signed-off-by: Brian Nguyen Reviewed-by: Shuicheng Lin Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251212213225.3564537-15-brian3.nguyen@intel.com --- drivers/gpu/drm/xe/xe_device_types.h | 2 ++ drivers/gpu/drm/xe/xe_pci.c | 1 + drivers/gpu/drm/xe/xe_pci_types.h | 1 + 3 files changed, 4 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index ffce1dcca982..4bb01c648700 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -318,6 +318,8 @@ struct xe_device { u8 has_mem_copy_instr:1; /** @info.has_mert: Device has standalone MERT */ u8 has_mert:1; + /** @info.has_page_reclaim_hw_assist: Device supports page reclamation feature */ + u8 has_page_reclaim_hw_assist:1; /** @info.has_pxp: Device has PXP support */ u8 has_pxp:1; /** @info.has_range_tlb_inval: Has range based TLB invalidations */ diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 0a4e9d59859e..7ff2eb96b841 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -684,6 +684,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.has_late_bind = desc->has_late_bind; xe->info.has_llc = desc->has_llc; xe->info.has_mert = desc->has_mert; + xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist; xe->info.has_pxp = desc->has_pxp; xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) && desc->has_sriov; diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index bfac64d04dee..602efc5c1252 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -50,6 +50,7 @@ struct xe_device_desc { u8 has_mbx_power_limits:1; u8 has_mem_copy_instr:1; u8 has_mert:1; + u8 has_page_reclaim_hw_assist:1; u8 has_pxp:1; u8 has_sriov:1; u8 needs_scratch:1; From 77ebc7c10d1607533cf7cf6c7a7b77105498d8b0 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sat, 13 Dec 2025 05:32:30 +0800 Subject: [PATCH 146/187] drm/xe/guc: Add page reclamation interface to GuC Add page reclamation related changes to GuC interface, handlers, and senders to support page reclamation. Currently TLB invalidations will perform an entire PPC flush in order to prevent stale memory access for noncoherent system memory. Page reclamation is an extension of the typical TLB invalidation workflow, allowing disabling of full PPC flush and enable selective PPC flushing. Selective flushing will be decided by a list of pages whom's address is passed to GuC at time of action. Page reclamation interfaces require at least GuC FW ver 70.31.0. v2: - Moved send_page_reclaim to first patch usage. - Add comments explaining shared done handler. (Matthew B) - Add FW version fallback to disable page reclaim on older versions. (Matthew B, Shuicheng) Signed-off-by: Brian Nguyen Reviewed-by: Matthew Brost Reviewed-by: Shuicheng Lin Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251212213225.3564537-16-brian3.nguyen@intel.com --- drivers/gpu/drm/xe/abi/guc_actions_abi.h | 2 ++ drivers/gpu/drm/xe/xe_guc.c | 4 ++++ drivers/gpu/drm/xe/xe_guc_ct.c | 17 +++++++++++++++++ drivers/gpu/drm/xe/xe_guc_fwif.h | 1 + 4 files changed, 24 insertions(+) diff --git a/drivers/gpu/drm/xe/abi/guc_actions_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_abi.h index 8af3691626bf..83a6e7794982 100644 --- a/drivers/gpu/drm/xe/abi/guc_actions_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_actions_abi.h @@ -155,6 +155,8 @@ enum xe_guc_action { XE_GUC_ACTION_TLB_INVALIDATION = 0x7000, XE_GUC_ACTION_TLB_INVALIDATION_DONE = 0x7001, XE_GUC_ACTION_TLB_INVALIDATION_ALL = 0x7002, + XE_GUC_ACTION_PAGE_RECLAMATION = 0x7003, + XE_GUC_ACTION_PAGE_RECLAMATION_DONE = 0x7004, XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION = 0x8002, XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE = 0x8003, XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED = 0x8004, diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index f0407bab9a0c..7daae3294665 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -767,6 +767,10 @@ int xe_guc_init(struct xe_guc *guc) if (!xe_uc_fw_is_enabled(&guc->fw)) return 0; + /* Disable page reclaim if GuC FW does not support */ + if (GUC_FIRMWARE_VER(guc) < MAKE_GUC_VER(70, 31, 0)) + xe->info.has_page_reclaim_hw_assist = false; + if (IS_SRIOV_VF(xe)) { ret = xe_guc_ct_init(&guc->ct); if (ret) diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 3e49e7fd0031..c3df9b3f1b4d 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -1406,6 +1406,7 @@ static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: g2h_release_space(ct, len); } @@ -1592,6 +1593,15 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) ret = xe_guc_pagefault_handler(guc, payload, adj_len); break; case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: + /* + * Page reclamation is an extension of TLB invalidation. Both + * operations share the same seqno and fence. When either + * action completes, we need to signal the corresponding + * fence. Since the handling logic (lookup fence by seqno, + * fence signalling) is identical, we use the same handler + * for both G2H events. + */ ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); break; case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: @@ -1764,6 +1774,7 @@ static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) switch (action) { case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: break; /* Process these in fast-path */ default: return 0; @@ -1800,6 +1811,12 @@ static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) ret = xe_guc_pagefault_handler(guc, payload, adj_len); break; case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: + /* + * Seqno and fence handling of page reclamation and TLB + * invalidation is identical, so we can use the same handler + * for both actions. + */ __g2h_release_space(ct, len); ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); break; diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h index e27f0088f24f..a04faec477ae 100644 --- a/drivers/gpu/drm/xe/xe_guc_fwif.h +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -17,6 +17,7 @@ #define G2H_LEN_DW_TLB_INVALIDATE 3 #define G2H_LEN_DW_G2G_NOTIFY_MIN 3 #define G2H_LEN_DW_MULTI_QUEUE_CONTEXT 3 +#define G2H_LEN_DW_PAGE_RECLAMATION 3 #define GUC_ID_MAX 65535 #define GUC_ID_UNKNOWN 0xffffffff From b912138df2993b6271500e5ecbd933acff14ac43 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sat, 13 Dec 2025 05:32:31 +0800 Subject: [PATCH 147/187] drm/xe: Create page reclaim list on unbind Page reclaim list (PRL) is preparation work for the page reclaim feature. The PRL is firstly owned by pt_update_ops and all other page reclaim operations will point back to this PRL. PRL generates its entries during the unbind page walker, updating the PRL. This PRL is restricted to a 4K page, so 512 page entries at most. v2: - Removed unused function. (Shuicheng) - Compacted warning checking, update commit message, spelling, etc. (Shuicheng, Matthew B) - Fix kernel docs - Moved PRL max entries overflow handling out from generate_reclaim_entry to caller (Shuicheng) - Add xe_page_reclaim_list_init for clarity. (Matthew B) - Modify xe_guc_page_reclaim_entry to use macros for greater flexbility. (Matthew B) - Add fallback for PTE outside of page reclaim supported 4K, 64K, 2M pages (Matthew B) - Invalidate PRL for early abort page walk. - Removed page reclaim related variables from tlb fence (Matthew Brost) - Remove error handling in *alloc_entries failure. (Matthew B) v3: - Fix NULL pointer dereference check. - Modify reclaim_entry to QW and bitfields accordingly. (Matthew B) - Add vm_dbg prints for PRL generation and invalidation. (Matthew B) v4: - s/GENMASK/GENMASK_ULL && s/BIT/BIT_ULL (CI) v5: - Addition of xe_page_reclaim_list_is_new() to avoid continuous allocation of PRL if consecutive VMAs cause a PRL invalidation. - Add xe_page_reclaim_list_valid() helpers for clarity. (Matthew B) - Move xe_page_reclaim_list_entries_put in xe_page_reclaim_list_invalidate. Signed-off-by: Brian Nguyen Reviewed-by: Matthew Brost Cc: Shuicheng Lin Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251212213225.3564537-17-brian3.nguyen@intel.com --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/regs/xe_gtt_defs.h | 1 + drivers/gpu/drm/xe/xe_page_reclaim.c | 65 ++++++++++++++ drivers/gpu/drm/xe/xe_page_reclaim.h | 96 ++++++++++++++++++++ drivers/gpu/drm/xe/xe_pt.c | 123 +++++++++++++++++++++++++- drivers/gpu/drm/xe/xe_pt_types.h | 5 ++ 6 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/xe/xe_page_reclaim.c create mode 100644 drivers/gpu/drm/xe/xe_page_reclaim.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index e5f3c2ec9e9a..8b43e391b3d3 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -95,6 +95,7 @@ xe-y += xe_bb.o \ xe_oa.o \ xe_observation.o \ xe_pagefault.o \ + xe_page_reclaim.o \ xe_pat.o \ xe_pci.o \ xe_pcode.o \ diff --git a/drivers/gpu/drm/xe/regs/xe_gtt_defs.h b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h index 4389e5a76f89..4d83461e538b 100644 --- a/drivers/gpu/drm/xe/regs/xe_gtt_defs.h +++ b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h @@ -9,6 +9,7 @@ #define XELPG_GGTT_PTE_PAT0 BIT_ULL(52) #define XELPG_GGTT_PTE_PAT1 BIT_ULL(53) +#define XE_PTE_ADDR_MASK GENMASK_ULL(51, 12) #define GGTT_PTE_VFID GENMASK_ULL(11, 2) #define GUC_GGTT_TOP 0xFEE00000 diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.c b/drivers/gpu/drm/xe/xe_page_reclaim.c new file mode 100644 index 000000000000..018d546fcf50 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_page_reclaim.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include +#include +#include + +#include "xe_page_reclaim.h" + +#include "regs/xe_gt_regs.h" +#include "xe_assert.h" +#include "xe_macros.h" + +/** + * xe_page_reclaim_list_invalidate() - Mark a PRL as invalid + * @prl: Page reclaim list to reset + * + * Clears the entries pointer and marks the list as invalid so + * future use knows PRL is unusable. It is expected that the entries + * have already been released. + */ +void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl) +{ + xe_page_reclaim_entries_put(prl->entries); + prl->entries = NULL; + prl->num_entries = XE_PAGE_RECLAIM_INVALID_LIST; +} + +/** + * xe_page_reclaim_list_init() - Initialize a page reclaim list + * @prl: Page reclaim list to initialize + * + * NULLs both values in list to prepare on initalization. + */ +void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl) +{ + // xe_page_reclaim_list_invalidate(prl); + prl->entries = NULL; + prl->num_entries = 0; +} + +/** + * xe_page_reclaim_list_alloc_entries() - Allocate page reclaim list entries + * @prl: Page reclaim list to allocate entries for + * + * Allocate one 4K page for the PRL entries, otherwise assign prl->entries to NULL. + */ +int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl) +{ + struct page *page; + + if (XE_WARN_ON(prl->entries)) + return 0; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (page) { + prl->entries = page_address(page); + prl->num_entries = 0; + } + + return page ? 0 : -ENOMEM; +} diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.h b/drivers/gpu/drm/xe/xe_page_reclaim.h new file mode 100644 index 000000000000..e776bad5189d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_page_reclaim.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PAGE_RECLAIM_H_ +#define _XE_PAGE_RECLAIM_H_ + +#include +#include +#include +#include +#include +#include + +#define XE_PAGE_RECLAIM_MAX_ENTRIES 512 +#define XE_PAGE_RECLAIM_LIST_MAX_SIZE SZ_4K + +struct xe_guc_page_reclaim_entry { + u64 qw; +/* valid reclaim entry bit */ +#define XE_PAGE_RECLAIM_VALID BIT_ULL(0) +/* + * offset order of page size to be reclaimed + * page_size = 1 << (XE_PTE_SHIFT + reclamation_size) + */ +#define XE_PAGE_RECLAIM_SIZE GENMASK_ULL(6, 1) +#define XE_PAGE_RECLAIM_RSVD_0 GENMASK_ULL(11, 7) +/* lower 20 bits of the physical address */ +#define XE_PAGE_RECLAIM_ADDR_LO GENMASK_ULL(31, 12) +/* upper 20 bits of the physical address */ +#define XE_PAGE_RECLAIM_ADDR_HI GENMASK_ULL(51, 32) +#define XE_PAGE_RECLAIM_RSVD_1 GENMASK_ULL(63, 52) +} __packed; + +struct xe_page_reclaim_list { + /** @entries: array of page reclaim entries, page allocated */ + struct xe_guc_page_reclaim_entry *entries; + /** @num_entries: number of entries */ + int num_entries; +#define XE_PAGE_RECLAIM_INVALID_LIST -1 +}; + +/** + * xe_page_reclaim_list_is_new() - Check if PRL is new allocation + * @prl: Pointer to page reclaim list + * + * PRL indicates it hasn't been allocated through both values being NULL + */ +static inline bool xe_page_reclaim_list_is_new(struct xe_page_reclaim_list *prl) +{ + return !prl->entries && prl->num_entries == 0; +} + +/** + * xe_page_reclaim_list_valid() - Check if the page reclaim list is valid + * @prl: Pointer to page reclaim list + * + * PRL uses the XE_PAGE_RECLAIM_INVALID_LIST to indicate that a PRL + * is unusable. + */ +static inline bool xe_page_reclaim_list_valid(struct xe_page_reclaim_list *prl) +{ + return !xe_page_reclaim_list_is_new(prl) && + prl->num_entries != XE_PAGE_RECLAIM_INVALID_LIST; +} + +void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl); +void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl); +int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl); +/** + * xe_page_reclaim_entries_get() - Increment the reference count of page reclaim entries. + * @entries: Pointer to the array of page reclaim entries. + * + * This function increments the reference count of the backing page. + */ +static inline void xe_page_reclaim_entries_get(struct xe_guc_page_reclaim_entry *entries) +{ + if (entries) + get_page(virt_to_page(entries)); +} + +/** + * xe_page_reclaim_entries_put() - Decrement the reference count of page reclaim entries. + * @entries: Pointer to the array of page reclaim entries. + * + * This function decrements the reference count of the backing page + * and frees it if the count reaches zero. + */ +static inline void xe_page_reclaim_entries_put(struct xe_guc_page_reclaim_entry *entries) +{ + if (entries) + put_page(virt_to_page(entries)); +} + +#endif /* _XE_PAGE_RECLAIM_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 884127b4d97d..ac17d5702030 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -12,6 +12,7 @@ #include "xe_exec_queue.h" #include "xe_gt.h" #include "xe_migrate.h" +#include "xe_page_reclaim.h" #include "xe_pt_types.h" #include "xe_pt_walk.h" #include "xe_res_cursor.h" @@ -1535,6 +1536,9 @@ struct xe_pt_stage_unbind_walk { /** @modified_end: Walk range start, modified like @modified_start. */ u64 modified_end; + /** @prl: Backing pointer to page reclaim list in pt_update_ops */ + struct xe_page_reclaim_list *prl; + /* Output */ /* @wupd: Structure to track the page-table updates we're building */ struct xe_walk_update wupd; @@ -1572,6 +1576,68 @@ static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level, return false; } +/* Huge 2MB leaf lives directly in a level-1 table and has no children */ +static bool is_2m_pte(struct xe_pt *pte) +{ + return pte->level == 1 && !pte->base.children; +} + +/* page_size = 2^(reclamation_size + XE_PTE_SHIFT) */ +#define COMPUTE_RECLAIM_ADDRESS_MASK(page_size) \ +({ \ + BUILD_BUG_ON(!__builtin_constant_p(page_size)); \ + ilog2(page_size) - XE_PTE_SHIFT; \ +}) + +static int generate_reclaim_entry(struct xe_tile *tile, + struct xe_page_reclaim_list *prl, + u64 pte, struct xe_pt *xe_child) +{ + struct xe_guc_page_reclaim_entry *reclaim_entries = prl->entries; + u64 phys_page = (pte & XE_PTE_ADDR_MASK) >> XE_PTE_SHIFT; + int num_entries = prl->num_entries; + u32 reclamation_size; + + xe_tile_assert(tile, xe_child->level <= MAX_HUGEPTE_LEVEL); + xe_tile_assert(tile, reclaim_entries); + xe_tile_assert(tile, num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1); + + if (!xe_page_reclaim_list_valid(prl)) + return -EINVAL; + + /** + * reclamation_size indicates the size of the page to be + * invalidated and flushed from non-coherent cache. + * Page size is computed as 2^(reclamation_size + XE_PTE_SHIFT) bytes. + * Only 4K, 64K (level 0), and 2M pages are supported by hardware for page reclaim + */ + if (xe_child->level == 0 && !(pte & XE_PTE_PS64)) { + reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_4K); /* reclamation_size = 0 */ + } else if (xe_child->level == 0) { + reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_64K); /* reclamation_size = 4 */ + } else if (is_2m_pte(xe_child)) { + reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_2M); /* reclamation_size = 9 */ + } else { + xe_page_reclaim_list_invalidate(prl); + vm_dbg(&tile_to_xe(tile)->drm, + "PRL invalidate: unsupported PTE level=%u pte=%#llx\n", + xe_child->level, pte); + return -EINVAL; + } + + reclaim_entries[num_entries].qw = + FIELD_PREP(XE_PAGE_RECLAIM_VALID, 1) | + FIELD_PREP(XE_PAGE_RECLAIM_SIZE, reclamation_size) | + FIELD_PREP(XE_PAGE_RECLAIM_ADDR_LO, phys_page) | + FIELD_PREP(XE_PAGE_RECLAIM_ADDR_HI, phys_page >> 20); + prl->num_entries++; + vm_dbg(&tile_to_xe(tile)->drm, + "PRL add entry: level=%u pte=%#llx reclamation_size=%u prl_idx=%d\n", + xe_child->level, pte, reclamation_size, num_entries); + + return 0; +} + static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, unsigned int level, u64 addr, u64 next, struct xe_ptw **child, @@ -1579,11 +1645,48 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, struct xe_pt_walk *walk) { struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); + struct xe_pt_stage_unbind_walk *xe_walk = + container_of(walk, typeof(*xe_walk), base); + struct xe_device *xe = tile_to_xe(xe_walk->tile); XE_WARN_ON(!*child); XE_WARN_ON(!level); + /* Check for leaf node */ + if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) && + !xe_child->base.children) { + struct iosys_map *leaf_map = &xe_child->bo->vmap; + pgoff_t first = xe_pt_offset(addr, 0, walk); + pgoff_t count = xe_pt_num_entries(addr, next, 0, walk); - xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk); + for (pgoff_t i = 0; i < count; i++) { + u64 pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64); + int ret; + + /* Account for NULL terminated entry on end (-1) */ + if (xe_walk->prl->num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1) { + ret = generate_reclaim_entry(xe_walk->tile, xe_walk->prl, + pte, xe_child); + if (ret) + break; + } else { + /* overflow, mark as invalid */ + xe_page_reclaim_list_invalidate(xe_walk->prl); + vm_dbg(&xe->drm, + "PRL invalidate: overflow while adding pte=%#llx", + pte); + break; + } + } + } + + /* If aborting page walk early, invalidate PRL since PTE may be dropped from this abort */ + if (xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk) && + xe_walk->prl && level > 1 && xe_child->base.children && xe_child->num_live != 0) { + xe_page_reclaim_list_invalidate(xe_walk->prl); + vm_dbg(&xe->drm, + "PRL invalidate: kill at level=%u addr=%#llx next=%#llx num_live=%u\n", + level, addr, next, xe_child->num_live); + } return 0; } @@ -1654,6 +1757,8 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, { u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma); u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma); + struct xe_vm_pgtable_update_op *pt_update_op = + container_of(entries, struct xe_vm_pgtable_update_op, entries[0]); struct xe_pt_stage_unbind_walk xe_walk = { .base = { .ops = &xe_pt_stage_unbind_ops, @@ -1665,6 +1770,7 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, .modified_start = start, .modified_end = end, .wupd.entries = entries, + .prl = pt_update_op->prl, }; struct xe_pt *pt = vm->pt_root[tile->id]; @@ -1897,6 +2003,7 @@ static int unbind_op_prepare(struct xe_tile *tile, struct xe_vm_pgtable_update_ops *pt_update_ops, struct xe_vma *vma) { + struct xe_device *xe = tile_to_xe(tile); u32 current_op = pt_update_ops->current_op; struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; int err; @@ -1914,6 +2021,16 @@ static int unbind_op_prepare(struct xe_tile *tile, pt_op->vma = vma; pt_op->bind = false; pt_op->rebind = false; + /* + * Maintain one PRL located in pt_update_ops that all others in unbind op reference. + * Ensure that PRL is allocated only once, and if invalidated, remains an invalidated PRL. + */ + if (xe->info.has_page_reclaim_hw_assist && + xe_page_reclaim_list_is_new(&pt_update_ops->prl)) + xe_page_reclaim_list_alloc_entries(&pt_update_ops->prl); + + /* Page reclaim may not be needed due to other features, so skip the corresponding VMA */ + pt_op->prl = (xe_page_reclaim_list_valid(&pt_update_ops->prl)) ? &pt_update_ops->prl : NULL; err = vma_reserve_fences(tile_to_xe(tile), vma); if (err) @@ -1979,6 +2096,7 @@ static int unbind_range_prepare(struct xe_vm *vm, pt_op->vma = XE_INVALID_VMA; pt_op->bind = false; pt_op->rebind = false; + pt_op->prl = NULL; pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range, pt_op->entries); @@ -2096,6 +2214,7 @@ xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) init_llist_head(&pt_update_ops->deferred); pt_update_ops->start = ~0x0ull; pt_update_ops->last = 0x0ull; + xe_page_reclaim_list_init(&pt_update_ops->prl); } /** @@ -2518,6 +2637,8 @@ void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops) &vops->pt_update_ops[tile->id]; int i; + xe_page_reclaim_entries_put(pt_update_ops->prl.entries); + lockdep_assert_held(&vops->vm->lock); xe_vm_assert_held(vops->vm); diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h index 881f01e14db8..88fabf8e2655 100644 --- a/drivers/gpu/drm/xe/xe_pt_types.h +++ b/drivers/gpu/drm/xe/xe_pt_types.h @@ -8,6 +8,7 @@ #include +#include "xe_page_reclaim.h" #include "xe_pt_walk.h" struct xe_bo; @@ -79,6 +80,8 @@ struct xe_vm_pgtable_update_op { struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1]; /** @vma: VMA for operation, operation not valid if NULL */ struct xe_vma *vma; + /** @prl: Backing pointer to page reclaim list of pt_update_ops */ + struct xe_page_reclaim_list *prl; /** @num_entries: number of entries for this update operation */ u32 num_entries; /** @bind: is a bind */ @@ -95,6 +98,8 @@ struct xe_vm_pgtable_update_ops { struct llist_head deferred; /** @q: exec queue for PT operations */ struct xe_exec_queue *q; + /** @prl: embedded page reclaim list */ + struct xe_page_reclaim_list prl; /** @start: start address of ops */ u64 start; /** @last: last address of ops */ From 2b192bebaf610abd4aa4f5fdf7282f3c2a347898 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sat, 13 Dec 2025 05:32:32 +0800 Subject: [PATCH 148/187] drm/xe: Suballocate BO for page reclaim Page reclamation feature needs the PRL to be suballocated into a GGTT-mapped BO. On allocation failure, fallback to default tlb invalidation with full PPC flush. PRL's BO allocation is managed in separate pool to ensure 4K alignment for proper GGTT address. With BO, pass into TLB invalidation backend and modify fence to accomadate accordingly. v2: - Removed page reclaim related variables from TLB fence. (Matthew B) - Allocate PRL bo size to num_entries. (Matthew B) - Move PRL bo allocation to tlb_inval run_job. (Matthew B) v5: - Use xe_page_reclaim_list_valid. (Matthew B) Signed-off-by: Brian Nguyen Suggested-by: Matthew Brost Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251212213225.3564537-18-brian3.nguyen@intel.com --- drivers/gpu/drm/xe/xe_device_types.h | 7 +++++ drivers/gpu/drm/xe/xe_page_reclaim.c | 39 +++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_page_reclaim.h | 6 +++++ drivers/gpu/drm/xe/xe_tile.c | 5 ++++ drivers/gpu/drm/xe/xe_tlb_inval_job.c | 9 +++++++ 5 files changed, 66 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 4bb01c648700..918739f85366 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -185,6 +185,13 @@ struct xe_tile { * Media GT shares a pool with its primary GT. */ struct xe_sa_manager *kernel_bb_pool; + + /** + * @mem.reclaim_pool: Pool for PRLs allocated. + * + * Only main GT has page reclaim list allocations. + */ + struct xe_sa_manager *reclaim_pool; } mem; /** @sriov: tile level virtualization data */ diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.c b/drivers/gpu/drm/xe/xe_page_reclaim.c index 018d546fcf50..0cce5ad2e33b 100644 --- a/drivers/gpu/drm/xe/xe_page_reclaim.c +++ b/drivers/gpu/drm/xe/xe_page_reclaim.c @@ -13,6 +13,45 @@ #include "regs/xe_gt_regs.h" #include "xe_assert.h" #include "xe_macros.h" +#include "xe_sa.h" +#include "xe_tlb_inval_types.h" + +/** + * xe_page_reclaim_create_prl_bo() - Back a PRL with a suballocated GGTT BO + * @tlb_inval: TLB invalidation frontend associated with the request + * @prl: page reclaim list data that bo will copy from + * @fence: tlb invalidation fence that page reclaim action is paired to + * + * Suballocates a 4K BO out of the tile reclaim pool, copies the PRL CPU + * copy into the BO and queues the buffer for release when @fence signals. + * + * Return: struct drm_suballoc pointer on success or ERR_PTR on failure. + */ +struct drm_suballoc *xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval, + struct xe_page_reclaim_list *prl, + struct xe_tlb_inval_fence *fence) +{ + struct xe_gt *gt = container_of(tlb_inval, struct xe_gt, tlb_inval); + struct xe_tile *tile = gt_to_tile(gt); + /* (+1) for NULL page_reclaim_entry to indicate end of list */ + int prl_size = min(prl->num_entries + 1, XE_PAGE_RECLAIM_MAX_ENTRIES) * + sizeof(struct xe_guc_page_reclaim_entry); + struct drm_suballoc *prl_sa; + + /* Maximum size of PRL is 1 4K-page */ + prl_sa = __xe_sa_bo_new(tile->mem.reclaim_pool, + prl_size, GFP_ATOMIC); + if (IS_ERR(prl_sa)) + return prl_sa; + + memcpy(xe_sa_bo_cpu_addr(prl_sa), prl->entries, + prl_size); + xe_sa_bo_flush_write(prl_sa); + /* Queue up sa_bo_free on tlb invalidation fence signal */ + xe_sa_bo_free(prl_sa, &fence->base); + + return prl_sa; +} /** * xe_page_reclaim_list_invalidate() - Mark a PRL as invalid diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.h b/drivers/gpu/drm/xe/xe_page_reclaim.h index e776bad5189d..ded098298d72 100644 --- a/drivers/gpu/drm/xe/xe_page_reclaim.h +++ b/drivers/gpu/drm/xe/xe_page_reclaim.h @@ -16,6 +16,9 @@ #define XE_PAGE_RECLAIM_MAX_ENTRIES 512 #define XE_PAGE_RECLAIM_LIST_MAX_SIZE SZ_4K +struct xe_tlb_inval; +struct xe_tlb_inval_fence; + struct xe_guc_page_reclaim_entry { u64 qw; /* valid reclaim entry bit */ @@ -65,6 +68,9 @@ static inline bool xe_page_reclaim_list_valid(struct xe_page_reclaim_list *prl) prl->num_entries != XE_PAGE_RECLAIM_INVALID_LIST; } +struct drm_suballoc *xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval, + struct xe_page_reclaim_list *prl, + struct xe_tlb_inval_fence *fence); void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl); void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl); int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl); diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c index 4f4f9a5c43af..63c060c2ea5c 100644 --- a/drivers/gpu/drm/xe/xe_tile.c +++ b/drivers/gpu/drm/xe/xe_tile.c @@ -209,6 +209,11 @@ int xe_tile_init(struct xe_tile *tile) if (IS_ERR(tile->mem.kernel_bb_pool)) return PTR_ERR(tile->mem.kernel_bb_pool); + /* Optimistically anticipate at most 256 TLB fences with PRL */ + tile->mem.reclaim_pool = xe_sa_bo_manager_init(tile, SZ_1M, XE_PAGE_RECLAIM_LIST_MAX_SIZE); + if (IS_ERR(tile->mem.reclaim_pool)) + return PTR_ERR(tile->mem.reclaim_pool); + return 0; } void xe_tile_migrate_wait(struct xe_tile *tile) diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.c b/drivers/gpu/drm/xe/xe_tlb_inval_job.c index 1ae0dec2cf31..78e39a4fb264 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval_job.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.c @@ -24,6 +24,8 @@ struct xe_tlb_inval_job { struct xe_exec_queue *q; /** @vm: VM which TLB invalidation is being issued for */ struct xe_vm *vm; + /** @prl: Embedded copy of page reclaim list */ + struct xe_page_reclaim_list prl; /** @refcount: ref count of this job */ struct kref refcount; /** @@ -47,6 +49,13 @@ static struct dma_fence *xe_tlb_inval_job_run(struct xe_dep_job *dep_job) container_of(dep_job, typeof(*job), dep); struct xe_tlb_inval_fence *ifence = container_of(job->fence, typeof(*ifence), base); + struct drm_suballoc *prl_sa = NULL; + + if (xe_page_reclaim_list_valid(&job->prl)) { + prl_sa = xe_page_reclaim_create_prl_bo(job->tlb_inval, &job->prl, ifence); + if (IS_ERR(prl_sa)) + prl_sa = NULL; /* Indicate fall back PPC flush with NULL */ + } xe_tlb_inval_range(job->tlb_inval, ifence, job->start, job->end, job->vm->usm.asid); From 9945e6a52f3c66b40ae30c2f4b79312a56fc9ba7 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sat, 13 Dec 2025 05:32:33 +0800 Subject: [PATCH 149/187] drm/xe: Prep page reclaim in tlb inval job Use page reclaim list as indicator if page reclaim action is desired and pass it to tlb inval fence to handle. Job will need to maintain its own embedded copy to ensure lifetime of PRL exist until job has run. v2: - Use xe variant of WARN_ON (Michal) v3: - Add comments for PRL tile handling and flush behavior with media. (Matthew Brost) Signed-off-by: Brian Nguyen Reviewed-by: Matthew Brost Cc: Michal Wajdeczko Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251212213225.3564537-19-brian3.nguyen@intel.com --- drivers/gpu/drm/xe/xe_pt.c | 11 +++++++++++ drivers/gpu/drm/xe/xe_tlb_inval_job.c | 25 +++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_tlb_inval_job.h | 4 ++++ 3 files changed, 40 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index ac17d5702030..6e01675213c7 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -2512,6 +2512,17 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) goto kill_vm_tile1; } update.ijob = ijob; + /* + * Only add page reclaim for the primary GT. Media GT does not have + * any PPC to flush, so enabling the PPC flush bit for media is + * effectively a NOP and provides no performance benefit nor + * interfere with primary GT. + */ + if (xe_page_reclaim_list_valid(&pt_update_ops->prl)) { + xe_tlb_inval_job_add_page_reclaim(ijob, &pt_update_ops->prl); + /* Release ref from alloc, job will now handle it */ + xe_page_reclaim_list_invalidate(&pt_update_ops->prl); + } if (tile->media_gt) { dep_scheduler = to_dep_scheduler(q, tile->media_gt); diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.c b/drivers/gpu/drm/xe/xe_tlb_inval_job.c index 78e39a4fb264..fc5b4a32a32d 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval_job.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.c @@ -7,7 +7,9 @@ #include "xe_dep_job_types.h" #include "xe_dep_scheduler.h" #include "xe_exec_queue.h" +#include "xe_gt_printk.h" #include "xe_gt_types.h" +#include "xe_page_reclaim.h" #include "xe_tlb_inval.h" #include "xe_tlb_inval_job.h" #include "xe_migrate.h" @@ -116,6 +118,7 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, job->start = start; job->end = end; job->fence_armed = false; + xe_page_reclaim_list_init(&job->prl); job->dep.ops = &dep_job_ops; job->type = type; kref_init(&job->refcount); @@ -149,6 +152,25 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, return ERR_PTR(err); } +/** + * xe_tlb_inval_job_add_page_reclaim() - Embed PRL into a TLB job + * @job: TLB invalidation job that may trigger reclamation + * @prl: Page reclaim list populated during unbind + * + * Copies @prl into the job and takes an extra reference to the entry page so + * ownership can transfer to the TLB fence when the job is pushed. + */ +void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job, + struct xe_page_reclaim_list *prl) +{ + struct xe_device *xe = gt_to_xe(job->q->gt); + + xe_gt_WARN_ON(job->q->gt, !xe->info.has_page_reclaim_hw_assist); + job->prl = *prl; + /* Pair with put in job_destroy */ + xe_page_reclaim_entries_get(job->prl.entries); +} + static void xe_tlb_inval_job_destroy(struct kref *ref) { struct xe_tlb_inval_job *job = container_of(ref, typeof(*job), @@ -159,6 +181,9 @@ static void xe_tlb_inval_job_destroy(struct kref *ref) struct xe_device *xe = gt_to_xe(q->gt); struct xe_vm *vm = job->vm; + /* BO creation retains a copy (if used), so no longer needed */ + xe_page_reclaim_entries_put(job->prl.entries); + if (!job->fence_armed) kfree(ifence); else diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.h b/drivers/gpu/drm/xe/xe_tlb_inval_job.h index 4d6df1a6c6ca..03d6e21cd611 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval_job.h +++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.h @@ -12,6 +12,7 @@ struct dma_fence; struct xe_dep_scheduler; struct xe_exec_queue; struct xe_migrate; +struct xe_page_reclaim_list; struct xe_tlb_inval; struct xe_tlb_inval_job; struct xe_vm; @@ -21,6 +22,9 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, struct xe_dep_scheduler *dep_scheduler, struct xe_vm *vm, u64 start, u64 end, int type); +void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job, + struct xe_page_reclaim_list *prl); + int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job); struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job, From 684965d96a918f78c3fbd3ef55444aa9cdd7c5f6 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sat, 13 Dec 2025 05:32:34 +0800 Subject: [PATCH 150/187] drm/xe: Append page reclamation action to tlb inval Add page reclamation action to tlb inval backend. The page reclamation action is paired with range tlb invalidations so both are issued at the same time. Page reclamation will issue the TLB invalidation with an invalid seqno and a H2G page reclamation action with the fence's corresponding seqno and handle the fence accordingly on page reclaim action done handler. If page reclamation fails, tlb timeout handler will be responsible for signalling fence and cleaning up. v2: - add send_page_reclaim to patch. - Remove flush_cache and use prl_sa pointer to determine PPC flush instead of explicit bool. Add NULL as fallback for others. (Matthew B) v3: - Add comments for flush_cache with media. Signed-off-by: Brian Nguyen Suggested-by: Matthew Brost Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251212213225.3564537-20-brian3.nguyen@intel.com --- drivers/gpu/drm/xe/xe_guc_tlb_inval.c | 30 ++++++++++++++++++++----- drivers/gpu/drm/xe/xe_tlb_inval.c | 7 +++--- drivers/gpu/drm/xe/xe_tlb_inval.h | 2 +- drivers/gpu/drm/xe/xe_tlb_inval_job.c | 2 +- drivers/gpu/drm/xe/xe_tlb_inval_types.h | 4 +++- drivers/gpu/drm/xe/xe_vm.c | 4 ++-- 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c index 37ac943cb10f..6532a88d51e2 100644 --- a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c @@ -13,6 +13,7 @@ #include "xe_guc_tlb_inval.h" #include "xe_force_wake.h" #include "xe_mmio.h" +#include "xe_sa.h" #include "xe_tlb_inval.h" #include "regs/xe_guc_regs.h" @@ -93,6 +94,20 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) return -ECANCELED; } +static int send_page_reclaim(struct xe_guc *guc, u32 seqno, + u64 gpu_addr) +{ + u32 action[] = { + XE_GUC_ACTION_PAGE_RECLAMATION, + seqno, + lower_32_bits(gpu_addr), + upper_32_bits(gpu_addr), + }; + + return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_PAGE_RECLAMATION, 1); +} + /* * Ensure that roundup_pow_of_two(length) doesn't overflow. * Note that roundup_pow_of_two() operates on unsigned long, @@ -101,20 +116,21 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) #define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX)) static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, - u64 start, u64 end, u32 asid) + u64 start, u64 end, u32 asid, + struct drm_suballoc *prl_sa) { #define MAX_TLB_INVALIDATION_LEN 7 struct xe_guc *guc = tlb_inval->private; struct xe_gt *gt = guc_to_gt(guc); u32 action[MAX_TLB_INVALIDATION_LEN]; u64 length = end - start; - int len = 0; + int len = 0, err; if (guc_to_xe(guc)->info.force_execlist) return -ECANCELED; action[len++] = XE_GUC_ACTION_TLB_INVALIDATION; - action[len++] = seqno; + action[len++] = !prl_sa ? seqno : TLB_INVALIDATION_SEQNO_INVALID; if (!gt_to_xe(gt)->info.has_range_tlb_inval || length > MAX_RANGE_TLB_INVALIDATION_LENGTH) { action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL); @@ -155,7 +171,8 @@ static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, ilog2(SZ_2M) + 1))); xe_gt_assert(gt, IS_ALIGNED(start, length)); - action[len++] = MAKE_INVAL_OP_FLUSH(XE_GUC_TLB_INVAL_PAGE_SELECTIVE, true); + /* Flush on NULL case, Media is not required to modify flush due to no PPC so NOP */ + action[len++] = MAKE_INVAL_OP_FLUSH(XE_GUC_TLB_INVAL_PAGE_SELECTIVE, !prl_sa); action[len++] = asid; action[len++] = lower_32_bits(start); action[len++] = upper_32_bits(start); @@ -164,7 +181,10 @@ static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN); - return send_tlb_inval(guc, action, len); + err = send_tlb_inval(guc, action, len); + if (!err && prl_sa) + err = send_page_reclaim(guc, seqno, xe_sa_bo_gpu_addr(prl_sa)); + return err; } static bool tlb_inval_initialized(struct xe_tlb_inval *tlb_inval) diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c index a122fbb9fc4a..dec042248164 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -313,6 +313,7 @@ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval) * @start: start address * @end: end address * @asid: address space id + * @prl_sa: suballocation of page reclaim list if used, NULL indicates PPC flush * * Issue a range based TLB invalidation if supported, if not fallback to a full * TLB invalidation. Completion of TLB is asynchronous and caller can use @@ -322,10 +323,10 @@ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval) */ int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence, u64 start, u64 end, - u32 asid) + u32 asid, struct drm_suballoc *prl_sa) { return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt, - start, end, asid); + start, end, asid, prl_sa); } /** @@ -341,7 +342,7 @@ void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm) u64 range = 1ull << vm->xe->info.va_bits; xe_tlb_inval_fence_init(tlb_inval, &fence, true); - xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid); + xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid, NULL); xe_tlb_inval_fence_wait(&fence); } diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.h b/drivers/gpu/drm/xe/xe_tlb_inval.h index 05614915463a..858d0690f995 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.h +++ b/drivers/gpu/drm/xe/xe_tlb_inval.h @@ -23,7 +23,7 @@ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval); void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm); int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence, - u64 start, u64 end, u32 asid); + u64 start, u64 end, u32 asid, struct drm_suballoc *prl_sa); void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval, struct xe_tlb_inval_fence *fence, diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.c b/drivers/gpu/drm/xe/xe_tlb_inval_job.c index fc5b4a32a32d..6a7bd6315797 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval_job.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.c @@ -60,7 +60,7 @@ static struct dma_fence *xe_tlb_inval_job_run(struct xe_dep_job *dep_job) } xe_tlb_inval_range(job->tlb_inval, ifence, job->start, - job->end, job->vm->usm.asid); + job->end, job->vm->usm.asid, prl_sa); return job->fence; } diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h index 7a6967ce3b76..48d1503e8460 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval_types.h +++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h @@ -9,6 +9,7 @@ #include #include +struct drm_suballoc; struct xe_tlb_inval; /** struct xe_tlb_inval_ops - TLB invalidation ops (backend) */ @@ -40,12 +41,13 @@ struct xe_tlb_inval_ops { * @start: Start address * @end: End address * @asid: Address space ID + * @prl_sa: Suballocation for page reclaim list * * Return 0 on success, -ECANCELED if backend is mid-reset, error on * failure */ int (*ppgtt)(struct xe_tlb_inval *tlb_inval, u32 seqno, u64 start, - u64 end, u32 asid); + u64 end, u32 asid, struct drm_suballoc *prl_sa); /** * @initialized: Backend is initialized diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index c2012d20faa6..bd787aae4248 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -3928,7 +3928,7 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval, &fence[fence_id], start, end, - vm->usm.asid); + vm->usm.asid, NULL); if (err) goto wait; ++fence_id; @@ -3941,7 +3941,7 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, err = xe_tlb_inval_range(&tile->media_gt->tlb_inval, &fence[fence_id], start, end, - vm->usm.asid); + vm->usm.asid, NULL); if (err) goto wait; ++fence_id; From 7c52f13b76c531ee2c503baafe52d357cab0c54a Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sat, 13 Dec 2025 05:32:35 +0800 Subject: [PATCH 151/187] drm/xe: Optimize flushing of L2$ by skipping unnecessary page reclaim There are additional hardware managed L2$ flushing such as the transient display. In those scenarios, page reclamation is unnecessary resulting in redundant cacheline flushes, so skip over those corresponding ranges. v2: - Elaborated on reasoning for page reclamation skip based on Tejas's discussion. (Matthew A, Tejas) v3: - Removed MEDIA_IS_ON due to racy condition resulting in removal of relevant registers and values. (Matthew A) - Moved l3 policy access to xe_pat. (Matthew A) v4: - Updated comments based on previous change. (Tejas) - Move back PAT index macros to xe_pat.c. Signed-off-by: Brian Nguyen Reviewed-by: Tejas Upadhyay Cc: Matthew Auld Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251212213225.3564537-21-brian3.nguyen@intel.com --- drivers/gpu/drm/xe/xe_page_reclaim.c | 32 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_page_reclaim.h | 3 +++ drivers/gpu/drm/xe/xe_pat.c | 8 +++++++ drivers/gpu/drm/xe/xe_pat.h | 10 +++++++++ drivers/gpu/drm/xe/xe_pt.c | 3 ++- 5 files changed, 55 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.c b/drivers/gpu/drm/xe/xe_page_reclaim.c index 0cce5ad2e33b..fd8c33761127 100644 --- a/drivers/gpu/drm/xe/xe_page_reclaim.c +++ b/drivers/gpu/drm/xe/xe_page_reclaim.c @@ -13,8 +13,40 @@ #include "regs/xe_gt_regs.h" #include "xe_assert.h" #include "xe_macros.h" +#include "xe_mmio.h" +#include "xe_pat.h" #include "xe_sa.h" #include "xe_tlb_inval_types.h" +#include "xe_vm.h" + +/** + * xe_page_reclaim_skip() - Decide whether PRL should be skipped for a VMA + * @tile: Tile owning the VMA + * @vma: VMA under consideration + * + * PPC flushing may be handled by HW for specific PAT encodings. + * Skip PPC flushing/Page Reclaim for scenarios below due to redundant + * flushes. + * - pat_index is transient display (1) + * + * Return: true when page reclamation is unnecessary, false otherwise. + */ +bool xe_page_reclaim_skip(struct xe_tile *tile, struct xe_vma *vma) +{ + u8 l3_policy; + + l3_policy = xe_pat_index_get_l3_policy(tile->xe, vma->attr.pat_index); + + /* + * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC + * Transient display flushes is taken care by HW, l3_policy = 1. + * + * HW will sequence these transient flushes at various sync points so + * any event of page reclamation will hit these sync points before + * page reclamation could execute. + */ + return (l3_policy == XE_L3_POLICY_XD); +} /** * xe_page_reclaim_create_prl_bo() - Back a PRL with a suballocated GGTT BO diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.h b/drivers/gpu/drm/xe/xe_page_reclaim.h index ded098298d72..a4f58e0ce9b4 100644 --- a/drivers/gpu/drm/xe/xe_page_reclaim.h +++ b/drivers/gpu/drm/xe/xe_page_reclaim.h @@ -18,6 +18,8 @@ struct xe_tlb_inval; struct xe_tlb_inval_fence; +struct xe_tile; +struct xe_vma; struct xe_guc_page_reclaim_entry { u64 qw; @@ -68,6 +70,7 @@ static inline bool xe_page_reclaim_list_valid(struct xe_page_reclaim_list *prl) prl->num_entries != XE_PAGE_RECLAIM_INVALID_LIST; } +bool xe_page_reclaim_skip(struct xe_tile *tile, struct xe_vma *vma); struct drm_suballoc *xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval, struct xe_page_reclaim_list *prl, struct xe_tlb_inval_fence *fence); diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c index 6f48d34711a6..2c3375e0250b 100644 --- a/drivers/gpu/drm/xe/xe_pat.c +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -9,6 +9,7 @@ #include +#include "regs/xe_gt_regs.h" #include "regs/xe_reg_defs.h" #include "xe_assert.h" #include "xe_device.h" @@ -231,6 +232,13 @@ bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index) return !!(xe->pat.table[pat_index].value & XE2_COMP_EN); } +u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index) +{ + WARN_ON(pat_index >= xe->pat.n_entries); + + return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value); +} + static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], int n_entries) { diff --git a/drivers/gpu/drm/xe/xe_pat.h b/drivers/gpu/drm/xe/xe_pat.h index 5749a488d9a9..d5dadfb7f924 100644 --- a/drivers/gpu/drm/xe/xe_pat.h +++ b/drivers/gpu/drm/xe/xe_pat.h @@ -69,4 +69,14 @@ u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index); */ bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index); +#define XE_L3_POLICY_WB 0 /* Write-back */ +#define XE_L3_POLICY_XD 1 /* WB - Transient Display */ +#define XE_L3_POLICY_UC 3 /* Uncached */ +/** + * xe_pat_index_get_l3_policy - Extract the L3 policy for the given pat_index. + * @xe: xe device + * @pat_index: The pat_index to query + */ +u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index); + #endif diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 6e01675213c7..6cd78bb2b652 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -2030,7 +2030,8 @@ static int unbind_op_prepare(struct xe_tile *tile, xe_page_reclaim_list_alloc_entries(&pt_update_ops->prl); /* Page reclaim may not be needed due to other features, so skip the corresponding VMA */ - pt_op->prl = (xe_page_reclaim_list_valid(&pt_update_ops->prl)) ? &pt_update_ops->prl : NULL; + pt_op->prl = (xe_page_reclaim_list_valid(&pt_update_ops->prl) && + !xe_page_reclaim_skip(tile, vma)) ? &pt_update_ops->prl : NULL; err = vma_reserve_fences(tile_to_xe(tile), vma); if (err) From 13d99b01c0c9d93afb9413cc97a05854ae40f6ab Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sat, 13 Dec 2025 05:32:36 +0800 Subject: [PATCH 152/187] drm/xe: Add debugfs support for page reclamation Allow for runtime modification to page reclamation feature through debugfs configuration. This parameter will only take effect if the platform supports the page reclamation feature by default. v2: - Minor comment tweaks. (Shuicheng) - Convert to kstrtobool_from_user. (Michal) - Only expose page reclaim file if page reclaim flag initially supported and with that, remove xe_match_desc usage. (Michal) Signed-off-by: Brian Nguyen Reviewed-by: Matthew Brost Reviewed-by: Shuicheng Lin Cc: Michal Wajdeczko Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251212213225.3564537-22-brian3.nguyen@intel.com --- drivers/gpu/drm/xe/xe_debugfs.c | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index 4fa423a82bea..ad070055cef1 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -293,6 +293,39 @@ static const struct file_operations wedged_mode_fops = { .write = wedged_mode_set, }; +static ssize_t page_reclaim_hw_assist_show(struct file *f, char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + char buf[8]; + int len; + + len = scnprintf(buf, sizeof(buf), "%d\n", xe->info.has_page_reclaim_hw_assist); + return simple_read_from_buffer(ubuf, size, pos, buf, len); +} + +static ssize_t page_reclaim_hw_assist_set(struct file *f, const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + bool val; + ssize_t ret; + + ret = kstrtobool_from_user(ubuf, size, &val); + if (ret) + return ret; + + xe->info.has_page_reclaim_hw_assist = val; + + return size; +} + +static const struct file_operations page_reclaim_hw_assist_fops = { + .owner = THIS_MODULE, + .read = page_reclaim_hw_assist_show, + .write = page_reclaim_hw_assist_set, +}; + static ssize_t atomic_svm_timeslice_ms_show(struct file *f, char __user *ubuf, size_t size, loff_t *pos) { @@ -398,6 +431,14 @@ void xe_debugfs_register(struct xe_device *xe) debugfs_create_file("disable_late_binding", 0600, root, xe, &disable_late_binding_fops); + /* + * Don't expose page reclaim configuration file if not supported by the + * hardware initially. + */ + if (xe->info.has_page_reclaim_hw_assist) + debugfs_create_file("page_reclaim_hw_assist", 0600, root, xe, + &page_reclaim_hw_assist_fops); + man = ttm_manager_type(bdev, XE_PL_TT); ttm_resource_manager_create_debugfs(man, root, "gtt_mm"); From 7ef2d25e477368bbd5c32e2265210e55644fdd48 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Fri, 12 Dec 2025 10:14:12 -0800 Subject: [PATCH 153/187] drm/xe: Track pre-production workaround support When we're initially enabling driver support for a new platform/IP, we usually implement all workarounds documented in the WA database in the driver. Many of those workarounds are restricted to early steppings that only showed up in pre-production hardware (i.e., internal test chips that are not available to the general public). Since the workarounds for early, pre-production steppings tend to be some of the ugliest and most complicated workarounds, we generally want to eliminate them and simplify the code once the platform has launched and our internal usage of those pre-production parts have been phased out. Let's add a flag to the device info that tracks which platforms still have support for pre-production workarounds for so that we can print a warning and taint if someone tries to load the driver on a pre-production part for a platform without pre-production workarounds. This will help our internal users understand the likely problems they'll encounter if they try to load the driver on an old pre-production device. The Xe behavior here is similar to what we've done for many years on i915 (see intel_detect_preproduction_hw()), except that instead of manually coding up ranges of device steppings that we believe to be pre-production hardware, Xe will use the hardware's own production vs pre-production fusing status, which we can read from the FUSE2 register. This fuse didn't exist on older Intel hardware, but should be present on all platforms supported by the Xe driver. Going forward, let's set the expectation that we'll start looking into removing pre-production workarounds for a platform around the time that platforms of the next major IP stepping are having their force_probe requirement lifted. This timing is just a rough guideline; there may be cases where some instances of pre-production parts are still being actively used in CI farms, internal device pools, etc. and we'll need to wait a bit longer for those to be swapped out. v2: - Fix inverted forcewake check v3: - Invert flag and add it to the platforms on which we still have pre-prod workarounds. (Jani, Lucas) v4: - Avoid checking pre-production on VF since they don't have access to the FUSE2 register. Bspec: 78271, 52544 Reviewed-by: Lucas De Marchi Link: https://patch.msgid.link/20251212181411.294854-3-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/regs/xe_gt_regs.h | 3 ++ drivers/gpu/drm/xe/xe_device.c | 57 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_device_types.h | 2 + drivers/gpu/drm/xe/xe_pci.c | 6 +++ drivers/gpu/drm/xe/xe_pci_types.h | 1 + 5 files changed, 69 insertions(+) diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index 917a088c28f2..93643da57428 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -227,6 +227,9 @@ #define MIRROR_FUSE1 XE_REG(0x911c) +#define FUSE2 XE_REG(0x9120) +#define PRODUCTION_HW REG_BIT(2) + #define MIRROR_L3BANK_ENABLE XE_REG(0x9130) #define XE3_L3BANK_ENABLE REG_GENMASK(31, 0) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 339b9aef9499..cdaa1c1e73f5 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -804,6 +804,61 @@ static int probe_has_flat_ccs(struct xe_device *xe) return 0; } +/* + * Detect if the driver is being run on pre-production hardware. We don't + * keep workarounds for pre-production hardware long term, so print an + * error and add taint if we're being loaded on a pre-production platform + * for which the pre-prod workarounds have already been removed. + * + * The general policy is that we'll remove any workarounds that only apply to + * pre-production hardware around the time force_probe restrictions are lifted + * for a platform of the next major IP generation (for example, Xe2 pre-prod + * workarounds should be removed around the time the first Xe3 platforms have + * force_probe lifted). + */ +static void detect_preproduction_hw(struct xe_device *xe) +{ + struct xe_gt *gt; + int id; + + /* + * SR-IOV VFs don't have access to the FUSE2 register, so we can't + * check pre-production status there. But the host OS will notice + * and report the pre-production status, which should be enough to + * help us catch mistaken use of pre-production hardware. + */ + if (IS_SRIOV_VF(xe)) + return; + + /* + * The "SW_CAP" fuse contains a bit indicating whether the device is a + * production or pre-production device. This fuse is reflected through + * the GT "FUSE2" register, even though the contents of the fuse are + * not GT-specific. Every GT's reflection of this fuse should show the + * same value, so we'll just use the first available GT for lookup. + */ + for_each_gt(gt, xe, id) + break; + + if (!gt) + return; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) { + xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n"); + return; + } + + if (xe_mmio_read32(>->mmio, FUSE2) & PRODUCTION_HW) + return; + + xe_info(xe, "Pre-production hardware detected.\n"); + if (!xe->info.has_pre_prod_wa) { + xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n"); + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); + } +} + int xe_device_probe(struct xe_device *xe) { struct xe_tile *tile; @@ -974,6 +1029,8 @@ int xe_device_probe(struct xe_device *xe) if (err) goto err_unregister_display; + detect_preproduction_hw(xe); + return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); err_unregister_display: diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 918739f85366..85700533db52 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -327,6 +327,8 @@ struct xe_device { u8 has_mert:1; /** @info.has_page_reclaim_hw_assist: Device supports page reclamation feature */ u8 has_page_reclaim_hw_assist:1; + /** @info.has_pre_prod_wa: Pre-production workarounds still present in driver */ + u8 has_pre_prod_wa:1; /** @info.has_pxp: Device has PXP support */ u8 has_pxp:1; /** @info.has_range_tlb_inval: Has range based TLB invalidations */ diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 7ff2eb96b841..179797e875b7 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -347,6 +347,7 @@ static const struct xe_device_desc lnl_desc = { .dma_mask_size = 46, .has_display = true, .has_flat_ccs = 1, + .has_pre_prod_wa = 1, .has_pxp = true, .has_mem_copy_instr = true, .max_gt_per_tile = 2, @@ -369,6 +370,7 @@ static const struct xe_device_desc bmg_desc = { .has_heci_cscfi = 1, .has_i2c = true, .has_late_bind = true, + .has_pre_prod_wa = 1, .has_sriov = true, .has_mem_copy_instr = true, .max_gt_per_tile = 2, @@ -388,6 +390,7 @@ static const struct xe_device_desc ptl_desc = { .has_flat_ccs = 1, .has_sriov = true, .has_mem_copy_instr = true, + .has_pre_prod_wa = 1, .max_gt_per_tile = 2, .needs_scratch = true, .needs_shared_vf_gt_wq = true, @@ -401,6 +404,7 @@ static const struct xe_device_desc nvls_desc = { .has_display = true, .has_flat_ccs = 1, .has_mem_copy_instr = true, + .has_pre_prod_wa = 1, .max_gt_per_tile = 2, .require_force_probe = true, .va_bits = 48, @@ -416,6 +420,7 @@ static const struct xe_device_desc cri_desc = { .has_i2c = true, .has_mbx_power_limits = true, .has_mert = true, + .has_pre_prod_wa = 1, .has_sriov = true, .max_gt_per_tile = 2, .require_force_probe = true, @@ -685,6 +690,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.has_llc = desc->has_llc; xe->info.has_mert = desc->has_mert; xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist; + xe->info.has_pre_prod_wa = desc->has_pre_prod_wa; xe->info.has_pxp = desc->has_pxp; xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) && desc->has_sriov; diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index 602efc5c1252..3bb51d155951 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -50,6 +50,7 @@ struct xe_device_desc { u8 has_mbx_power_limits:1; u8 has_mem_copy_instr:1; u8 has_mert:1; + u8 has_pre_prod_wa:1; u8 has_page_reclaim_hw_assist:1; u8 has_pxp:1; u8 has_sriov:1; From 83f4151787c5dd2f38a426cb01423149baa3bbf5 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Fri, 12 Dec 2025 10:14:13 -0800 Subject: [PATCH 154/187] drm/xe/lnl: Drop pre-production workaround support LNL has been out long enough that all of our internal usage of pre-production hardware has been phased out and we no longer need to maintain workarounds that were exclusive to pre-production parts. Production LNL hardware always has B0 or later steppings for both graphics and media IP. Eliminate all workarounds that were exclusive to A-step hardware and set the 'has_prod_wa_only' device flag for LNL to make sure we warn and taint if someone tries to load the driver on an old pre-production part. Bspec: 70821 Reviewed-by: Dnyaneshwar Bhadane Link: https://patch.msgid.link/20251212181411.294854-4-matthew.d.roper@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_guc_ads.c | 2 +- drivers/gpu/drm/xe/xe_pci.c | 1 - drivers/gpu/drm/xe/xe_ring_ops.c | 18 ------------ drivers/gpu/drm/xe/xe_wa.c | 45 ------------------------------ drivers/gpu/drm/xe/xe_wa_oob.rules | 6 +--- 5 files changed, 2 insertions(+), 70 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index bcb85a1bf26d..e06c6aa335bf 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -317,7 +317,7 @@ static void guc_waklv_init(struct xe_guc_ads *ads) offset = guc_ads_waklv_offset(ads); remain = guc_ads_waklv_size(ads); - if (XE_GT_WA(gt, 14019882105) || XE_GT_WA(gt, 16021333562)) + if (XE_GT_WA(gt, 16021333562)) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED); if (XE_GT_WA(gt, 18024947630)) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 179797e875b7..673761c8623e 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -347,7 +347,6 @@ static const struct xe_device_desc lnl_desc = { .dma_mask_size = 46, .has_display = true, .has_flat_ccs = 1, - .has_pre_prod_wa = 1, .has_pxp = true, .has_mem_copy_instr = true, .max_gt_per_tile = 2, diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index 96a14fb74507..957b9e2fd138 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -211,22 +211,6 @@ static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) return emit_pipe_control(dw, i, flags0, flags1, 0, 0); } -static int emit_pipe_control_to_ring_end(struct xe_exec_queue *q, u32 *dw, int i) -{ - struct xe_hw_engine *hwe = q->hwe; - - if (hwe->class != XE_ENGINE_CLASS_RENDER) - return i; - - xe_gt_assert(q->gt, !xe_exec_queue_is_multi_queue(q)); - - if (XE_GT_WA(hwe->gt, 16020292621)) - i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC, - RING_NOPID(hwe->mmio_base).addr, 0); - - return i; -} - static int emit_pipe_imm_ggtt(struct xe_exec_queue *q, u32 addr, u32 value, bool stall_only, u32 *dw, int i) { @@ -413,8 +397,6 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, i = emit_user_interrupt(dw, i); - i = emit_pipe_control_to_ring_end(job->q, dw, i); - xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index 914244943ac8..a93717e77da0 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -217,20 +217,6 @@ static const struct xe_rtp_entry_sr gt_was[] = { XE_RTP_ACTIONS(SET(XELPMP_SQCNT1, ENFORCE_RAR)) }, - /* Xe2_LPG */ - - { XE_RTP_NAME("16020975621"), - XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)), - XE_RTP_ACTIONS(SET(XEHP_SLICE_UNIT_LEVEL_CLKGATE, SBEUNIT_CLKGATE_DIS)) - }, - { XE_RTP_NAME("14018157293"), - XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)), - XE_RTP_ACTIONS(SET(XEHPC_L3CLOS_MASK(0), ~0), - SET(XEHPC_L3CLOS_MASK(1), ~0), - SET(XEHPC_L3CLOS_MASK(2), ~0), - SET(XEHPC_L3CLOS_MASK(3), ~0)) - }, - /* Xe2_LPM */ { XE_RTP_NAME("14017421178"), @@ -509,11 +495,6 @@ static const struct xe_rtp_entry_sr engine_was[] = { XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS)) }, - { XE_RTP_NAME("14018957109"), - XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0), - FUNC(xe_rtp_match_first_render_or_compute)), - XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN5, DISABLE_SAMPLE_G_PERFORMANCE)) - }, { XE_RTP_NAME("14020338487"), XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS)) @@ -523,11 +504,6 @@ static const struct xe_rtp_entry_sr engine_was[] = { FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH)) }, - { XE_RTP_NAME("14019322943"), - XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0), - FUNC(xe_rtp_match_first_render_or_compute)), - XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, TGM_WRITE_EOM_FORCE)) - }, { XE_RTP_NAME("14018471104"), XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL)) @@ -804,17 +780,6 @@ static const struct xe_rtp_entry_sr lrc_was[] = { /* Xe2_LPG */ - { XE_RTP_NAME("16020518922"), - XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0), - ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(FF_MODE, - DIS_TE_AUTOSTRIP | - DIS_MESH_PARTIAL_AUTOSTRIP | - DIS_MESH_AUTOSTRIP), - SET(VFLSKPD, - DIS_PARTIAL_AUTOSTRIP | - DIS_AUTOSTRIP)) - }, { XE_RTP_NAME("14019386621"), XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE)) @@ -823,20 +788,10 @@ static const struct xe_rtp_entry_sr lrc_was[] = { XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) }, - { XE_RTP_NAME("14020013138"), - XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0), - ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS)) - }, { XE_RTP_NAME("14019988906"), XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) }, - { XE_RTP_NAME("16020183090"), - XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0), - ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(INSTPM(RENDER_RING_BASE), ENABLE_SEMAPHORE_POLL_BIT)) - }, { XE_RTP_NAME("18033852989"), XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST)) diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index 7ca7258eb5d8..5cd7fa6d2a5c 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -16,15 +16,11 @@ 16017236439 PLATFORM(PVC) 14019821291 MEDIA_VERSION_RANGE(1300, 2000) 14015076503 MEDIA_VERSION(1300) -16020292621 GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0) -14018913170 GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0) - MEDIA_VERSION(2000), GRAPHICS_STEP(A0, A1) - GRAPHICS_VERSION_RANGE(1270, 1274) +14018913170 GRAPHICS_VERSION_RANGE(1270, 1274) MEDIA_VERSION(1300) PLATFORM(DG2) 14018094691 GRAPHICS_VERSION_RANGE(2001, 2002) GRAPHICS_VERSION(2004) -14019882105 GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0) 18024947630 GRAPHICS_VERSION(2001) GRAPHICS_VERSION(2004) MEDIA_VERSION(2000) From 47f5cee4197719f5d06a899a2e827968a4e85d0b Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Mon, 15 Dec 2025 18:04:33 +0100 Subject: [PATCH 155/187] drm/xe/guc: Fix version check for page-reclaim feature Page reclamation interfaces were introduced in GuC firmware version 70.31.0 (which corresponds to GuC ABI version 1.14.0), but since this feature is also available for the VFs and VFs don't know the firmware version, use GuC compatibility version check instead. Fixes: 77ebc7c10d16 ("drm/xe/guc: Add page reclamation interface to GuC") Signed-off-by: Michal Wajdeczko Cc: Brian Nguyen Cc: Matthew Brost Cc: Shuicheng Lin Reviewed-by: Brian Nguyen Reviewed-by: Daniele Ceraolo Spurio Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251215170433.196398-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_guc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index 7daae3294665..09ac092c3687 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -768,7 +768,7 @@ int xe_guc_init(struct xe_guc *guc) return 0; /* Disable page reclaim if GuC FW does not support */ - if (GUC_FIRMWARE_VER(guc) < MAKE_GUC_VER(70, 31, 0)) + if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 14, 0)) xe->info.has_page_reclaim_hw_assist = false; if (IS_SRIOV_VF(xe)) { From b07bac9bd708ec468cd1b8a5fe70ae2ac9b0a11c Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Fri, 5 Dec 2025 23:47:17 +0000 Subject: [PATCH 156/187] drm/xe: Limit num_syncs to prevent oversized allocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exec and vm_bind ioctl allow userspace to specify an arbitrary num_syncs value. Without bounds checking, a very large num_syncs can force an excessively large allocation, leading to kernel warnings from the page allocator as below. Introduce DRM_XE_MAX_SYNCS (set to 1024) and reject any request exceeding this limit. " ------------[ cut here ]------------ WARNING: CPU: 0 PID: 1217 at mm/page_alloc.c:5124 __alloc_frozen_pages_noprof+0x2f8/0x2180 mm/page_alloc.c:5124 ... Call Trace: alloc_pages_mpol+0xe4/0x330 mm/mempolicy.c:2416 ___kmalloc_large_node+0xd8/0x110 mm/slub.c:4317 __kmalloc_large_node_noprof+0x18/0xe0 mm/slub.c:4348 __do_kmalloc_node mm/slub.c:4364 [inline] __kmalloc_noprof+0x3d4/0x4b0 mm/slub.c:4388 kmalloc_noprof include/linux/slab.h:909 [inline] kmalloc_array_noprof include/linux/slab.h:948 [inline] xe_exec_ioctl+0xa47/0x1e70 drivers/gpu/drm/xe/xe_exec.c:158 drm_ioctl_kernel+0x1f1/0x3e0 drivers/gpu/drm/drm_ioctl.c:797 drm_ioctl+0x5e7/0xc50 drivers/gpu/drm/drm_ioctl.c:894 xe_drm_ioctl+0x10b/0x170 drivers/gpu/drm/xe/xe_device.c:224 vfs_ioctl fs/ioctl.c:51 [inline] __do_sys_ioctl fs/ioctl.c:598 [inline] __se_sys_ioctl fs/ioctl.c:584 [inline] __x64_sys_ioctl+0x18b/0x210 fs/ioctl.c:584 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0xbb/0x380 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x77/0x7f ... " v2: Add "Reported-by" and Cc stable kernels. v3: Change XE_MAX_SYNCS from 64 to 1024. (Matt & Ashutosh) v4: s/XE_MAX_SYNCS/DRM_XE_MAX_SYNCS/ (Matt) v5: Do the check at the top of the exec func. (Matt) Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Reported-by: Koen Koning Reported-by: Peter Senna Tschudin Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/6450 Cc: # v6.12+ Cc: Matthew Brost Cc: Michal Mrozek Cc: Carl Zhang Cc: José Roberto de Souza Cc: Lionel Landwerlin Cc: Ivan Briano Cc: Thomas Hellström Cc: Ashutosh Dixit Signed-off-by: Shuicheng Lin Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251205234715.2476561-5-shuicheng.lin@intel.com --- drivers/gpu/drm/xe/xe_exec.c | 3 ++- drivers/gpu/drm/xe/xe_vm.c | 3 +++ include/uapi/drm/xe_drm.h | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c index 4d81210e41f5..fd9480031750 100644 --- a/drivers/gpu/drm/xe/xe_exec.c +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -132,7 +132,8 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) if (XE_IOCTL_DBG(xe, args->extensions) || XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || - XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]) || + XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS)) return -EINVAL; q = xe_exec_queue_lookup(xef, args->exec_queue_id); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index bd787aae4248..ca546666a5c9 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -3341,6 +3341,9 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, if (XE_IOCTL_DBG(xe, args->extensions)) return -EINVAL; + if (XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS)) + return -EINVAL; + if (args->num_binds > 1) { u64 __user *bind_user = u64_to_user_ptr(args->vector_of_binds); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index bd6154e3b728..c59587529986 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1504,6 +1504,7 @@ struct drm_xe_exec { /** @exec_queue_id: Exec queue ID for the batch buffer */ __u32 exec_queue_id; +#define DRM_XE_MAX_SYNCS 1024 /** @num_syncs: Amount of struct drm_xe_sync in array. */ __u32 num_syncs; From e057b2d2b8d815df3858a87dffafa2af37e5945b Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Fri, 5 Dec 2025 23:47:18 +0000 Subject: [PATCH 157/187] drm/xe/oa: Limit num_syncs to prevent oversized allocations The OA open parameters did not validate num_syncs, allowing userspace to pass arbitrarily large values, potentially leading to excessive allocations. Add check to ensure that num_syncs does not exceed DRM_XE_MAX_SYNCS, returning -EINVAL when the limit is violated. v2: use XE_IOCTL_DBG() and drop duplicated check. (Ashutosh) Fixes: c8507a25cebd ("drm/xe/oa/uapi: Define and parse OA sync properties") Cc: Matthew Brost Cc: Ashutosh Dixit Signed-off-by: Shuicheng Lin Reviewed-by: Ashutosh Dixit Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251205234715.2476561-6-shuicheng.lin@intel.com --- drivers/gpu/drm/xe/xe_oa.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index cc48663c2b48..92aa25fc0422 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1254,6 +1254,9 @@ static int xe_oa_set_no_preempt(struct xe_oa *oa, u64 value, static int xe_oa_set_prop_num_syncs(struct xe_oa *oa, u64 value, struct xe_oa_open_param *param) { + if (XE_IOCTL_DBG(oa->xe, value > DRM_XE_MAX_SYNCS)) + return -EINVAL; + param->num_syncs = value; return 0; } From 33a5abd9a68394aa67f9618b20eee65ee8702ff4 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 12 Dec 2025 10:28:41 -0800 Subject: [PATCH 158/187] drm/xe: Adjust long-running workload timeslices to reasonable values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 10ms timeslice for long-running workloads is far too long and causes significant jitter in benchmarks when the system is shared. Adjust the value to 5ms for preempt-fencing VMs, as the resume step there is quite costly as memory is moved around, and set it to zero for pagefault VMs, since switching back to pagefault mode after dma-fence mode is relatively fast. Also change min_run_period_ms to 'unsiged int' type rather than 's64' as only positive values make sense. Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Cc: stable@vger.kernel.org Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20251212182847.1683222-2-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 5 ++++- drivers/gpu/drm/xe/xe_vm_types.h | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index ca546666a5c9..68dffbf97680 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1508,7 +1508,10 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) INIT_WORK(&vm->destroy_work, vm_destroy_work_func); INIT_LIST_HEAD(&vm->preempt.exec_queues); - vm->preempt.min_run_period_ms = 10; /* FIXME: Wire up to uAPI */ + if (flags & XE_VM_FLAG_FAULT_MODE) + vm->preempt.min_run_period_ms = 0; + else + vm->preempt.min_run_period_ms = 5; for_each_tile(tile, xe, id) xe_range_fence_tree_init(&vm->rftree[id]); diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 3bf912bfbdcc..18bad1dd08e6 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -263,7 +263,7 @@ struct xe_vm { * @min_run_period_ms: The minimum run period before preempting * an engine again */ - s64 min_run_period_ms; + unsigned int min_run_period_ms; /** @exec_queues: list of exec queues attached to this VM */ struct list_head exec_queues; /** @num_exec_queues: number exec queues attached to this VM */ From ca415c4d4c17ad676a2c8981e1fcc432221dce79 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 12 Dec 2025 10:28:42 -0800 Subject: [PATCH 159/187] drm/xe: Use usleep_range for accurate long-running workload timeslicing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit msleep is not very accurate in terms of how long it actually sleeps, whereas usleep_range is precise. Replace the timeslice sleep for long-running workloads with the more accurate usleep_range to avoid jitter if the sleep period is less than 20ms. Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Cc: stable@vger.kernel.org Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20251212182847.1683222-3-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 21a8bd2ec672..18cac5594d6a 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -990,6 +990,24 @@ static u32 wq_space_until_wrap(struct xe_exec_queue *q) return (WQ_SIZE - q->guc->wqi_tail); } +static inline void relaxed_ms_sleep(unsigned int delay_ms) +{ + unsigned long min_us, max_us; + + if (!delay_ms) + return; + + if (delay_ms > 20) { + msleep(delay_ms); + return; + } + + min_us = mul_u32_u32(delay_ms, 1000); + max_us = min_us + 500; + + usleep_range(min_us, max_us); +} + static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) { struct xe_guc *guc = exec_queue_to_guc(q); @@ -1903,7 +1921,7 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) since_resume_ms; if (wait_ms > 0 && q->guc->resume_time) - msleep(wait_ms); + relaxed_ms_sleep(wait_ms); set_exec_queue_suspended(q); disable_scheduling(q, false); From 6e608bff259fd1eae5d381c5eb16b88413e16209 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 12 Dec 2025 10:28:43 -0800 Subject: [PATCH 160/187] drm/xe: Add debugfs knobs to control long running workload timeslicing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add debugfs knobs to control timeslicing for long-running workloads, allowing quick tuning of values when running benchmarks. Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20251212182847.1683222-4-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_debugfs.c | 74 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_device.c | 1 + drivers/gpu/drm/xe/xe_device_types.h | 6 +++ drivers/gpu/drm/xe/xe_vm.c | 4 +- 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index ad070055cef1..0907868b32d6 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -361,6 +361,74 @@ static const struct file_operations atomic_svm_timeslice_ms_fops = { .write = atomic_svm_timeslice_ms_set, }; +static ssize_t min_run_period_lr_ms_show(struct file *f, char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + char buf[32]; + int len = 0; + + len = scnprintf(buf, sizeof(buf), "%d\n", xe->min_run_period_lr_ms); + + return simple_read_from_buffer(ubuf, size, pos, buf, len); +} + +static ssize_t min_run_period_lr_ms_set(struct file *f, const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + u32 min_run_period_lr_ms; + ssize_t ret; + + ret = kstrtouint_from_user(ubuf, size, 0, &min_run_period_lr_ms); + if (ret) + return ret; + + xe->min_run_period_lr_ms = min_run_period_lr_ms; + + return size; +} + +static const struct file_operations min_run_period_lr_ms_fops = { + .owner = THIS_MODULE, + .read = min_run_period_lr_ms_show, + .write = min_run_period_lr_ms_set, +}; + +static ssize_t min_run_period_pf_ms_show(struct file *f, char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + char buf[32]; + int len = 0; + + len = scnprintf(buf, sizeof(buf), "%d\n", xe->min_run_period_pf_ms); + + return simple_read_from_buffer(ubuf, size, pos, buf, len); +} + +static ssize_t min_run_period_pf_ms_set(struct file *f, const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + u32 min_run_period_pf_ms; + ssize_t ret; + + ret = kstrtouint_from_user(ubuf, size, 0, &min_run_period_pf_ms); + if (ret) + return ret; + + xe->min_run_period_pf_ms = min_run_period_pf_ms; + + return size; +} + +static const struct file_operations min_run_period_pf_ms_fops = { + .owner = THIS_MODULE, + .read = min_run_period_pf_ms_show, + .write = min_run_period_pf_ms_set, +}; + static ssize_t disable_late_binding_show(struct file *f, char __user *ubuf, size_t size, loff_t *pos) { @@ -428,6 +496,12 @@ void xe_debugfs_register(struct xe_device *xe) debugfs_create_file("atomic_svm_timeslice_ms", 0600, root, xe, &atomic_svm_timeslice_ms_fops); + debugfs_create_file("min_run_period_lr_ms", 0600, root, xe, + &min_run_period_lr_ms_fops); + + debugfs_create_file("min_run_period_pf_ms", 0600, root, xe, + &min_run_period_pf_ms_fops); + debugfs_create_file("disable_late_binding", 0600, root, xe, &disable_late_binding_fops); diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index cdaa1c1e73f5..00afc84a8683 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -460,6 +460,7 @@ struct xe_device *xe_device_create(struct pci_dev *pdev, xe->info.revid = pdev->revision; xe->info.force_execlist = xe_modparam.force_execlist; xe->atomic_svm_timeslice_ms = 5; + xe->min_run_period_lr_ms = 5; err = xe_irq_init(xe); if (err) diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 85700533db52..413ba4c8b62e 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -627,6 +627,12 @@ struct xe_device { /** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */ u32 atomic_svm_timeslice_ms; + /** @min_run_period_lr_ms: LR VM (preempt fence mode) timeslice */ + u32 min_run_period_lr_ms; + + /** @min_run_period_pf_ms: LR VM (page fault mode) timeslice */ + u32 min_run_period_pf_ms; + #ifdef TEST_VM_OPS_ERROR /** * @vm_inject_error_position: inject errors at different places in VM diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 68dffbf97680..95e22ff95ea8 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1509,9 +1509,9 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) INIT_LIST_HEAD(&vm->preempt.exec_queues); if (flags & XE_VM_FLAG_FAULT_MODE) - vm->preempt.min_run_period_ms = 0; + vm->preempt.min_run_period_ms = xe->min_run_period_pf_ms; else - vm->preempt.min_run_period_ms = 5; + vm->preempt.min_run_period_ms = xe->min_run_period_lr_ms; for_each_tile(tile, xe, id) xe_range_fence_tree_init(&vm->rftree[id]); From 8533051ce92015e9cc6f75e0d52119b9d91610b6 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 12 Dec 2025 10:28:44 -0800 Subject: [PATCH 161/187] drm/xe: Skip exec queue schedule toggle if queue is idle during suspend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If an exec queue is idle, there is no need to issue a schedule disable to the GuC when suspending the queue’s execution. Opportunistically skip this step if the queue is idle and not a parallel queue. Parallel queues must have their scheduling state flipped in the GuC due to limitations in how submission is implemented in run_job(). Also if all pagefault queues can skip the schedule disable during a switch to dma-fence mode, do not schedule a resume for the pagefault queues after the next submission. v2: - Don't touch the LRC tail is queue is suspended but enabled in run_job (CI) Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20251212182847.1683222-5-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.h | 17 ++++++++ drivers/gpu/drm/xe/xe_guc_submit.c | 55 +++++++++++++++++++++++-- drivers/gpu/drm/xe/xe_hw_engine_group.c | 2 +- 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index 10abed98fb6b..b5ad975d7e97 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -162,4 +162,21 @@ int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch); struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q); +/** + * xe_exec_queue_idle_skip_suspend() - Can exec queue skip suspend + * @q: The exec_queue + * + * If an exec queue is not parallel and is idle, the suspend steps can be + * skipped in the submission backend immediatley signaling the suspend fence. + * Parallel queues cannot skip this step due to limitations in the submission + * backend. + * + * Return: True if exec queue is idle and can skip suspend steps, False + * otherwise + */ +static inline bool xe_exec_queue_idle_skip_suspend(struct xe_exec_queue *q) +{ + return !xe_exec_queue_is_parallel(q) && xe_exec_queue_is_idle(q); +} + #endif diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 18cac5594d6a..43fd2069f9b2 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -75,6 +75,7 @@ exec_queue_to_guc(struct xe_exec_queue *q) #define EXEC_QUEUE_STATE_EXTRA_REF (1 << 11) #define EXEC_QUEUE_STATE_PENDING_RESUME (1 << 12) #define EXEC_QUEUE_STATE_PENDING_TDR_EXIT (1 << 13) +#define EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND (1 << 14) static bool exec_queue_registered(struct xe_exec_queue *q) { @@ -266,6 +267,21 @@ static void clear_exec_queue_pending_tdr_exit(struct xe_exec_queue *q) atomic_and(~EXEC_QUEUE_STATE_PENDING_TDR_EXIT, &q->guc->state); } +static bool exec_queue_idle_skip_suspend(struct xe_exec_queue *q) +{ + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND; +} + +static void set_exec_queue_idle_skip_suspend(struct xe_exec_queue *q) +{ + atomic_or(EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state); +} + +static void clear_exec_queue_idle_skip_suspend(struct xe_exec_queue *q) +{ + atomic_and(~EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state); +} + static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) { return (atomic_read(&q->guc->state) & @@ -1118,7 +1134,7 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) if (!job->restore_replay || job->last_replay) { if (xe_exec_queue_is_parallel(q)) wq_item_append(q); - else + else if (!exec_queue_idle_skip_suspend(q)) xe_lrc_set_ring_tail(lrc, lrc->ring.tail); job->last_replay = false; } @@ -1906,9 +1922,10 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) { struct xe_exec_queue *q = msg->private_data; struct xe_guc *guc = exec_queue_to_guc(q); + bool idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q); - if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && - exec_queue_enabled(q)) { + if (!idle_skip_suspend && guc_exec_queue_allowed_to_change_state(q) && + !exec_queue_suspended(q) && exec_queue_enabled(q)) { wait_event(guc->ct.wq, vf_recovery(guc) || ((q->guc->resume_time != RESUME_PENDING || xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q))); @@ -1927,11 +1944,33 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) disable_scheduling(q, false); } } else if (q->guc->suspend_pending) { + if (idle_skip_suspend) + set_exec_queue_idle_skip_suspend(q); set_exec_queue_suspended(q); suspend_fence_signal(q); } } +static void sched_context(struct xe_exec_queue *q) +{ + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_lrc *lrc = q->lrc[0]; + u32 action[] = { + XE_GUC_ACTION_SCHED_CONTEXT, + q->guc->id, + }; + + xe_gt_assert(guc_to_gt(guc), !xe_exec_queue_is_parallel(q)); + xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); + xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); + xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); + + trace_xe_exec_queue_submit(q); + + xe_lrc_set_ring_tail(lrc, lrc->ring.tail); + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); +} + static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) { struct xe_exec_queue *q = msg->private_data; @@ -1939,12 +1978,22 @@ static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) if (guc_exec_queue_allowed_to_change_state(q)) { clear_exec_queue_suspended(q); if (!exec_queue_enabled(q)) { + if (exec_queue_idle_skip_suspend(q)) { + struct xe_lrc *lrc = q->lrc[0]; + + clear_exec_queue_idle_skip_suspend(q); + xe_lrc_set_ring_tail(lrc, lrc->ring.tail); + } q->guc->resume_time = RESUME_PENDING; set_exec_queue_pending_resume(q); enable_scheduling(q); + } else if (exec_queue_idle_skip_suspend(q)) { + clear_exec_queue_idle_skip_suspend(q); + sched_context(q); } } else { clear_exec_queue_suspended(q); + clear_exec_queue_idle_skip_suspend(q); } } diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c index 290205a266b8..4d9263a1a208 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_group.c +++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c @@ -205,7 +205,7 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group continue; xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1); - need_resume = true; + need_resume |= !xe_exec_queue_idle_skip_suspend(q); q->ops->suspend(q); } From 4ac9048d05017449dde6320694d6e4700a8b9f5f Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 12 Dec 2025 10:28:45 -0800 Subject: [PATCH 162/187] drm/xe: Wait on in-syncs when swicthing to dma-fence mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a dma-fence submission has in-fences and pagefault queues are running work, there is little incentive to kick the pagefault queues off the hardware until the dma-fence submission is ready to run. Therefore, wait on the in-fences of the dma-fence submission before removing the pagefault queues from the hardware. v2: - Fix kernel doc (CI) - Don't wait under lock (Thomas) - Make wait interruptable Suggested-by: Thomas Hellström Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20251212182847.1683222-6-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_exec.c | 9 +++- drivers/gpu/drm/xe/xe_hw_engine_group.c | 55 +++++++++++++++++++++---- drivers/gpu/drm/xe/xe_hw_engine_group.h | 4 +- drivers/gpu/drm/xe/xe_sync.c | 28 +++++++++++++ drivers/gpu/drm/xe/xe_sync.h | 2 + 5 files changed, 87 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c index fd9480031750..730a5c9c2637 100644 --- a/drivers/gpu/drm/xe/xe_exec.c +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -121,7 +121,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) u64 addresses[XE_HW_ENGINE_MAX_INSTANCE]; struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn}; struct drm_exec *exec = &vm_exec.exec; - u32 i, num_syncs, num_ufence = 0; + u32 i, num_syncs, num_in_sync = 0, num_ufence = 0; struct xe_validation_ctx ctx; struct xe_sched_job *job; struct xe_vm *vm; @@ -183,6 +183,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) if (xe_sync_is_ufence(&syncs[num_syncs])) num_ufence++; + + if (!num_in_sync && xe_sync_needs_wait(&syncs[num_syncs])) + num_in_sync++; } if (XE_IOCTL_DBG(xe, num_ufence > 1)) { @@ -203,7 +206,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) mode = xe_hw_engine_group_find_exec_mode(q); if (mode == EXEC_MODE_DMA_FENCE) { - err = xe_hw_engine_group_get_mode(group, mode, &previous_mode); + err = xe_hw_engine_group_get_mode(group, mode, &previous_mode, + syncs, num_in_sync ? + num_syncs : 0); if (err) goto err_syncs; } diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c index 4d9263a1a208..40ce5d5f543c 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_group.c +++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c @@ -11,6 +11,7 @@ #include "xe_gt.h" #include "xe_gt_stats.h" #include "xe_hw_engine_group.h" +#include "xe_sync.h" #include "xe_vm.h" static void @@ -21,7 +22,8 @@ hw_engine_group_resume_lr_jobs_func(struct work_struct *w) int err; enum xe_hw_engine_group_execution_mode previous_mode; - err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode); + err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode, + NULL, 0); if (err) return; @@ -189,10 +191,12 @@ void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group /** * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group * @group: The hw engine group + * @has_deps: dma-fence job triggering suspend has dependencies * * Return: 0 on success, negative error code on error. */ -static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group) +static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group, + bool has_deps) { int err; struct xe_exec_queue *q; @@ -201,11 +205,18 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group lockdep_assert_held_write(&group->mode_sem); list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { + bool idle_skip_suspend; + if (!xe_vm_in_fault_mode(q->vm)) continue; + idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q); + if (!idle_skip_suspend && has_deps) + return -EAGAIN; + xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1); - need_resume |= !xe_exec_queue_idle_skip_suspend(q); + + need_resume |= !idle_skip_suspend; q->ops->suspend(q); } @@ -258,7 +269,7 @@ static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group return 0; } -static int switch_mode(struct xe_hw_engine_group *group) +static int switch_mode(struct xe_hw_engine_group *group, bool has_deps) { int err = 0; enum xe_hw_engine_group_execution_mode new_mode; @@ -268,7 +279,8 @@ static int switch_mode(struct xe_hw_engine_group *group) switch (group->cur_mode) { case EXEC_MODE_LR: new_mode = EXEC_MODE_DMA_FENCE; - err = xe_hw_engine_group_suspend_faulting_lr_jobs(group); + err = xe_hw_engine_group_suspend_faulting_lr_jobs(group, + has_deps); break; case EXEC_MODE_DMA_FENCE: new_mode = EXEC_MODE_LR; @@ -284,19 +296,36 @@ static int switch_mode(struct xe_hw_engine_group *group) return 0; } +static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs) +{ + int err, i; + + for (i = 0; i < num_syncs; ++i) { + err = xe_sync_entry_wait(syncs + i); + if (err) + return err; + } + + return 0; +} + /** * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode * @group: The hw engine group * @new_mode: The new execution mode * @previous_mode: Pointer to the previous mode provided for use by caller + * @syncs: Syncs from exec IOCTL + * @num_syncs: Number of syncs from exec IOCTL * * Return: 0 if successful, -EINTR if locking failed. */ int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group, enum xe_hw_engine_group_execution_mode new_mode, - enum xe_hw_engine_group_execution_mode *previous_mode) + enum xe_hw_engine_group_execution_mode *previous_mode, + struct xe_sync_entry *syncs, int num_syncs) __acquires(&group->mode_sem) { + bool has_deps = !!num_syncs; int err = down_read_interruptible(&group->mode_sem); if (err) @@ -306,15 +335,25 @@ __acquires(&group->mode_sem) if (new_mode != group->cur_mode) { up_read(&group->mode_sem); +retry: err = down_write_killable(&group->mode_sem); if (err) return err; if (new_mode != group->cur_mode) { - err = switch_mode(group); + err = switch_mode(group, has_deps); if (err) { up_write(&group->mode_sem); - return err; + + if (err != -EAGAIN) + return err; + + err = wait_syncs(syncs, num_syncs); + if (err) + return err; + + has_deps = false; + goto retry; } } downgrade_write(&group->mode_sem); diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.h b/drivers/gpu/drm/xe/xe_hw_engine_group.h index 797ee81acbf2..8b17ccd30b70 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_group.h +++ b/drivers/gpu/drm/xe/xe_hw_engine_group.h @@ -11,6 +11,7 @@ struct drm_device; struct xe_exec_queue; struct xe_gt; +struct xe_sync_entry; int xe_hw_engine_setup_groups(struct xe_gt *gt); @@ -19,7 +20,8 @@ void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group, enum xe_hw_engine_group_execution_mode new_mode, - enum xe_hw_engine_group_execution_mode *previous_mode); + enum xe_hw_engine_group_execution_mode *previous_mode, + struct xe_sync_entry *syncs, int num_syncs); void xe_hw_engine_group_put(struct xe_hw_engine_group *group); enum xe_hw_engine_group_execution_mode diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index 1fc4fa278b78..ee1344a880b9 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -228,6 +228,34 @@ int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) return 0; } +/** + * xe_sync_entry_wait() - Wait on in-sync + * @sync: Sync object + * + * If the sync is in an in-sync, wait on the sync to signal. + * + * Return: 0 on success, -ERESTARTSYS on failure (interruption) + */ +int xe_sync_entry_wait(struct xe_sync_entry *sync) +{ + if (sync->flags & DRM_XE_SYNC_FLAG_SIGNAL) + return 0; + + return dma_fence_wait(sync->fence, true); +} + +/** + * xe_sync_needs_wait() - Sync needs a wait (input dma-fence not signaled) + * @sync: Sync object + * + * Return: True if sync needs a wait, False otherwise + */ +bool xe_sync_needs_wait(struct xe_sync_entry *sync) +{ + return !(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL) && + !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags); +} + void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) { if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) diff --git a/drivers/gpu/drm/xe/xe_sync.h b/drivers/gpu/drm/xe/xe_sync.h index 51f2d803e977..6b949194acff 100644 --- a/drivers/gpu/drm/xe/xe_sync.h +++ b/drivers/gpu/drm/xe/xe_sync.h @@ -29,6 +29,8 @@ int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job); void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence); +int xe_sync_entry_wait(struct xe_sync_entry *sync); +bool xe_sync_needs_wait(struct xe_sync_entry *sync); void xe_sync_entry_cleanup(struct xe_sync_entry *sync); struct dma_fence * xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, From ce3d65549c7a4ea4497546f49d18128281258ec5 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 12 Dec 2025 10:28:46 -0800 Subject: [PATCH 163/187] drm/xe: Add GT stats ktime helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Normalize GT stats that record execution periods in code paths by adding helpers to perform the ktime calculation. Use these helpers in the SVM code. Suggested-by: Francois Dugast Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20251212182847.1683222-7-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_gt_stats.h | 32 ++++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_svm.c | 29 +++++++++-------------------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_stats.h b/drivers/gpu/drm/xe/xe_gt_stats.h index e8aea32bc971..59a7bf60e242 100644 --- a/drivers/gpu/drm/xe/xe_gt_stats.h +++ b/drivers/gpu/drm/xe/xe_gt_stats.h @@ -6,6 +6,8 @@ #ifndef _XE_GT_STATS_H_ #define _XE_GT_STATS_H_ +#include + #include "xe_gt_stats_types.h" struct xe_gt; @@ -23,4 +25,34 @@ xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id, } #endif + +/** + * xe_gt_stats_ktime_us_delta() - Get delta in microseconds between now and a + * start time + * @start: Start time + * + * Helper for GT stats to get delta in microseconds between now and a start + * time, compiles out if GT stats are disabled. + * + * Return: Delta in microseconds between now and a start time + */ +static inline s64 xe_gt_stats_ktime_us_delta(ktime_t start) +{ + return IS_ENABLED(CONFIG_DEBUG_FS) ? + ktime_us_delta(ktime_get(), start) : 0; +} + +/** + * xe_gt_stats_ktime_get() - Get current ktime + * + * Helper for GT stats to get current ktime, compiles out if GT stats are + * disabled. + * + * Return: Get current ktime + */ +static inline ktime_t xe_gt_stats_ktime_get(void) +{ + return IS_ENABLED(CONFIG_DEBUG_FS) ? ktime_get() : 0; +} + #endif diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 46977ec1e0de..93550c7c84ac 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -176,24 +176,13 @@ xe_svm_range_notifier_event_end(struct xe_vm *vm, struct drm_gpusvm_range *r, mmu_range); } -static s64 xe_svm_stats_ktime_us_delta(ktime_t start) -{ - return IS_ENABLED(CONFIG_DEBUG_FS) ? - ktime_us_delta(ktime_get(), start) : 0; -} - static void xe_svm_tlb_inval_us_stats_incr(struct xe_gt *gt, ktime_t start) { - s64 us_delta = xe_svm_stats_ktime_us_delta(start); + s64 us_delta = xe_gt_stats_ktime_us_delta(start); xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_TLB_INVAL_US, us_delta); } -static ktime_t xe_svm_stats_ktime_get(void) -{ - return IS_ENABLED(CONFIG_DEBUG_FS) ? ktime_get() : 0; -} - static void xe_svm_invalidate(struct drm_gpusvm *gpusvm, struct drm_gpusvm_notifier *notifier, const struct mmu_notifier_range *mmu_range) @@ -202,7 +191,7 @@ static void xe_svm_invalidate(struct drm_gpusvm *gpusvm, struct xe_device *xe = vm->xe; struct drm_gpusvm_range *r, *first; struct xe_tile *tile; - ktime_t start = xe_svm_stats_ktime_get(); + ktime_t start = xe_gt_stats_ktime_get(); u64 adj_start = mmu_range->start, adj_end = mmu_range->end; u8 tile_mask = 0, id; long err; @@ -442,7 +431,7 @@ static void xe_svm_copy_us_stats_incr(struct xe_gt *gt, unsigned long npages, ktime_t start) { - s64 us_delta = xe_svm_stats_ktime_us_delta(start); + s64 us_delta = xe_gt_stats_ktime_us_delta(start); if (dir == XE_SVM_COPY_TO_VRAM) { switch (npages) { @@ -494,7 +483,7 @@ static int xe_svm_copy(struct page **pages, u64 vram_addr = XE_VRAM_ADDR_INVALID; int err = 0, pos = 0; bool sram = dir == XE_SVM_COPY_TO_SRAM; - ktime_t start = xe_svm_stats_ktime_get(); + ktime_t start = xe_gt_stats_ktime_get(); /* * This flow is complex: it locates physically contiguous device pages, @@ -986,7 +975,7 @@ static void xe_svm_range_##elem##_us_stats_incr(struct xe_gt *gt, \ struct xe_svm_range *range, \ ktime_t start) \ { \ - s64 us_delta = xe_svm_stats_ktime_us_delta(start); \ + s64 us_delta = xe_gt_stats_ktime_us_delta(start); \ \ switch (xe_svm_range_size(range)) { \ case SZ_4K: \ @@ -1031,7 +1020,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, struct drm_pagemap *dpagemap; struct xe_tile *tile = gt_to_tile(gt); int migrate_try_count = ctx.devmem_only ? 3 : 1; - ktime_t start = xe_svm_stats_ktime_get(), bind_start, get_pages_start; + ktime_t start = xe_gt_stats_ktime_get(), bind_start, get_pages_start; int err; lockdep_assert_held_write(&vm->lock); @@ -1070,7 +1059,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, if (--migrate_try_count >= 0 && xe_svm_range_needs_migrate_to_vram(range, vma, !!dpagemap || ctx.devmem_only)) { - ktime_t migrate_start = xe_svm_stats_ktime_get(); + ktime_t migrate_start = xe_gt_stats_ktime_get(); /* TODO : For multi-device dpagemap will be used to find the * remote tile and remote device. Will need to modify @@ -1107,7 +1096,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, } get_pages: - get_pages_start = xe_svm_stats_ktime_get(); + get_pages_start = xe_gt_stats_ktime_get(); range_debug(range, "GET PAGES"); err = xe_svm_range_get_pages(vm, range, &ctx); @@ -1134,7 +1123,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, xe_svm_range_get_pages_us_stats_incr(gt, range, get_pages_start); range_debug(range, "PAGE FAULT - BIND"); - bind_start = xe_svm_stats_ktime_get(); + bind_start = xe_gt_stats_ktime_get(); xe_validation_guard(&vctx, &vm->xe->val, &exec, (struct xe_val_flags) {}, err) { err = xe_vm_drm_exec_lock(vm, &exec); drm_exec_retry_on_contention(&exec); From 2b277b506138f501693f6278e675da259299e8aa Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 12 Dec 2025 10:28:47 -0800 Subject: [PATCH 164/187] drm/xe: Add more GT stats around pagefault mode switch flows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add GT stats to measure the time spent switching between pagefault mode and dma-fence mode. Also add a GT stat to indicate when pagefault suspend is skipped because the system is idle. These metrics will help profile pagefault workloads while 3D and display are enabled. v2: - Use GT stats helper functions (Francois) Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Reviewed-by: Francois Dugast Link: https://patch.msgid.link/20251212182847.1683222-8-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_gt_stats.c | 6 ++++++ drivers/gpu/drm/xe/xe_gt_stats_types.h | 3 +++ drivers/gpu/drm/xe/xe_hw_engine_group.c | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_gt_stats.c b/drivers/gpu/drm/xe/xe_gt_stats.c index 714045ad9354..fb2904bd0abd 100644 --- a/drivers/gpu/drm/xe/xe_gt_stats.c +++ b/drivers/gpu/drm/xe/xe_gt_stats.c @@ -68,8 +68,14 @@ static const char *const stat_description[__XE_GT_STATS_NUM_IDS] = { DEF_STAT_STR(SVM_2M_BIND_US, "svm_2M_bind_us"), DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, "hw_engine_group_suspend_lr_queue_count"), + DEF_STAT_STR(HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, + "hw_engine_group_skip_lr_queue_count"), DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, "hw_engine_group_wait_dma_queue_count"), + DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US, + "hw_engine_group_suspend_lr_queue_us"), + DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US, + "hw_engine_group_wait_dma_queue_us"), }; /** diff --git a/drivers/gpu/drm/xe/xe_gt_stats_types.h b/drivers/gpu/drm/xe/xe_gt_stats_types.h index aada5df421e5..b92d013091d5 100644 --- a/drivers/gpu/drm/xe/xe_gt_stats_types.h +++ b/drivers/gpu/drm/xe/xe_gt_stats_types.h @@ -45,7 +45,10 @@ enum xe_gt_stats_id { XE_GT_STATS_ID_SVM_64K_BIND_US, XE_GT_STATS_ID_SVM_2M_BIND_US, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, + XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, + XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US, + XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US, /* must be the last entry */ __XE_GT_STATS_NUM_IDS, }; diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c index 40ce5d5f543c..f69a32c27458 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_group.c +++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c @@ -200,7 +200,9 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group { int err; struct xe_exec_queue *q; + struct xe_gt *gt = NULL; bool need_resume = false; + ktime_t start = xe_gt_stats_ktime_get(); lockdep_assert_held_write(&group->mode_sem); @@ -215,9 +217,13 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group return -EAGAIN; xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1); + if (idle_skip_suspend) + xe_gt_stats_incr(q->gt, + XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, 1); need_resume |= !idle_skip_suspend; q->ops->suspend(q); + gt = q->gt; } list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { @@ -229,6 +235,12 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group return err; } + if (gt) { + xe_gt_stats_incr(gt, + XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US, + xe_gt_stats_ktime_us_delta(start)); + } + if (need_resume) xe_hw_engine_group_resume_faulting_lr_jobs(group); @@ -249,7 +261,9 @@ static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group { long timeout; struct xe_exec_queue *q; + struct xe_gt *gt = NULL; struct dma_fence *fence; + ktime_t start = xe_gt_stats_ktime_get(); lockdep_assert_held_write(&group->mode_sem); @@ -261,11 +275,18 @@ static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm); timeout = dma_fence_wait(fence, false); dma_fence_put(fence); + gt = q->gt; if (timeout < 0) return -ETIME; } + if (gt) { + xe_gt_stats_incr(gt, + XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US, + xe_gt_stats_ktime_us_delta(start)); + } + return 0; } From ab39e2a8f7aed72929bfc1d58eb5e8766f1d85db Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Fri, 5 Dec 2025 13:26:11 -0800 Subject: [PATCH 165/187] drm/xe/oa/uapi: Expose MERT OA unit A MERT OA unit is available in the SoC on some platforms. Add support for this OA unit and expose it to userspace. The MERT OA unit does not have any HW engines attached, but is otherwise similar to an OAM unit. Signed-off-by: Lucas De Marchi Reviewed-by: Umesh Nerlige Ramappa Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20251205212613.826224-2-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/regs/xe_oa_regs.h | 9 +++++++ drivers/gpu/drm/xe/xe_oa.c | 37 +++++++++++++++++++++++++--- include/uapi/drm/xe_drm.h | 3 +++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/regs/xe_oa_regs.h b/drivers/gpu/drm/xe/regs/xe_oa_regs.h index 638ab3b99eb0..04a729e610aa 100644 --- a/drivers/gpu/drm/xe/regs/xe_oa_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_oa_regs.h @@ -108,4 +108,13 @@ #define XE_OAM_SCMI_0_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_0_BASE) #define XE_OAM_SCMI_1_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_1_BASE) +#define OAMERT_CONTROL XE_REG(0x1453a0) +#define OAMERT_DEBUG XE_REG(0x1453a4) +#define OAMERT_STATUS XE_REG(0x1453a8) +#define OAMERT_HEAD_POINTER XE_REG(0x1453ac) +#define OAMERT_TAIL_POINTER XE_REG(0x1453b0) +#define OAMERT_BUFFER XE_REG(0x1453b4) +#define OAMERT_CONTEXT_CONTROL XE_REG(0x1453c8) +#define OAMERT_MMIO_TRG XE_REG(0x1453cc) + #endif diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 92aa25fc0422..d4e1585004e2 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1940,6 +1940,7 @@ static bool oa_unit_supports_oa_format(struct xe_oa_open_param *param, int type) type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC; case DRM_XE_OA_UNIT_TYPE_OAM: case DRM_XE_OA_UNIT_TYPE_OAM_SAG: + case DRM_XE_OA_UNIT_TYPE_MERT: return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; default: return false; @@ -2227,6 +2228,8 @@ static const struct xe_mmio_range xe2_oa_mux_regs[] = { { .start = 0xE18C, .end = 0xE18C }, /* SAMPLER_MODE */ { .start = 0xE590, .end = 0xE590 }, /* TDL_LSC_LAT_MEASURE_TDL_GFX */ { .start = 0x13000, .end = 0x137FC }, /* PES_0_PESL0 - PES_63_UPPER_PESL3 */ + { .start = 0x145194, .end = 0x145194 }, /* SYS_MEM_LAT_MEASURE */ + { .start = 0x145340, .end = 0x14537C }, /* MERTSS_PES_0 - MERTSS_PES_7 */ {}, }; @@ -2518,7 +2521,12 @@ int xe_oa_register(struct xe_device *xe) static u32 num_oa_units_per_gt(struct xe_gt *gt) { if (xe_gt_is_main_type(gt) || GRAPHICS_VER(gt_to_xe(gt)) < 20) - return 1; + /* + * Mert OA unit belongs to the SoC, not a gt, so should be accessed using + * xe_root_tile_mmio(). However, for all known platforms this is the same as + * accessing via xe_root_mmio_gt()->mmio. + */ + return xe_device_has_mert(gt_to_xe(gt)) ? 2 : 1; else if (!IS_DGFX(gt_to_xe(gt))) return XE_OAM_UNIT_SCMI_0 + 1; /* SAG + SCMI_0 */ else @@ -2602,6 +2610,22 @@ static struct xe_oa_regs __oag_regs(void) }; } +static struct xe_oa_regs __oamert_regs(void) +{ + return (struct xe_oa_regs) { + .base = 0, + .oa_head_ptr = OAMERT_HEAD_POINTER, + .oa_tail_ptr = OAMERT_TAIL_POINTER, + .oa_buffer = OAMERT_BUFFER, + .oa_ctx_ctrl = OAMERT_CONTEXT_CONTROL, + .oa_ctrl = OAMERT_CONTROL, + .oa_debug = OAMERT_DEBUG, + .oa_status = OAMERT_STATUS, + .oa_mmio_trg = OAMERT_MMIO_TRG, + .oa_ctrl_counter_select_mask = OAM_CONTROL_COUNTER_SEL_MASK, + }; +} + static void __xe_oa_init_oa_units(struct xe_gt *gt) { const u32 oam_base_addr[] = { @@ -2615,8 +2639,15 @@ static void __xe_oa_init_oa_units(struct xe_gt *gt) struct xe_oa_unit *u = >->oa.oa_unit[i]; if (xe_gt_is_main_type(gt)) { - u->regs = __oag_regs(); - u->type = DRM_XE_OA_UNIT_TYPE_OAG; + if (!i) { + u->regs = __oag_regs(); + u->type = DRM_XE_OA_UNIT_TYPE_OAG; + } else { + xe_gt_assert(gt, xe_device_has_mert(gt_to_xe(gt))); + xe_gt_assert(gt, gt == xe_root_mmio_gt(gt_to_xe(gt))); + u->regs = __oamert_regs(); + u->type = DRM_XE_OA_UNIT_TYPE_MERT; + } } else { xe_gt_assert(gt, GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270); u->regs = __oam_regs(oam_base_addr[i]); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index c59587529986..726e481574fe 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1696,6 +1696,9 @@ enum drm_xe_oa_unit_type { /** @DRM_XE_OA_UNIT_TYPE_OAM_SAG: OAM_SAG OA unit */ DRM_XE_OA_UNIT_TYPE_OAM_SAG, + + /** @DRM_XE_OA_UNIT_TYPE_MERT: MERT OA unit */ + DRM_XE_OA_UNIT_TYPE_MERT, }; /** From ec02e49f21bc161ba19eca4a696613717b2ce2d2 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Fri, 5 Dec 2025 13:26:12 -0800 Subject: [PATCH 166/187] drm/xe/rtp: Whitelist OAMERT MMIO trigger registers Whitelist OAMERT registers to enable userspace to execute MMIO triggers on OAMERT units. Registers are whitelisted for compute and copy class engines. Signed-off-by: Ashutosh Dixit Reviewed-by: Umesh Nerlige Ramappa Link: https://patch.msgid.link/20251205212613.826224-3-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_reg_whitelist.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c index da49c69076a4..1391cb6ec9c6 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.c +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c @@ -9,6 +9,7 @@ #include "regs/xe_gt_regs.h" #include "regs/xe_oa_regs.h" #include "regs/xe_regs.h" +#include "xe_device.h" #include "xe_gt_types.h" #include "xe_gt_printk.h" #include "xe_platform_types.h" @@ -26,6 +27,13 @@ static bool match_not_render(const struct xe_device *xe, return hwe->class != XE_ENGINE_CLASS_RENDER; } +static bool match_has_mert(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_device_has_mert((struct xe_device *)xe); +} + static const struct xe_rtp_entry_sr register_whitelist[] = { { XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), @@ -94,6 +102,9 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { OAM_STATUS(XE_OAM_SCMI_1_BASE_ADJ), \ OAM_HEAD_POINTER(XE_OAM_SCMI_1_BASE_ADJ)) +#define WHITELIST_OA_MERT_MMIO_TRG \ + WHITELIST_OA_MMIO_TRG(OAMERT_MMIO_TRG, OAMERT_STATUS, OAMERT_HEAD_POINTER) + { XE_RTP_NAME("oag_mmio_trg_rcs"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), ENGINE_CLASS(RENDER)), @@ -114,6 +125,14 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { ENGINE_CLASS(VIDEO_ENHANCE)), XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG) }, + { XE_RTP_NAME("oa_mert_mmio_trg_ccs"), + XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COMPUTE)), + XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG) + }, + { XE_RTP_NAME("oa_mert_mmio_trg_bcs"), + XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COPY)), + XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG) + }, }; static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) From 88d98e74adf3e20f678bb89581a5c3149fdbdeaa Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Fri, 5 Dec 2025 13:26:13 -0800 Subject: [PATCH 167/187] drm/xe/oa: Always set OAG_OAGLBCTXCTRL_COUNTER_RESUME Reports can be written out to the OA buffer using ways other than periodic sampling. These include mmio trigger and context switches. To support these use cases, when periodic sampling is not enabled, OAG_OAGLBCTXCTRL_COUNTER_RESUME must be set. Fixes: 1db9a9dc90ae ("drm/xe/oa: OA stream initialization (OAG)") Signed-off-by: Ashutosh Dixit Reviewed-by: Umesh Nerlige Ramappa Link: https://patch.msgid.link/20251205212613.826224-4-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_oa.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index d4e1585004e2..7dfc59277d6c 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1105,11 +1105,12 @@ static int xe_oa_enable_metric_set(struct xe_oa_stream *stream) oag_buf_size_select(stream) | oag_configure_mmio_trigger(stream, true)); - xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ? - (OAG_OAGLBCTXCTRL_COUNTER_RESUME | + xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, + OAG_OAGLBCTXCTRL_COUNTER_RESUME | + (stream->periodic ? OAG_OAGLBCTXCTRL_TIMER_ENABLE | REG_FIELD_PREP(OAG_OAGLBCTXCTRL_TIMER_PERIOD_MASK, - stream->period_exponent)) : 0); + stream->period_exponent) : 0)); /* * Initialize Super Queue Internal Cnt Register From 5ae6cb153abc0448d28e6969f72fa5f7578048c1 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 5 Dec 2025 14:10:31 +0300 Subject: [PATCH 168/187] drm/xe/vf: fix return type in vf_migration_init_late() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vf_migration_init_late() function is supposed to return zero on success and negative error codes on failure. The error code eventually gets propagated back to the probe() function and returned. The problem is it's declared as type bool so it returns true on error. Change it to type int instead. Fixes: 2e2dab20dd66 ("drm/xe/vf: Enable VF migration only on supported GuC versions") Signed-off-by: Dan Carpenter Reviewed-by: Michal Wajdeczko Link: https://patch.msgid.link/aTK9pwJ_roc8vpDi@stanley.mountain Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_sriov_vf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c index 86423a799d57..1b75405b8d02 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf.c @@ -226,7 +226,7 @@ void xe_sriov_vf_init_early(struct xe_device *xe) vf_migration_init_early(xe); } -static bool vf_migration_init_late(struct xe_device *xe) +static int vf_migration_init_late(struct xe_device *xe) { struct xe_gt *gt = xe_root_mmio_gt(xe); struct xe_uc_fw_version guc_version; From 0a2404c8f6a3a120f79c57ef8a3302c8e8bc34d9 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 5 Dec 2025 14:39:19 +0300 Subject: [PATCH 169/187] drm/xe/xe_sriov_vfio: Fix return value in xe_sriov_vfio_migration_supported() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The xe_sriov_vfio_migration_supported() function is type bool so returning -EPERM means returning true. Return false instead. Fixes: 17f22465c5a5 ("drm/xe/pf: Export helpers for VFIO") Signed-off-by: Dan Carpenter Reviewed-by: Michal Wajdeczko Link: https://patch.msgid.link/aTLEZ4g-FD-iMQ2V@stanley.mountain Signed-off-by: Michał Winiarski --- drivers/gpu/drm/xe/xe_sriov_vfio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_sriov_vfio.c b/drivers/gpu/drm/xe/xe_sriov_vfio.c index e9a7615bb5c5..3da81af97b8b 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vfio.c +++ b/drivers/gpu/drm/xe/xe_sriov_vfio.c @@ -21,7 +21,7 @@ EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_get_pf, "xe-vfio-pci"); bool xe_sriov_vfio_migration_supported(struct xe_device *xe) { if (!IS_SRIOV_PF(xe)) - return -EPERM; + return false; return xe_sriov_pf_migration_supported(xe); } From 7b800ab1b7f60dd3652c06b3d518e9458da5b1cd Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Thu, 4 Dec 2025 14:34:04 -0500 Subject: [PATCH 170/187] MAINTAINERS: Update Xe driver maintainers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Matt Brost, one of the Xe driver creators, as maintainer. Cc: Matthew Brost Cc: Thomas Hellström Cc: David Airlie Cc: Simona Vetter Cc: dri-devel@lists.freedesktop.org Cc: linux-kernel@vger.kernel.org Acked-by: Simona Vetter Acked-by: Matthew Brost Acked-by: Thomas Hellström Link: https://patch.msgid.link/20251204193403.930328-2-rodrigo.vivi@intel.com Signed-off-by: Rodrigo Vivi --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 75b32e1fe962..6d5102d324e8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12518,6 +12518,7 @@ F: include/drm/intel/ F: include/uapi/drm/i915_drm.h INTEL DRM XE DRIVER (Lunar Lake and newer) +M: Matthew Brost M: Thomas Hellström M: Rodrigo Vivi L: intel-xe@lists.freedesktop.org From 2f9405aaa4297f95b42c39779e24f74587a0b6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tapani=20P=C3=A4lli?= Date: Wed, 17 Dec 2025 15:24:12 +0200 Subject: [PATCH 171/187] drm/xe: Fix NULL pointer dereference in xe_exec_ioctl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Helper function xe_sync_needs_wait expects sync->fence when accessing flags, patch makes sure we call only when sync->fence exists. v2: move null checking to xe_sync_needs_wait and make xe_sync_entry_wait utilize this helper (Matthew Auld) v3: further simplify code (Matthew Auld) Fixes NULL pointer dereference seen with Vulkan workloads: [ 118.410401] RIP: 0010:xe_sync_needs_wait+0x27/0x50 [xe] Fixes: 4ac9048d0501 ("drm/xe: Wait on in-syncs when swicthing to dma-fence mode") Signed-off-by: Tapani Pälli Reviewed-by: Matthew Auld Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20251217132412.435755-1-tapani.palli@intel.com --- drivers/gpu/drm/xe/xe_sync.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index ee1344a880b9..c8fdcdbd6ae7 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -238,10 +238,8 @@ int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) */ int xe_sync_entry_wait(struct xe_sync_entry *sync) { - if (sync->flags & DRM_XE_SYNC_FLAG_SIGNAL) - return 0; - - return dma_fence_wait(sync->fence, true); + return xe_sync_needs_wait(sync) ? + dma_fence_wait(sync->fence, true) : 0; } /** @@ -252,8 +250,8 @@ int xe_sync_entry_wait(struct xe_sync_entry *sync) */ bool xe_sync_needs_wait(struct xe_sync_entry *sync) { - return !(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL) && - !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags); + return sync->fence && + !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags); } void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) From ce5971773651ad5c7e26aea29d72ea8887428774 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Wed, 17 Dec 2025 23:40:18 +0100 Subject: [PATCH 172/187] drm/xe: Introduce IF_ARGS macro utility We want to extend our macro-based KLV list definitions with new information about the version from which given KLV is supported. Add utility IF_ARGS macro that can be used in code generators to emit different code based on the presence of additional arguments. Introduce macro itself and extend our kunit tests to cover it. We will use this macro in next patch. Signed-off-by: Michal Wajdeczko Reviewed-by: Daniele Ceraolo Spurio Acked-by: Matthew Brost Link: https://patch.msgid.link/20251217224018.3490-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/tests/xe_args_test.c | 54 +++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_args.h | 27 +++++++++++++ 2 files changed, 81 insertions(+) diff --git a/drivers/gpu/drm/xe/tests/xe_args_test.c b/drivers/gpu/drm/xe/tests/xe_args_test.c index f3fb23aa5d2e..2687a1b054dd 100644 --- a/drivers/gpu/drm/xe/tests/xe_args_test.c +++ b/drivers/gpu/drm/xe/tests/xe_args_test.c @@ -78,6 +78,24 @@ static void pick_arg_example(struct kunit *test) #undef buz } +static void if_args_example(struct kunit *test) +{ + enum { Z = 1, Q }; + +#define foo X, Y +#define bar IF_ARGS(Z, Q, foo) +#define buz IF_ARGS(Z, Q, DROP_FIRST_ARG(FIRST_ARG(foo))) + + KUNIT_EXPECT_EQ(test, bar, Z); + KUNIT_EXPECT_EQ(test, buz, Q); + KUNIT_EXPECT_STREQ(test, __stringify(bar), "Z"); + KUNIT_EXPECT_STREQ(test, __stringify(buz), "Q"); + +#undef foo +#undef bar +#undef buz +} + static void sep_comma_example(struct kunit *test) { #define foo(f) f(X) f(Y) f(Z) f(Q) @@ -198,6 +216,40 @@ static void last_arg_test(struct kunit *test) KUNIT_EXPECT_STREQ(test, __stringify(LAST_ARG(MAX_ARGS)), "-12"); } +static void if_args_test(struct kunit *test) +{ + bool with_args = true; + bool no_args = false; + enum { X = 100 }; + + KUNIT_EXPECT_TRUE(test, IF_ARGS(true, false, FOO_ARGS)); + KUNIT_EXPECT_FALSE(test, IF_ARGS(true, false, NO_ARGS)); + + KUNIT_EXPECT_TRUE(test, CONCATENATE(IF_ARGS(with, no, FOO_ARGS), _args)); + KUNIT_EXPECT_FALSE(test, CONCATENATE(IF_ARGS(with, no, NO_ARGS), _args)); + + KUNIT_EXPECT_STREQ(test, __stringify(IF_ARGS(yes, no, FOO_ARGS)), "yes"); + KUNIT_EXPECT_STREQ(test, __stringify(IF_ARGS(yes, no, NO_ARGS)), "no"); + + KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, FOO_ARGS), -1, FOO_ARGS), 4); + KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, FOO_ARGS), -1, NO_ARGS), -1); + KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, NO_ARGS), -1, FOO_ARGS), 0); + KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, NO_ARGS), -1, NO_ARGS), -1); + + KUNIT_EXPECT_EQ(test, + CALL_ARGS(FIRST_ARG, + CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, FOO_ARGS), _ARGS)), X); + KUNIT_EXPECT_EQ(test, + CALL_ARGS(FIRST_ARG, + CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, NO_ARGS), _ARGS)), -1); + KUNIT_EXPECT_EQ(test, + CALL_ARGS(COUNT_ARGS, + CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, FOO_ARGS), _ARGS)), 4); + KUNIT_EXPECT_EQ(test, + CALL_ARGS(COUNT_ARGS, + CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, NO_ARGS), _ARGS)), 12); +} + static struct kunit_case args_tests[] = { KUNIT_CASE(count_args_test), KUNIT_CASE(call_args_example), @@ -209,6 +261,8 @@ static struct kunit_case args_tests[] = { KUNIT_CASE(last_arg_example), KUNIT_CASE(last_arg_test), KUNIT_CASE(pick_arg_example), + KUNIT_CASE(if_args_example), + KUNIT_CASE(if_args_test), KUNIT_CASE(sep_comma_example), {} }; diff --git a/drivers/gpu/drm/xe/xe_args.h b/drivers/gpu/drm/xe/xe_args.h index 4dbc7e53c624..f550b5e3b993 100644 --- a/drivers/gpu/drm/xe/xe_args.h +++ b/drivers/gpu/drm/xe/xe_args.h @@ -121,6 +121,33 @@ #define PICK_ARG11(args...) PICK_ARG10(DROP_FIRST_ARG(args)) #define PICK_ARG12(args...) PICK_ARG11(DROP_FIRST_ARG(args)) +/** + * IF_ARGS() - Make selection based on optional argument list. + * @then: token to return if arguments are present + * @else: token to return if arguments are empty + * @...: arguments to check (optional) + * + * This macro allows to select a token based on the presence of the argument list. + * + * Example: + * + * #define foo X, Y + * #define bar IF_ARGS(Z, Q, foo) + * #define buz IF_ARGS(Z, Q, DROP_FIRST_ARG(FIRST_ARG(foo))) + * + * With above definitions bar expands to Z while buz expands to Q. + */ +#if defined(CONFIG_CC_IS_CLANG) || GCC_VERSION >= 100100 +#define IF_ARGS(then, else, ...) FIRST_ARG(__VA_OPT__(then,) else) +#else +#define IF_ARGS(then, else, ...) _IF_ARGS(then, else, CALL_ARGS(FIRST_ARG, __VA_ARGS__)) +#define _IF_ARGS(then, else, ...) __IF_ARGS(then, else, CALL_ARGS(COUNT_ARGS, __VA_ARGS__)) +#define __IF_ARGS(then, else, n) ___IF_ARGS(then, else, CALL_ARGS(CONCATENATE, ___IF_ARG, n)) +#define ___IF_ARGS(then, else, if) CALL_ARGS(if, then, else) +#define ___IF_ARG1(then, else) then +#define ___IF_ARG0(then, else) else +#endif + /** * ARGS_SEP_COMMA - Definition of a comma character. * From 09af64eba63ece2d928295e61ab1d56ae264447c Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Tue, 16 Dec 2025 22:48:57 +0100 Subject: [PATCH 173/187] drm/xe/guc: Introduce GUC_FIRMWARE_VER_AT_LEAST helper There are already few places in the code where we need to check GuC firmware version. Wrap existing raw conditions into a named helper macro to make it clear and avoid explicit call of the MAKE_GUC_VER. Suggested-by: Daniele Ceraolo Spurio Signed-off-by: Michal Wajdeczko Cc: Daniele Ceraolo Spurio Cc: Matthew Brost Reviewed-by: Daniele Ceraolo Spurio Acked-by: Matthew Brost Link: https://patch.msgid.link/20251216214902.1429-3-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c | 2 +- drivers/gpu/drm/xe/xe_guc.h | 21 +++++++++++++++++++ drivers/gpu/drm/xe/xe_guc_ads.c | 4 ++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c index 3174a8dee779..7410e7b93256 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c @@ -1026,7 +1026,7 @@ static void action_ring_cleanup(void *arg) static void pf_gt_migration_check_support(struct xe_gt *gt) { - if (GUC_FIRMWARE_VER(>->uc.guc) < MAKE_GUC_VER(70, 54, 0)) + if (!GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 54)) xe_sriov_pf_migration_disable(gt_to_xe(gt), "requires GuC version >= 70.54.0"); } diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h index fdb08658d05a..a169f231cbd8 100644 --- a/drivers/gpu/drm/xe/xe_guc.h +++ b/drivers/gpu/drm/xe/xe_guc.h @@ -18,10 +18,16 @@ */ #define MAKE_GUC_VER(maj, min, pat) (((maj) << 16) | ((min) << 8) | (pat)) #define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch) +#define MAKE_GUC_VER_ARGS(ver...) \ + (BUILD_BUG_ON_ZERO(COUNT_ARGS(ver) < 2 || COUNT_ARGS(ver) > 3) + \ + MAKE_GUC_VER(PICK_ARG1(ver), PICK_ARG2(ver), IF_ARGS(PICK_ARG3(ver), 0, PICK_ARG3(ver)))) + #define GUC_SUBMIT_VER(guc) \ MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY]) #define GUC_FIRMWARE_VER(guc) \ MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_RELEASE]) +#define GUC_FIRMWARE_VER_AT_LEAST(guc, ver...) \ + xe_guc_fw_version_at_least((guc), MAKE_GUC_VER_ARGS(ver)) struct drm_printer; @@ -96,4 +102,19 @@ static inline struct drm_device *guc_to_drm(struct xe_guc *guc) return &guc_to_xe(guc)->drm; } +/** + * xe_guc_fw_version_at_least() - Check if GuC is at least of given version. + * @guc: the &xe_guc + * @ver: the version to check + * + * The @ver should be prepared using MAKE_GUC_VER(major, minor, patch). + * + * Return: true if loaded GuC firmware is at least of given version, + * false otherwise. + */ +static inline bool xe_guc_fw_version_at_least(const struct xe_guc *guc, u32 ver) +{ + return GUC_FIRMWARE_VER(guc) >= ver; +} + #endif diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index e06c6aa335bf..5feeb91426ee 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -347,10 +347,10 @@ static void guc_waklv_init(struct xe_guc_ads *ads) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET); - if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 44, 0) && XE_GT_WA(gt, 16026508708)) + if (GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 44) && XE_GT_WA(gt, 16026508708)) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH); - if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 47, 0) && XE_GT_WA(gt, 16026007364)) { + if (GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 47) && XE_GT_WA(gt, 16026007364)) { u32 data[] = { 0x0, 0xF, From 4cffecaf127ae10ada4da0636f87e0170e347402 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Tue, 16 Dec 2025 22:48:58 +0100 Subject: [PATCH 174/187] drm/xe/pf: Prepare for new threshold KLVs We want to extend our macro-based KLV list definitions with new information about the version from which given KLV is supported. Prepare our code generators to emit dedicated version check if a KLV was defined with the version information. Signed-off-by: Michal Wajdeczko Cc: Daniele Ceraolo Spurio Reviewed-by: Daniele Ceraolo Spurio Acked-by: Matthew Brost Link: https://patch.msgid.link/20251216214902.1429-4-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c | 19 ++++++++++++------- drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c | 9 ++++++--- .../drm/xe/xe_guc_klv_thresholds_set_types.h | 5 +++++ 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index 59c5c6b4d994..6e8507c24986 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -269,7 +269,8 @@ static u32 encode_config_ggtt(u32 *cfg, const struct xe_gt_sriov_config *config, } /* Return: number of configuration dwords written */ -static u32 encode_config(u32 *cfg, const struct xe_gt_sriov_config *config, bool details) +static u32 encode_config(struct xe_gt *gt, u32 *cfg, const struct xe_gt_sriov_config *config, + bool details) { u32 n = 0; @@ -303,9 +304,11 @@ static u32 encode_config(u32 *cfg, const struct xe_gt_sriov_config *config, bool cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_PREEMPT_TIMEOUT); cfg[n++] = config->preempt_timeout; -#define encode_threshold_config(TAG, ...) ({ \ - cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_THRESHOLD_##TAG); \ - cfg[n++] = config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)]; \ +#define encode_threshold_config(TAG, NAME, VER...) ({ \ + if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, VER), true, VER)) { \ + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_THRESHOLD_##TAG); \ + cfg[n++] = config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)]; \ + } \ }); MAKE_XE_GUC_KLV_THRESHOLDS_SET(encode_threshold_config); @@ -328,7 +331,7 @@ static int pf_push_full_vf_config(struct xe_gt *gt, unsigned int vfid) return -ENOBUFS; cfg = xe_guc_buf_cpu_ptr(buf); - num_dwords = encode_config(cfg, config, true); + num_dwords = encode_config(gt, cfg, config, true); xe_gt_assert(gt, num_dwords <= max_cfg_dwords); if (xe_gt_is_media_type(gt)) { @@ -2518,7 +2521,7 @@ ssize_t xe_gt_sriov_pf_config_save(struct xe_gt *gt, unsigned int vfid, void *bu ret = -ENOBUFS; } else { config = pf_pick_vf_config(gt, vfid); - ret = encode_config(buf, config, false) * sizeof(u32); + ret = encode_config(gt, buf, config, false) * sizeof(u32); } } mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); @@ -2551,11 +2554,13 @@ static int pf_restore_vf_config_klv(struct xe_gt *gt, unsigned int vfid, return pf_provision_preempt_timeout(gt, vfid, value[0]); /* auto-generate case statements */ -#define define_threshold_key_to_provision_case(TAG, ...) \ +#define define_threshold_key_to_provision_case(TAG, NAME, VER...) \ case MAKE_GUC_KLV_VF_CFG_THRESHOLD_KEY(TAG): \ BUILD_BUG_ON(MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG) != 1u); \ if (len != MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG)) \ return -EBADMSG; \ + if (IF_ARGS(!GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, VER), false, VER)) \ + return -EKEYREJECTED; \ return pf_provision_threshold(gt, vfid, \ MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG), \ value[0]); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c index 0fd863609848..ece9eed5d7c5 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c @@ -21,6 +21,7 @@ #include "xe_gt_sriov_pf_monitor.h" #include "xe_gt_sriov_pf_policy.h" #include "xe_gt_sriov_pf_service.h" +#include "xe_guc.h" #include "xe_pm.h" #include "xe_sriov_pf.h" #include "xe_sriov_pf_provision.h" @@ -301,9 +302,11 @@ static void pf_add_config_attrs(struct xe_gt *gt, struct dentry *parent, unsigne &sched_priority_fops); /* register all threshold attributes */ -#define register_threshold_attribute(TAG, NAME, ...) \ - debugfs_create_file_unsafe("threshold_" #NAME, 0644, parent, parent, \ - &NAME##_fops); +#define register_threshold_attribute(TAG, NAME, VER...) ({ \ + if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, VER), true, VER)) \ + debugfs_create_file_unsafe("threshold_" #NAME, 0644, parent, parent, \ + &NAME##_fops); \ +}); MAKE_XE_GUC_KLV_THRESHOLDS_SET(register_threshold_attribute) #undef register_threshold_attribute } diff --git a/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h b/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h index 0a028c94756d..5f84da3d10d3 100644 --- a/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h +++ b/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h @@ -24,6 +24,11 @@ * ABI and the associated &NAME, that may be used in code or debugfs/sysfs:: * * define(TAG, NAME) + * + * If required, KLVs can be labeled with GuC firmware version that added them:: + * + * define(TAG, NAME, MAJOR, MINOR) + * define(TAG, NAME, MAJOR, MINOR, PATCH) */ #define MAKE_XE_GUC_KLV_THRESHOLDS_SET(define) \ define(CAT_ERR, cat_error_count) \ From 487524c891d3cc21eb6d243c2885273c5b142b44 Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Tue, 16 Dec 2025 22:48:59 +0100 Subject: [PATCH 175/187] drm/xe/pf: Add handling for MLRC adverse event threshold Since it is illegal to register a MLRC context when scheduler groups are enabled, the GuC consider the VF doing so as an adverse event. Like for other adverse event, there is a threshold for how many times the event can happen before the GuC throws an error, which we need to add support for. Signed-off-by: Daniele Ceraolo Spurio Cc: Michal Wajdeczko Reviewed-by: Michal Wajdeczko Signed-off-by: Michal Wajdeczko Acked-by: Matthew Brost Link: https://patch.msgid.link/20251216214902.1429-5-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/abi/guc_klvs_abi.h | 9 +++++++++ drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h | 1 + 2 files changed, 10 insertions(+) diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h index 265a135e7061..89a4f8c504e6 100644 --- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h @@ -352,6 +352,12 @@ enum { * :1: NORMAL = schedule VF always, irrespective of whether it has work or not * :2: HIGH = schedule VF in the next time-slice after current active * time-slice completes if it has active work + * + * _`GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT` : 0x8A0D + * Given that multi-LRC contexts are incompatible with SRIOV scheduler + * groups and cause the latter to be turned off when registered with the + * GuC, this config allows the PF to set a threshold for multi-LRC context + * registrations by VFs to monitor their behavior. */ #define GUC_KLV_VF_CFG_GGTT_START_KEY 0x0001 @@ -410,6 +416,9 @@ enum { #define GUC_SCHED_PRIORITY_NORMAL 1u #define GUC_SCHED_PRIORITY_HIGH 2u +#define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_KEY 0x8a0d +#define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_LEN 1u + /* * Workaround keys: */ diff --git a/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h b/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h index 5f84da3d10d3..45ab5a3b5218 100644 --- a/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h +++ b/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h @@ -37,6 +37,7 @@ define(H2G_STORM, guc_time_us) \ define(IRQ_STORM, irq_time_us) \ define(DOORBELL_STORM, doorbell_time_us) \ + define(MULTI_LRC_COUNT, multi_lrc_count, 70, 53)\ /* end */ /** From 97e16068d77a036908f69aba35486b21dee17a40 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Thu, 11 Dec 2025 22:18:48 -0800 Subject: [PATCH 176/187] drm/xe/oa: Move default oa unit assignment earlier during stream open De-referencing param.oa_unit, when an OA unit id is not provided during stream open, results in NPD below. Oops: general protection fault, probably for non-canonical address... KASAN: null-ptr-deref in range... RIP: 0010:xe_oa_stream_open_ioctl+0x169/0x38a0 xe_observation_ioctl+0x19f/0x270 drm_ioctl_kernel+0x1f4/0x410 Fix this by moving default oa unit assignment before the dereference. Reported-by: Peter Senna Tschudin Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/6840 Fixes: c7e269aa565f ("drm/xe/oa: Allow exec_queue's to be specified only for OAG OA unit") Signed-off-by: Ashutosh Dixit Reviewed-by: Harish Chegondi Link: https://patch.msgid.link/20251212061850.1565459-2-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_oa.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 7dfc59277d6c..d9573cdbcfd2 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1967,10 +1967,6 @@ static int xe_oa_assign_hwe(struct xe_oa *oa, struct xe_oa_open_param *param) enum xe_hw_engine_id id; int ret = 0; - /* If not provided, OA unit defaults to OA unit 0 as per uapi */ - if (!param->oa_unit) - param->oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0]; - /* When we have an exec_q, get hwe from the exec_q */ if (param->exec_q) { param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class, @@ -2036,6 +2032,10 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f if (ret) return ret; + /* If not provided, OA unit defaults to OA unit 0 as per uapi */ + if (!param.oa_unit) + param.oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0]; + if (param.exec_queue_id > 0) { /* An exec_queue is only needed for OAR/OAC functionality on OAG */ if (XE_IOCTL_DBG(oa->xe, param.oa_unit->type != DRM_XE_OA_UNIT_TYPE_OAG)) From 7a100e6ddcc47c1f6ba7a19402de86ce24790621 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Thu, 11 Dec 2025 22:18:49 -0800 Subject: [PATCH 177/187] drm/xe/oa: Disallow 0 OA property values An OA property value of 0 is invalid and will cause a NPD. Reported-by: Peter Senna Tschudin Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/6452 Fixes: cc4e6994d5a2 ("drm/xe/oa: Move functions up so they can be reused for config ioctl") Cc: stable@vger.kernel.org Signed-off-by: Ashutosh Dixit Reviewed-by: Harish Chegondi Link: https://patch.msgid.link/20251212061850.1565459-3-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_oa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index d9573cdbcfd2..abf87fe0b345 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1347,7 +1347,7 @@ static int xe_oa_user_ext_set_property(struct xe_oa *oa, enum xe_oa_user_extn_fr ARRAY_SIZE(xe_oa_set_property_funcs_config)); if (XE_IOCTL_DBG(oa->xe, ext.property >= ARRAY_SIZE(xe_oa_set_property_funcs_open)) || - XE_IOCTL_DBG(oa->xe, ext.pad)) + XE_IOCTL_DBG(oa->xe, !ext.property) || XE_IOCTL_DBG(oa->xe, ext.pad)) return -EINVAL; idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_oa_set_property_funcs_open)); From 5bf763e908bf795da4ad538d21c1ec41f8021f76 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Thu, 11 Dec 2025 22:18:50 -0800 Subject: [PATCH 178/187] drm/xe/eustall: Disallow 0 EU stall property values An EU stall property value of 0 is invalid and will cause a NPD. Reported-by: Peter Senna Tschudin Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/6453 Fixes: 1537ec85ebd7 ("drm/xe/uapi: Introduce API for EU stall sampling") Cc: stable@vger.kernel.org Signed-off-by: Ashutosh Dixit Reviewed-by: Harish Chegondi Link: https://patch.msgid.link/20251212061850.1565459-4-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_eu_stall.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index 97dfb7945b7a..a5c36a317a70 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -315,7 +315,7 @@ static int xe_eu_stall_user_ext_set_property(struct xe_device *xe, u64 extension return -EFAULT; if (XE_IOCTL_DBG(xe, ext.property >= ARRAY_SIZE(xe_set_eu_stall_property_funcs)) || - XE_IOCTL_DBG(xe, ext.pad)) + XE_IOCTL_DBG(xe, !ext.property) || XE_IOCTL_DBG(xe, ext.pad)) return -EINVAL; idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_set_eu_stall_property_funcs)); From 425fe550fb513b567bd6d01f397d274092a9c274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Wed, 17 Dec 2025 10:34:41 +0100 Subject: [PATCH 179/187] drm/xe: Drop preempt-fences when destroying imported dma-bufs. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When imported dma-bufs are destroyed, TTM is not fully individualizing the dma-resv, but it *is* copying the fences that need to be waited for before declaring idle. So in the case where the bo->resv != bo->_resv we can still drop the preempt-fences, but make sure we do that on bo->_resv which contains the fence-pointer copy. In the case where the copying fails, bo->_resv will typically not contain any fences pointers at all, so there will be nothing to drop. In that case, TTM would have ensured all fences that would have been copied are signaled, including any remaining preempt fences. Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Fixes: fa0af721bd1f ("drm/ttm: test private resv obj on release/destroy") Cc: Matthew Brost Cc: # v6.16+ Signed-off-by: Thomas Hellström Tested-by: Matthew Brost Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20251217093441.5073-1-thomas.hellstrom@linux.intel.com --- drivers/gpu/drm/xe/xe_bo.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 6280e6a013ff..8b6474cd3eaf 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1526,7 +1526,7 @@ static bool xe_ttm_bo_lock_in_destructor(struct ttm_buffer_object *ttm_bo) * always succeed here, as long as we hold the lru lock. */ spin_lock(&ttm_bo->bdev->lru_lock); - locked = dma_resv_trylock(ttm_bo->base.resv); + locked = dma_resv_trylock(&ttm_bo->base._resv); spin_unlock(&ttm_bo->bdev->lru_lock); xe_assert(xe, locked); @@ -1546,13 +1546,6 @@ static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo) bo = ttm_to_xe_bo(ttm_bo); xe_assert(xe_bo_device(bo), !(bo->created && kref_read(&ttm_bo->base.refcount))); - /* - * Corner case where TTM fails to allocate memory and this BOs resv - * still points the VMs resv - */ - if (ttm_bo->base.resv != &ttm_bo->base._resv) - return; - if (!xe_ttm_bo_lock_in_destructor(ttm_bo)) return; @@ -1562,14 +1555,14 @@ static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo) * TODO: Don't do this for external bos once we scrub them after * unbind. */ - dma_resv_for_each_fence(&cursor, ttm_bo->base.resv, + dma_resv_for_each_fence(&cursor, &ttm_bo->base._resv, DMA_RESV_USAGE_BOOKKEEP, fence) { if (xe_fence_is_xe_preempt(fence) && !dma_fence_is_signaled(fence)) { if (!replacement) replacement = dma_fence_get_stub(); - dma_resv_replace_fences(ttm_bo->base.resv, + dma_resv_replace_fences(&ttm_bo->base._resv, fence->context, replacement, DMA_RESV_USAGE_BOOKKEEP); @@ -1577,7 +1570,7 @@ static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo) } dma_fence_put(replacement); - dma_resv_unlock(ttm_bo->base.resv); + dma_resv_unlock(&ttm_bo->base._resv); } static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo) From 9dde74fd9e65ddd952413d98a8a756e12d087627 Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Tue, 16 Dec 2025 13:10:34 +0200 Subject: [PATCH 180/187] drm/xe/nvm: enable cri platform Mark CRI as one that have the CSC NVM device. Update the writable override flow to take the information from the scratch register for CRI. Signed-off-by: Alexander Usyskin Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20251216111034.3093507-1-alexander.usyskin@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_nvm.c | 34 +++++++++++++++++++++++----------- drivers/gpu/drm/xe/xe_pci.c | 1 + 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_nvm.c b/drivers/gpu/drm/xe/xe_nvm.c index 33f4ac82fc80..01510061d4d4 100644 --- a/drivers/gpu/drm/xe/xe_nvm.c +++ b/drivers/gpu/drm/xe/xe_nvm.c @@ -10,6 +10,7 @@ #include "xe_device_types.h" #include "xe_mmio.h" #include "xe_nvm.h" +#include "xe_pcode_api.h" #include "regs/xe_gsc_regs.h" #include "xe_sriov.h" @@ -45,39 +46,50 @@ static bool xe_nvm_non_posted_erase(struct xe_device *xe) { struct xe_mmio *mmio = xe_root_tile_mmio(xe); - if (xe->info.platform != XE_BATTLEMAGE) + switch (xe->info.platform) { + case XE_CRESCENTISLAND: + case XE_BATTLEMAGE: + return !(xe_mmio_read32(mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) & + NVM_NON_POSTED_ERASE_CHICKEN_BIT); + default: return false; - return !(xe_mmio_read32(mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) & - NVM_NON_POSTED_ERASE_CHICKEN_BIT); + } } static bool xe_nvm_writable_override(struct xe_device *xe) { struct xe_mmio *mmio = xe_root_tile_mmio(xe); bool writable_override; - resource_size_t base; + struct xe_reg reg; + u32 test_bit; switch (xe->info.platform) { + case XE_CRESCENTISLAND: + reg = PCODE_SCRATCH(0); + test_bit = FDO_MODE; + break; case XE_BATTLEMAGE: - base = DG2_GSC_HECI2_BASE; + reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE); + test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; break; case XE_PVC: - base = PVC_GSC_HECI2_BASE; + reg = HECI_FWSTS2(PVC_GSC_HECI2_BASE); + test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; break; case XE_DG2: - base = DG2_GSC_HECI2_BASE; + reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE); + test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; break; case XE_DG1: - base = DG1_GSC_HECI2_BASE; + reg = HECI_FWSTS2(DG1_GSC_HECI2_BASE); + test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; break; default: drm_err(&xe->drm, "Unknown platform\n"); return true; } - writable_override = - !(xe_mmio_read32(mmio, HECI_FWSTS2(base)) & - HECI_FW_STATUS_2_NVM_ACCESS_MODE); + writable_override = !(xe_mmio_read32(mmio, reg) & test_bit); if (writable_override) drm_info(&xe->drm, "NVM access overridden by jumper\n"); return writable_override; diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 673761c8623e..24dfde3c0cbf 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -416,6 +416,7 @@ static const struct xe_device_desc cri_desc = { .dma_mask_size = 52, .has_display = false, .has_flat_ccs = false, + .has_gsc_nvm = 1, .has_i2c = true, .has_mbx_power_limits = true, .has_mert = true, From fb084e4183e1d79ead103265cafc4b6bc0ecba03 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Thu, 18 Dec 2025 16:21:53 +0530 Subject: [PATCH 181/187] drm/xe/xe_survivability: Remove unused index Remove unused index variable and fix for loop. Fixes: f4e9fc967afd ("drm/xe/xe_survivability: Redesign survivability mode") Reported-by: Nathan Chancellor Closes: https://lore.kernel.org/intel-xe/20251210075757.GA1206705@ax162/ Signed-off-by: Riana Tauro Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20251218105151.586575-5-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_survivability_mode.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index 7520d7b7f9b8..2869866537ad 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -128,11 +128,6 @@ xe_survivability_attribute *dev_attr_to_survivability_attr(struct device_attribu return container_of(attr, struct xe_survivability_attribute, attr); } -static u32 aux_history_offset(u32 reg_value) -{ - return REG_FIELD_GET(AUXINFO_HISTORY_OFFSET, reg_value); -} - static void set_survivability_info(struct xe_mmio *mmio, u32 *info, int id) { info[id] = xe_mmio_read32(mmio, PCODE_SCRATCH(id)); @@ -144,7 +139,6 @@ static void populate_survivability_info(struct xe_device *xe) u32 *info = survivability->info; struct xe_mmio *mmio; u32 id = 0, reg_value; - int index; mmio = xe_root_tile_mmio(xe); set_survivability_info(mmio, info, CAPABILITY_INFO); @@ -162,13 +156,12 @@ static void populate_survivability_info(struct xe_device *xe) set_survivability_info(mmio, info, POSTCODE_TRACE_OVERFLOW); } + /* Traverse the linked list of aux info registers */ if (reg_value & AUXINFO_SUPPORT) { - id = REG_FIELD_GET(AUXINFO_REG_OFFSET, reg_value); - - for (index = 0; id >= AUX_INFO0 && id < MAX_SCRATCH_REG; index++) { + for (id = REG_FIELD_GET(AUXINFO_REG_OFFSET, reg_value); + id >= AUX_INFO0 && id < MAX_SCRATCH_REG; + id = REG_FIELD_GET(AUXINFO_HISTORY_OFFSET, info[id])) set_survivability_info(mmio, info, id); - id = aux_history_offset(info[id]); - } } } From c7b83a916d62c4d4447d7edf0bc5e06dc2afbdf8 Mon Sep 17 00:00:00 2001 From: Swaraj Gaikwad Date: Tue, 9 Dec 2025 09:48:36 +0000 Subject: [PATCH 182/187] drm/xe: Fix documentation heading levels in xe_guc_pc.c Sphinx reports htmldocs warnings: Documentation/gpu/xe/xe_firmware:31: ./drivers/gpu/drm/xe/xe_guc_pc.c:76: ERROR: A level 2 section cannot be used here. Documentation/gpu/xe/xe_firmware:31: ./drivers/gpu/drm/xe/xe_guc_pc.c:87: ERROR: A level 2 section cannot be used here. The xe_guc_pc.c documentation is included inside xe_firmware.rst. The headers in the C file currently use '=' underlines, which conflict with the parent document's section levels. Fix this by demoting "Frequency management" and "Render-C States" headers from '=' to '-' to correctly nest them as subsections. Build environment: Python 3.13.7 Sphinx 8.2.3 docutils 0.22.3 Signed-off-by: Swaraj Gaikwad Link: https://patch.msgid.link/20251209094836.18589-1-swarajgaikwad1925@gmail.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_guc_pc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index e2e6edb851ae..54702a0fd05b 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -76,7 +76,7 @@ * exposes a programming interface to the host for the control of SLPC. * * Frequency management: - * ===================== + * --------------------- * * Xe driver enables SLPC with all of its defaults features and frequency * selection, which varies per platform. @@ -87,7 +87,7 @@ * for any workload. * * Render-C States: - * ================ + * ---------------- * * Render-C states is also a GuC PC feature that is now enabled in Xe for * all platforms. From af90706f0eeca2ec78f73bdc164c5046bd8a9866 Mon Sep 17 00:00:00 2001 From: Madhur Kumar Date: Sun, 14 Dec 2025 14:06:59 +0530 Subject: [PATCH 183/187] drm/xe/pmu: Replace sprintf() with sysfs_emit() Replace sprintf() calls with sysfs_emit() to follow current kernel coding standards. sysfs_emit() is the preferred method for formatting sysfs output as it provides better bounds checking and is more secure. Signed-off-by: Madhur Kumar Link: https://patch.msgid.link/20251214083659.2412218-1-madhurkumar004@gmail.com Signed-off-by: Rodrigo Vivi [Rodrigo adjusted commit message while pushing it] --- drivers/gpu/drm/xe/xe_pmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_pmu.c b/drivers/gpu/drm/xe/xe_pmu.c index c63335eb69e5..0b20059dd7b3 100644 --- a/drivers/gpu/drm/xe/xe_pmu.c +++ b/drivers/gpu/drm/xe/xe_pmu.c @@ -425,7 +425,7 @@ static ssize_t event_attr_show(struct device *dev, struct perf_pmu_events_attr *pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); - return sprintf(buf, "event=%#04llx\n", pmu_attr->id); + return sysfs_emit(buf, "event=%#04llx\n", pmu_attr->id); } #define XE_EVENT_ATTR(name_, v_, id_) \ From 3d50c69c5819b20149946fda084bb3d6bfda2c44 Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Fri, 19 Dec 2025 16:22:26 +0530 Subject: [PATCH 184/187] drm/xe/xe_survivability: Use static for survivability info attributes Fix sparse warnings. Use static for survivability info attributes. Fixes: f4e9fc967afd ("drm/xe/xe_survivability: Redesign survivability mode") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202512101919.G12cuhBJ-lkp@intel.com/ Signed-off-by: Riana Tauro Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20251219105224.871930-5-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_survivability_mode.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index 2869866537ad..162b57a9ab94 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -221,15 +221,15 @@ static ssize_t survivability_info_show(struct device *dev, .index = _index, \ } -SURVIVABILITY_ATTR_RO(capability_info, CAPABILITY_INFO); -SURVIVABILITY_ATTR_RO(postcode_trace, POSTCODE_TRACE); -SURVIVABILITY_ATTR_RO(postcode_trace_overflow, POSTCODE_TRACE_OVERFLOW); -SURVIVABILITY_ATTR_RO(aux_info0, AUX_INFO0); -SURVIVABILITY_ATTR_RO(aux_info1, AUX_INFO1); -SURVIVABILITY_ATTR_RO(aux_info2, AUX_INFO2); -SURVIVABILITY_ATTR_RO(aux_info3, AUX_INFO3); -SURVIVABILITY_ATTR_RO(aux_info4, AUX_INFO4); -SURVIVABILITY_ATTR_RO(fdo_mode, FDO_INFO); +static SURVIVABILITY_ATTR_RO(capability_info, CAPABILITY_INFO); +static SURVIVABILITY_ATTR_RO(postcode_trace, POSTCODE_TRACE); +static SURVIVABILITY_ATTR_RO(postcode_trace_overflow, POSTCODE_TRACE_OVERFLOW); +static SURVIVABILITY_ATTR_RO(aux_info0, AUX_INFO0); +static SURVIVABILITY_ATTR_RO(aux_info1, AUX_INFO1); +static SURVIVABILITY_ATTR_RO(aux_info2, AUX_INFO2); +static SURVIVABILITY_ATTR_RO(aux_info3, AUX_INFO3); +static SURVIVABILITY_ATTR_RO(aux_info4, AUX_INFO4); +static SURVIVABILITY_ATTR_RO(fdo_mode, FDO_INFO); static void xe_survivability_mode_fini(void *arg) { From 13fe9fa7f41a2f3a0ce25d27297a379560e45b2d Mon Sep 17 00:00:00 2001 From: Riana Tauro Date: Fri, 19 Dec 2025 16:22:27 +0530 Subject: [PATCH 185/187] drm/xe/xe_survivability: Add index bound check Fix static analysis tool reported issue. Add index bound check before accessing info array to prevent out of bound. Fixes: f4e9fc967afd ("drm/xe/xe_survivability: Redesign survivability mode") Signed-off-by: Riana Tauro Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20251219105224.871930-6-riana.tauro@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_survivability_mode.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index 162b57a9ab94..4c716182ad3b 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -251,10 +251,14 @@ static umode_t survivability_info_attrs_visible(struct kobject *kobj, struct att struct xe_survivability *survivability = &xe->survivability; u32 *info = survivability->info; - /* FDO mode is visible only when supported */ - if (idx >= MAX_SCRATCH_REG && survivability->version >= 2) + /* + * Last index in survivability_info_attrs is fdo mode and is applicable only in + * version 2 of survivability mode + */ + if (idx == MAX_SCRATCH_REG && survivability->version >= 2) return 0400; - else if (info[idx]) + + if (idx < MAX_SCRATCH_REG && info[idx]) return 0400; return 0; From 651065dca3f2a2e2cd2ddd3cdebeb7fe6c0da882 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 18 Dec 2025 14:37:45 -0800 Subject: [PATCH 186/187] drm/xe: Increase log level for unhandled page faults Set the kernel log level for unhandled page faults to match the log level (info) for engine resets. Currently, dmesg output can be confusing because it shows an engine reset without indicating the page fault that caused it. Without this change, the GuC log must be examined to determine the source of the engine reset. Signed-off-by: Matthew Brost Reviewed-by: Rodrigo Vivi Reviewed-by: Stuart Summers Link: https://patch.msgid.link/20251218223745.4045207-1-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 12 +++++----- drivers/gpu/drm/xe/xe_pagefault.c | 36 +++++++++++++++--------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 43fd2069f9b2..259c4e1dcae7 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -3159,13 +3159,13 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, * See bspec 54047 and 72187 for details. */ if (type != XE_GUC_CAT_ERR_TYPE_INVALID) - xe_gt_dbg(gt, - "Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d", - type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); + xe_gt_info(gt, + "Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d", + type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); else - xe_gt_dbg(gt, - "Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d", - xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); + xe_gt_info(gt, + "Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d", + xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); trace_xe_exec_queue_memory_cat_error(q); diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index afb06598b6e1..6bee53d6ffc3 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -223,22 +223,22 @@ static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue, static void xe_pagefault_print(struct xe_pagefault *pf) { - xe_gt_dbg(pf->gt, "\n\tASID: %d\n" - "\tFaulted Address: 0x%08x%08x\n" - "\tFaultType: %d\n" - "\tAccessType: %d\n" - "\tFaultLevel: %d\n" - "\tEngineClass: %d %s\n" - "\tEngineInstance: %d\n", - pf->consumer.asid, - upper_32_bits(pf->consumer.page_addr), - lower_32_bits(pf->consumer.page_addr), - pf->consumer.fault_type, - pf->consumer.access_type, - pf->consumer.fault_level, - pf->consumer.engine_class, - xe_hw_engine_class_to_str(pf->consumer.engine_class), - pf->consumer.engine_instance); + xe_gt_info(pf->gt, "\n\tASID: %d\n" + "\tFaulted Address: 0x%08x%08x\n" + "\tFaultType: %d\n" + "\tAccessType: %d\n" + "\tFaultLevel: %d\n" + "\tEngineClass: %d %s\n" + "\tEngineInstance: %d\n", + pf->consumer.asid, + upper_32_bits(pf->consumer.page_addr), + lower_32_bits(pf->consumer.page_addr), + pf->consumer.fault_type, + pf->consumer.access_type, + pf->consumer.fault_level, + pf->consumer.engine_class, + xe_hw_engine_class_to_str(pf->consumer.engine_class), + pf->consumer.engine_instance); } static void xe_pagefault_queue_work(struct work_struct *w) @@ -260,8 +260,8 @@ static void xe_pagefault_queue_work(struct work_struct *w) err = xe_pagefault_service(&pf); if (err) { xe_pagefault_print(&pf); - xe_gt_dbg(pf.gt, "Fault response: Unsuccessful %pe\n", - ERR_PTR(err)); + xe_gt_info(pf.gt, "Fault response: Unsuccessful %pe\n", + ERR_PTR(err)); } pf.producer.ops->ack_fault(&pf, err); From 844758bd99a86e6a07247784727fb337c4b979ca Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 18 Dec 2025 14:45:46 -0800 Subject: [PATCH 187/187] drm/xe: Print GuC queue submission state on engine reset Print the GuC queue submission state when an engine reset occurs, as this provides clues about the cause of the reset. Signed-off-by: Matthew Brost Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20251218224546.4057424-1-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 259c4e1dcae7..0b590271c326 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -3080,8 +3080,9 @@ int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) if (unlikely(!q)) return -EPROTO; - xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d", - xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); + xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x", + xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id, + atomic_read(&q->guc->state)); trace_xe_exec_queue_reset(q);