From 9d770e72e1edb54beacfce5f402edb51632811e3 Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Tue, 28 Apr 2026 20:14:48 +0000 Subject: [PATCH 01/36] drm/xe/pf: Fix EAGAIN sign in pf_migration_consume() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PTR_ERR() returns a negative value, so comparing against the positive EAGAIN is always true for ERR_PTR(-EAGAIN), causing pf_migration_consume() to bail out instead of continuing to the remaining GTs. On multi-GT platforms this can skip GTs that already have data ready. Compare against -EAGAIN to match the intent (and the following line that correctly uses -EAGAIN). While at it, gate PTR_ERR() with IS_ERR(). v2: add IS_ERR() guard before PTR_ERR(). (Gustavo) Fixes: 67df4a5cbc58 ("drm/xe/pf: Add data structures and handlers for migration rings") Cc: Michał Winiarski Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20260428201448.3999428-1-shuicheng.lin@intel.com Signed-off-by: Shuicheng Lin --- drivers/gpu/drm/xe/xe_sriov_pf_migration.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c index 6c4b16409cc9..150a241110fb 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c @@ -149,10 +149,11 @@ pf_migration_consume(struct xe_device *xe, unsigned int vfid) for_each_gt(gt, xe, gt_id) { data = xe_gt_sriov_pf_migration_save_consume(gt, vfid); - if (data && PTR_ERR(data) != EAGAIN) + if (!data) + continue; + if (!IS_ERR(data) || PTR_ERR(data) != -EAGAIN) return data; - if (PTR_ERR(data) == -EAGAIN) - more_data = true; + more_data = true; } if (!more_data) From 7d9c39cfb31ff389490ca1308767c2807a9829a6 Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Wed, 29 Apr 2026 19:22:59 +0000 Subject: [PATCH 02/36] drm/xe/pf: Fix MMIO access using PF view instead of VF view during migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pf_migration_mmio_save() and pf_migration_mmio_restore() initialize a local VF-specific MMIO view via xe_mmio_init_vf_view() but then pass >->mmio (the PF base) to all xe_mmio_read32()/xe_mmio_write32() calls instead of the local &mmio. This causes the PF own SW flag registers to be saved/restored rather than the target VF registers, silently corrupting migration state. Use the VF MMIO view for all register accesses, matching the correct pattern used in pf_clear_vf_scratch_regs(). Fixes: b7c1b990f719 ("drm/xe/pf: Handle MMIO migration data as part of PF control") Cc: Michał Winiarski Assisted-by: Claude:claude-opus-4.6 Reviewed-by: Michal Wajdeczko Reviewed-by: Stuart Summers Link: https://patch.msgid.link/20260429192259.4009211-1-shuicheng.lin@intel.com Signed-off-by: Shuicheng Lin --- drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c index 87a164efcc33..01fe03b9efe8 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c @@ -385,10 +385,10 @@ static int pf_migration_mmio_save(struct xe_gt *gt, unsigned int vfid, void *buf if (xe_gt_is_media_type(gt)) for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++) - regs[n] = xe_mmio_read32(>->mmio, MED_VF_SW_FLAG(n)); + regs[n] = xe_mmio_read32(&mmio, MED_VF_SW_FLAG(n)); else for (n = 0; n < VF_SW_FLAG_COUNT; n++) - regs[n] = xe_mmio_read32(>->mmio, VF_SW_FLAG(n)); + regs[n] = xe_mmio_read32(&mmio, VF_SW_FLAG(n)); return 0; } @@ -407,10 +407,10 @@ static int pf_migration_mmio_restore(struct xe_gt *gt, unsigned int vfid, if (xe_gt_is_media_type(gt)) for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++) - xe_mmio_write32(>->mmio, MED_VF_SW_FLAG(n), regs[n]); + xe_mmio_write32(&mmio, MED_VF_SW_FLAG(n), regs[n]); else for (n = 0; n < VF_SW_FLAG_COUNT; n++) - xe_mmio_write32(>->mmio, VF_SW_FLAG(n), regs[n]); + xe_mmio_write32(&mmio, VF_SW_FLAG(n), regs[n]); return 0; } From ead21111e275c89828dc13ac8cfa657971ec874c Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Sat, 2 May 2026 23:31:43 +0530 Subject: [PATCH 03/36] drm/xe/hw_error: Cleanup array map xe_hw_error_map[] is not worth the memory needed to map two components. Clean it up and use switch() instead, which also, in turn, simplifies bounds checking logic. add/remove: 0/1 grow/shrink: 0/1 up/down: 0/-425 (-425) Function old new delta xe_hw_error_map 136 - -136 xe_hw_error_irq_handler 3728 3439 -289 Total: Before=7700, After=7275, chg -5.52% Signed-off-by: Raag Jadav Reviewed-by: Andi Shyti Link: https://patch.msgid.link/20260502180143.1450266-1-raag.jadav@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_hw_error.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c index 2a31b430570e..8b56ca9700f7 100644 --- a/drivers/gpu/drm/xe/xe_hw_error.c +++ b/drivers/gpu/drm/xe/xe_hw_error.c @@ -36,11 +36,6 @@ static const char * const hec_uncorrected_fw_errors[] = { "Data Corruption" }; -static const unsigned long xe_hw_error_map[] = { - [XE_GT_ERROR] = DRM_XE_RAS_ERR_COMP_CORE_COMPUTE, - [XE_SOC_ERROR] = DRM_XE_RAS_ERR_COMP_SOC_INTERNAL, -}; - enum gt_vector_regs { ERR_STAT_GT_VECTOR0 = 0, ERR_STAT_GT_VECTOR1, @@ -65,6 +60,18 @@ static enum drm_xe_ras_error_severity hw_err_to_severity(const enum hardware_err return DRM_XE_RAS_ERR_SEV_UNCORRECTABLE; } +static inline u32 err_src_to_id(u32 err_bit) +{ + switch (err_bit) { + case XE_GT_ERROR: + return DRM_XE_RAS_ERR_COMP_CORE_COMPUTE; + case XE_SOC_ERROR: + return DRM_XE_RAS_ERR_COMP_SOC_INTERNAL; + default: + return 0; + } +} + static const char * const pvc_master_global_err_reg[] = { [0 ... 1] = "Undefined", [2] = "HBM SS0: Channel0", @@ -459,14 +466,8 @@ static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_er const char *name; u32 error_id; - /* Check error bit is within bounds */ - if (err_bit >= ARRAY_SIZE(xe_hw_error_map)) - break; - - error_id = xe_hw_error_map[err_bit]; - - /* Check error component is within max */ - if (!error_id || error_id >= DRM_XE_RAS_ERR_COMP_MAX) + error_id = err_src_to_id(err_bit); + if (!error_id) continue; name = info[error_id].name; From 3ec5f003f6c377beda8bd5438941f5a7795e1848 Mon Sep 17 00:00:00 2001 From: Satyanarayana K V P Date: Mon, 4 May 2026 09:49:26 +0000 Subject: [PATCH 04/36] drm/xe/guc: Exclude indirect ring state page from ADS engine state size The engine state size reported to GuC via ADS should only include the engine state portion and should not include the indirect ring state page that comes after it in the context image. The GuC uses this size to overwrite the engine state in the LRC on watchdog resets and we don't want it to overwrite the indirect ring state as well. Fixes: d6219e1cd5e3 ("drm/xe: Add Indirect Ring State support") Suggested-by: Daniele Ceraolo Spurio Signed-off-by: Satyanarayana K V P Cc: Michal Wajdeczko Cc: Matthew Brost Reviewed-by: Matthew Brost Reviewed-by: Daniele Ceraolo Spurio Signed-off-by: Daniele Ceraolo Spurio Link: https://patch.msgid.link/20260504094924.3760713-4-satyanarayana.k.v.p@intel.com --- drivers/gpu/drm/xe/xe_guc_ads.c | 5 +---- drivers/gpu/drm/xe/xe_lrc.c | 11 +++++++++-- drivers/gpu/drm/xe/xe_lrc.h | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index ce651da6f318..b9bca6084a4f 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -515,12 +515,9 @@ static void guc_golden_lrc_init(struct xe_guc_ads *ads) * that starts after the execlists LRC registers. This is * required to allow the GuC to restore just the engine state * when a watchdog reset occurs. - * We calculate the engine state size by removing the size of - * what comes before it in the context image (which is identical - * on all engines). */ ads_blob_write(ads, ads.eng_state_size[guc_class], - real_size - xe_lrc_skip_size(xe)); + xe_lrc_engine_state_size(gt, class)); ads_blob_write(ads, ads.golden_context_lrca[guc_class], addr_ggtt); diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 9db914584347..fdfe2ed5f683 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -727,9 +727,16 @@ size_t xe_lrc_reg_size(struct xe_device *xe) return 80 * sizeof(u32); } -size_t xe_lrc_skip_size(struct xe_device *xe) +/** + * xe_lrc_engine_state_size() - Get size of the engine state within LRC + * @gt: the &xe_gt struct instance + * @class: Hardware engine class + * + * Returns: Size of the engine state + */ +size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class) { - return LRC_PPHWSP_SIZE + xe_lrc_reg_size(xe); + return xe_gt_lrc_hang_replay_size(gt, class) - xe_lrc_reg_size(gt_to_xe(gt)); } static inline u32 __xe_lrc_seqno_offset(struct xe_lrc *lrc) diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index e7c975f9e2d9..5440663183f6 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -130,7 +130,7 @@ u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc); struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc); size_t xe_lrc_reg_size(struct xe_device *xe); -size_t xe_lrc_skip_size(struct xe_device *xe); +size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class); void xe_lrc_dump_default(struct drm_printer *p, struct xe_gt *gt, From 2b8c1d7fe0d521f9d264ef08c15760235e61c341 Mon Sep 17 00:00:00 2001 From: Mallesh Koujalagi Date: Mon, 4 May 2026 16:33:01 +0530 Subject: [PATCH 05/36] drm/xe/xe_survivability: Simplify runtime survivability error handling xe_survivability_mode_runtime_enable() returns an int, but its caller csc_hw_error_work() ignores the return value and cannot take any meaningful recovery action on failure. The function logs errors via dev_err() and proceeds to declare the device wedged regardless of sysfs creation failure, making the return value redundant. Change the return type to void and remove the unnecessary error handling in the caller. v2: - Return is not require after the sysfs creation fail. (Rodrigo/Riana) - Change int to void return type. (Rodrigo) - Remove extra message from csc_hw_error_work(). v3: - Remove ret variable. (Raag) v4: - Drop ret variable from other part of code. v5: - Reframe as refactoring instead of bug fix. (Raag) - Remove Fixes tag and update subject line. Signed-off-by: Mallesh Koujalagi Reviewed-by: Raag Jadav Link: https://patch.msgid.link/20260504110300.1467303-2-mallesh.koujalagi@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_hw_error.c | 5 +---- drivers/gpu/drm/xe/xe_survivability_mode.c | 14 ++++---------- drivers/gpu/drm/xe/xe_survivability_mode.h | 2 +- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c index 8b56ca9700f7..5135e8e4093f 100644 --- a/drivers/gpu/drm/xe/xe_hw_error.c +++ b/drivers/gpu/drm/xe/xe_hw_error.c @@ -176,11 +176,8 @@ static void csc_hw_error_work(struct work_struct *work) { struct xe_tile *tile = container_of(work, typeof(*tile), csc_hw_error_work); struct xe_device *xe = tile_to_xe(tile); - int ret; - ret = xe_survivability_mode_runtime_enable(xe); - if (ret) - drm_err(&xe->drm, "Failed to enable runtime survivability mode\n"); + xe_survivability_mode_runtime_enable(xe); } static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err) diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index db64cac39c94..427afd144f3a 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -396,25 +396,21 @@ bool xe_survivability_mode_is_requested(struct xe_device *xe) * Runtime survivability mode is enabled when certain errors cause the device to be * in non-recoverable state. The device is declared wedged with the appropriate * recovery method and survivability mode sysfs exposed to userspace - * - * Return: 0 if runtime survivability mode is enabled, negative error code otherwise. */ -int xe_survivability_mode_runtime_enable(struct xe_device *xe) +void xe_survivability_mode_runtime_enable(struct xe_device *xe) { struct xe_survivability *survivability = &xe->survivability; struct pci_dev *pdev = to_pci_dev(xe->drm.dev); - int ret; if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || xe->info.platform < XE_BATTLEMAGE) { dev_err(&pdev->dev, "Runtime Survivability Mode not supported\n"); - return -EINVAL; + return; } populate_survivability_info(xe); - ret = create_survivability_sysfs(pdev); - if (ret) - dev_err(&pdev->dev, "Failed to create survivability mode sysfs\n"); + if (create_survivability_sysfs(pdev)) + dev_err(&pdev->dev, "Failed to create survivability sysfs\n"); survivability->type = XE_SURVIVABILITY_TYPE_RUNTIME; dev_err(&pdev->dev, "Runtime Survivability mode enabled\n"); @@ -422,8 +418,6 @@ int xe_survivability_mode_runtime_enable(struct xe_device *xe) xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_VENDOR); xe_device_declare_wedged(xe); dev_err(&pdev->dev, "Firmware flash required, Please refer to the userspace documentation for more details!\n"); - - return 0; } /** diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.h b/drivers/gpu/drm/xe/xe_survivability_mode.h index 1cc94226aa82..cd040e4d18bb 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.h +++ b/drivers/gpu/drm/xe/xe_survivability_mode.h @@ -11,7 +11,7 @@ struct xe_device; int xe_survivability_mode_boot_enable(struct xe_device *xe); -int xe_survivability_mode_runtime_enable(struct xe_device *xe); +void xe_survivability_mode_runtime_enable(struct xe_device *xe); bool xe_survivability_mode_is_boot_enabled(struct xe_device *xe); bool xe_survivability_mode_is_requested(struct xe_device *xe); From e68a60f9e6a602684f335ede8af1170c13998b37 Mon Sep 17 00:00:00 2001 From: Balasubramani Vivekanandan Date: Tue, 5 May 2026 13:18:56 +0530 Subject: [PATCH 06/36] drm/xe/cri: Add new PCI IDs Add support for new CRI PCI IDs. Bspec: 77979 Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20260505074855.3813063-2-balasubramani.vivekanandan@intel.com Signed-off-by: Balasubramani Vivekanandan --- include/drm/intel/pciids.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/drm/intel/pciids.h b/include/drm/intel/pciids.h index 33b91cb2e684..e32ef763427c 100644 --- a/include/drm/intel/pciids.h +++ b/include/drm/intel/pciids.h @@ -898,7 +898,11 @@ /* CRI */ #define INTEL_CRI_IDS(MACRO__, ...) \ - MACRO__(0x674C, ## __VA_ARGS__) + MACRO__(0x674C, ## __VA_ARGS__), \ + MACRO__(0x674D, ## __VA_ARGS__), \ + MACRO__(0x674E, ## __VA_ARGS__), \ + MACRO__(0x674F, ## __VA_ARGS__), \ + MACRO__(0x6750, ## __VA_ARGS__) /* NVL-P */ #define INTEL_NVLP_IDS(MACRO__, ...) \ From 23fb2ea56cb4fa2587bc072b04e4e698687a48e4 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Wed, 6 May 2026 18:50:27 +0530 Subject: [PATCH 07/36] drm/xe/madvise: Track purgeability with BO-local counters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xe_bo_recompute_purgeable_state() walks all VMAs of a BO to determine whether the BO can be made purgeable. This makes VMA create/destroy and madvise updates O(n) in the number of mappings. Replace the walk with BO-local counters protected by the BO dma-resv lock: - vma_count tracks the number of VMAs mapping the BO. - willneed_count tracks active WILLNEED holders, including WILLNEED VMAs and active dma-buf exports for non-imported BOs. A DONTNEED BO is promoted back to WILLNEED on a 0->1 transition of willneed_count. A BO is demoted to DONTNEED on a 1->0 transition only when it still has VMAs, preserving the previous behaviour where a BO with no mappings keeps its current madvise state. PURGED remains terminal, preserving the existing "once purged, always purged" rule. Fixes: 4f44961eab84 ("drm/xe/vm: Prevent binding of purged buffer objects") v2: - Use early return for imported BOs in all four helpers to avoid nesting (Matt B). - Group purgeability state into a purgeable sub-struct on struct xe_bo (Matt B). - Reword xe_bo_willneed_put_locked() kernel-doc to explain that a 1->0 transition means all remaining active VMAs are DONTNEED (Matt B). v3: - Move DONTNEED/PURGED reject from vma_lock_and_validate() into xe_vma_create(), gated on attr->purgeable_state == WILLNEED. Fixes vm_bind bypass and partial-unbind rejection on DONTNEED BOs (Matt B). - Drop .check_purged from MAP and REMAP; keep it for PREFETCH and add a comment why (Matt B). - Skip BO validation in vma_lock_and_validate() for non-WILLNEED VMA remnants so cleanup/remap paths do not repopulate DONTNEED/PURGED BOs. Suggested-by: Thomas Hellström Cc: Matthew Brost Cc: Thomas Hellström Cc: Himal Prasad Ghimiray Signed-off-by: Arvind Yadav Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260506132027.2556046-1-arvind.yadav@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_bo.c | 6 +- drivers/gpu/drm/xe/xe_bo.h | 88 +++++++++++++++- drivers/gpu/drm/xe/xe_bo_types.h | 28 ++++- drivers/gpu/drm/xe/xe_dma_buf.c | 28 ++++- drivers/gpu/drm/xe/xe_vm.c | 51 +++++++-- drivers/gpu/drm/xe/xe_vm_madvise.c | 162 ++--------------------------- drivers/gpu/drm/xe/xe_vm_madvise.h | 2 - 7 files changed, 190 insertions(+), 175 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 5ce60d161e09..eaa3a4ee9111 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -884,10 +884,10 @@ void xe_bo_set_purgeable_state(struct xe_bo *bo, new_state == XE_MADV_PURGEABLE_PURGED); /* Once purged, always purged - cannot transition out */ - xe_assert(xe, !(bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED && + xe_assert(xe, !(bo->purgeable.state == XE_MADV_PURGEABLE_PURGED && new_state != XE_MADV_PURGEABLE_PURGED)); - bo->madv_purgeable = new_state; + bo->purgeable.state = new_state; xe_bo_set_purgeable_shrinker(bo, new_state); } @@ -2355,7 +2355,7 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, INIT_LIST_HEAD(&bo->vram_userfault_link); /* Initialize purge advisory state */ - bo->madv_purgeable = XE_MADV_PURGEABLE_WILLNEED; + bo->purgeable.state = XE_MADV_PURGEABLE_WILLNEED; drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size); diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 68dea7d25a6b..6340317f7d2e 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -251,7 +251,7 @@ static inline bool xe_bo_is_protected(const struct xe_bo *bo) static inline bool xe_bo_is_purged(struct xe_bo *bo) { xe_bo_assert_held(bo); - return bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED; + return bo->purgeable.state == XE_MADV_PURGEABLE_PURGED; } /** @@ -268,11 +268,95 @@ static inline bool xe_bo_is_purged(struct xe_bo *bo) static inline bool xe_bo_madv_is_dontneed(struct xe_bo *bo) { xe_bo_assert_held(bo); - return bo->madv_purgeable == XE_MADV_PURGEABLE_DONTNEED; + return bo->purgeable.state == XE_MADV_PURGEABLE_DONTNEED; } void xe_bo_set_purgeable_state(struct xe_bo *bo, enum xe_madv_purgeable_state new_state); +/** + * xe_bo_willneed_get_locked() - Acquire a WILLNEED holder on a BO + * @bo: Buffer object + * + * Increments willneed_count and, on a 0->1 transition, promotes the BO + * from DONTNEED to WILLNEED. PURGED is terminal and is never modified. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_willneed_get_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + /* Imported BOs are owned externally; do not track purgeability. */ + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + if (bo->purgeable.willneed_count++ == 0 && xe_bo_madv_is_dontneed(bo)) + xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_WILLNEED); +} + +/** + * xe_bo_willneed_put_locked() - Release a WILLNEED holder on a BO + * @bo: Buffer object + * + * Decrements willneed_count and, on a 1->0 transition, marks the BO + * DONTNEED only if it still has VMAs (implying all active VMAs are + * DONTNEED). If the last VMA is being removed, preserve the current BO + * state to match the previous VMA-walk semantics. + * + * PURGED is terminal and the BO state is never modified. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_willneed_put_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + xe_assert(xe_bo_device(bo), bo->purgeable.willneed_count > 0); + if (--bo->purgeable.willneed_count == 0 && bo->purgeable.vma_count > 0 && + !xe_bo_is_purged(bo)) + xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_DONTNEED); +} + +/** + * xe_bo_vma_count_inc_locked() - Account a new VMA on a BO + * @bo: Buffer object + * + * Increments vma_count. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_vma_count_inc_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + bo->purgeable.vma_count++; +} + +/** + * xe_bo_vma_count_dec_locked() - Account a VMA removal on a BO + * @bo: Buffer object + * + * Decrements vma_count. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_vma_count_dec_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + xe_assert(xe_bo_device(bo), bo->purgeable.vma_count > 0); + bo->purgeable.vma_count--; +} + static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo) { if (likely(bo)) { diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index 9c199badd9b2..fcc63ae3f455 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -111,10 +111,32 @@ struct xe_bo { u64 min_align; /** - * @madv_purgeable: user space advise on BO purgeability, protected - * by BO's dma-resv lock. + * @purgeable: Purgeability state and accounting. + * + * All fields are protected by the BO's dma-resv lock. */ - u32 madv_purgeable; + struct { + /** + * @purgeable.state: BO purgeability state + * (WILLNEED/DONTNEED/PURGED). + */ + u32 state; + + /** + * @purgeable.vma_count: Number of VMAs currently mapping this BO. + */ + u32 vma_count; + + /** + * @purgeable.willneed_count: Number of active WILLNEED holders. + * + * Counts WILLNEED VMAs plus active dma-buf exports for + * non-imported BOs. The BO flips to DONTNEED on a 1->0 + * transition only when VMAs still exist; if the last VMA is + * removed, the previous BO state is preserved. + */ + u32 willneed_count; + } purgeable; }; #endif diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index b9828da15897..855d32ba314d 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -193,6 +193,18 @@ static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, return 0; } +static void xe_dma_buf_release(struct dma_buf *dmabuf) +{ + struct drm_gem_object *obj = dmabuf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + + xe_bo_lock(bo, false); + xe_bo_willneed_put_locked(bo); + xe_bo_unlock(bo); + + drm_gem_dmabuf_release(dmabuf); +} + static const struct dma_buf_ops xe_dmabuf_ops = { .attach = xe_dma_buf_attach, .detach = xe_dma_buf_detach, @@ -200,7 +212,7 @@ static const struct dma_buf_ops xe_dmabuf_ops = { .unpin = xe_dma_buf_unpin, .map_dma_buf = xe_dma_buf_map, .unmap_dma_buf = xe_dma_buf_unmap, - .release = drm_gem_dmabuf_release, + .release = xe_dma_buf_release, .begin_cpu_access = xe_dma_buf_begin_cpu_access, .mmap = drm_gem_dmabuf_mmap, .vmap = drm_gem_dmabuf_vmap, @@ -241,18 +253,26 @@ struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags) ret = -EINVAL; goto out_unlock; } + + xe_bo_willneed_get_locked(bo); xe_bo_unlock(bo); ret = ttm_bo_setup_export(&bo->ttm, &ctx); if (ret) - return ERR_PTR(ret); + goto out_put; buf = drm_gem_prime_export(obj, flags); - if (!IS_ERR(buf)) - buf->ops = &xe_dmabuf_ops; + if (IS_ERR(buf)) { + ret = PTR_ERR(buf); + goto out_put; + } + buf->ops = &xe_dmabuf_ops; return buf; +out_put: + xe_bo_lock(bo, false); + xe_bo_willneed_put_locked(bo); out_unlock: xe_bo_unlock(bo); return ERR_PTR(ret); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index c3836f6eab35..e5db3e25849c 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1120,6 +1120,25 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, xe_bo_assert_held(bo); + /* + * Reject only WILLNEED mappings on DONTNEED/PURGED BOs. This + * gates new vm_bind ioctls (user supplies WILLNEED) while + * still allowing partial-unbind / remap splits whose new VMAs + * inherit the parent's DONTNEED attr. It must also run before + * xe_bo_willneed_get_locked() below so a 0->1 holder bump + * cannot silently promote DONTNEED back to WILLNEED. + */ + if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) { + if (xe_bo_madv_is_dontneed(bo)) { + xe_vma_free(vma); + return ERR_PTR(-EBUSY); + } + if (xe_bo_is_purged(bo)) { + xe_vma_free(vma); + return ERR_PTR(-EINVAL); + } + } + vm_bo = drm_gpuvm_bo_obtain_locked(vma->gpuva.vm, &bo->ttm.base); if (IS_ERR(vm_bo)) { xe_vma_free(vma); @@ -1131,6 +1150,10 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, vma->gpuva.gem.offset = bo_offset_or_userptr; drm_gpuva_link(&vma->gpuva, vm_bo); drm_gpuvm_bo_put(vm_bo); + + xe_bo_vma_count_inc_locked(bo); + if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) + xe_bo_willneed_get_locked(bo); } else /* userptr or null */ { if (!is_null && !is_cpu_addr_mirror) { struct xe_userptr_vma *uvma = to_userptr_vma(vma); @@ -1208,7 +1231,10 @@ static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) xe_bo_assert_held(bo); drm_gpuva_unlink(&vma->gpuva); - xe_bo_recompute_purgeable_state(bo); + + xe_bo_vma_count_dec_locked(bo); + if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) + xe_bo_willneed_put_locked(bo); } xe_vm_assert_held(vm); @@ -3016,7 +3042,7 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, * @res_evict: Allow evicting resources during validation * @validate: Perform BO validation * @request_decompress: Request BO decompression - * @check_purged: Reject operation if BO is purged + * @check_purged: Reject operation if BO is DONTNEED or PURGED */ struct xe_vma_lock_and_validate_flags { u32 res_evict : 1; @@ -3030,6 +3056,7 @@ static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma, { struct xe_bo *bo = xe_vma_bo(vma); struct xe_vm *vm = xe_vma_vm(vma); + bool validate_bo = flags.validate; int err = 0; if (bo) { @@ -3044,7 +3071,11 @@ static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma, err = -EINVAL; /* BO already purged */ } - if (!err && flags.validate) + /* Don't validate the BO for DONTNEED/PURGED remap remnants. */ + if (vma->attr.purgeable_state != XE_MADV_PURGEABLE_WILLNEED) + validate_bo = false; + + if (!err && validate_bo) err = xe_bo_validate(bo, vm, xe_vm_allow_vm_eviction(vm) && flags.res_evict, exec); @@ -3152,7 +3183,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, op->map.immediate, .request_decompress = op->map.request_decompress, - .check_purged = true, + .check_purged = false, }); break; case DRM_GPUVA_OP_REMAP: @@ -3174,7 +3205,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, .res_evict = res_evict, .validate = true, .request_decompress = false, - .check_purged = true, + .check_purged = false, }); if (!err && op->remap.next) err = vma_lock_and_validate(exec, op->remap.next, @@ -3182,7 +3213,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, .res_evict = res_evict, .validate = true, .request_decompress = false, - .check_purged = true, + .check_purged = false, }); break; case DRM_GPUVA_OP_UNMAP: @@ -3211,9 +3242,11 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, } /* - * Prefetch attempts to migrate BO's backing store without - * repopulating it first. Purged BOs have no backing store - * to migrate, so reject the operation. + * PREFETCH is the only op that still gates on BO purge state. + * MAP/REMAP handle this inside xe_vma_create() so partial + * unbind on a DONTNEED BO still works. PREFETCH skips + * xe_vma_create() and would migrate a BO with no backing + * store, so reject DONTNEED/PURGED here. */ err = vma_lock_and_validate(exec, gpuva_to_vma(op->base.prefetch.va), diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index c78906dea82b..c4fb29004195 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -185,147 +185,6 @@ static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm, } } -/** - * xe_bo_is_dmabuf_shared() - Check if BO is shared via dma-buf - * @bo: Buffer object - * - * Prevent marking imported or exported dma-bufs as purgeable. - * For imported BOs, Xe doesn't own the backing store and cannot - * safely reclaim pages (exporter or other devices may still be - * using them). For exported BOs, external devices may have active - * mappings we cannot track. - * - * Return: true if BO is imported or exported, false otherwise - */ -static bool xe_bo_is_dmabuf_shared(struct xe_bo *bo) -{ - struct drm_gem_object *obj = &bo->ttm.base; - - /* Imported: exporter owns backing store */ - if (drm_gem_is_imported(obj)) - return true; - - /* Exported: external devices may be accessing */ - if (obj->dma_buf) - return true; - - return false; -} - -/** - * enum xe_bo_vmas_purge_state - VMA purgeable state aggregation - * - * Distinguishes whether a BO's VMAs are all DONTNEED, have at least - * one WILLNEED, or have no VMAs at all. - * - * Enum values align with XE_MADV_PURGEABLE_* states for consistency. - */ -enum xe_bo_vmas_purge_state { - /** @XE_BO_VMAS_STATE_WILLNEED: At least one VMA is WILLNEED */ - XE_BO_VMAS_STATE_WILLNEED = 0, - /** @XE_BO_VMAS_STATE_DONTNEED: All VMAs are DONTNEED */ - XE_BO_VMAS_STATE_DONTNEED = 1, - /** @XE_BO_VMAS_STATE_NO_VMAS: BO has no VMAs */ - XE_BO_VMAS_STATE_NO_VMAS = 2, -}; - -/* - * xe_bo_recompute_purgeable_state() casts between xe_bo_vmas_purge_state and - * xe_madv_purgeable_state. Enforce that WILLNEED=0 and DONTNEED=1 match across - * both enums so the single-line cast is always valid. - */ -static_assert(XE_BO_VMAS_STATE_WILLNEED == (int)XE_MADV_PURGEABLE_WILLNEED, - "VMA purge state WILLNEED must equal madv purgeable WILLNEED"); -static_assert(XE_BO_VMAS_STATE_DONTNEED == (int)XE_MADV_PURGEABLE_DONTNEED, - "VMA purge state DONTNEED must equal madv purgeable DONTNEED"); - -/** - * xe_bo_all_vmas_dontneed() - Determine BO VMA purgeable state - * @bo: Buffer object - * - * Check all VMAs across all VMs to determine aggregate purgeable state. - * Shared BOs require unanimous DONTNEED state from all mappings. - * - * Caller must hold BO dma-resv lock. - * - * Return: XE_BO_VMAS_STATE_DONTNEED if all VMAs are DONTNEED, - * XE_BO_VMAS_STATE_WILLNEED if at least one VMA is not DONTNEED, - * XE_BO_VMAS_STATE_NO_VMAS if BO has no VMAs - */ -static enum xe_bo_vmas_purge_state xe_bo_all_vmas_dontneed(struct xe_bo *bo) -{ - struct drm_gpuvm_bo *vm_bo; - struct drm_gpuva *gpuva; - struct drm_gem_object *obj = &bo->ttm.base; - bool has_vmas = false; - - xe_bo_assert_held(bo); - - /* Shared dma-bufs cannot be purgeable */ - if (xe_bo_is_dmabuf_shared(bo)) - return XE_BO_VMAS_STATE_WILLNEED; - - drm_gem_for_each_gpuvm_bo(vm_bo, obj) { - drm_gpuvm_bo_for_each_va(gpuva, vm_bo) { - struct xe_vma *vma = gpuva_to_vma(gpuva); - - has_vmas = true; - - /* Any non-DONTNEED VMA prevents purging */ - if (vma->attr.purgeable_state != XE_MADV_PURGEABLE_DONTNEED) - return XE_BO_VMAS_STATE_WILLNEED; - } - } - - /* - * No VMAs => preserve existing BO purgeable state. - * Avoids incorrectly flipping DONTNEED -> WILLNEED when last VMA unmapped. - */ - if (!has_vmas) - return XE_BO_VMAS_STATE_NO_VMAS; - - return XE_BO_VMAS_STATE_DONTNEED; -} - -/** - * xe_bo_recompute_purgeable_state() - Recompute BO purgeable state from VMAs - * @bo: Buffer object - * - * Walk all VMAs to determine if BO should be purgeable or not. - * Shared BOs require unanimous DONTNEED state from all mappings. - * If the BO has no VMAs the existing state is preserved. - * - * Locking: Caller must hold BO dma-resv lock. When iterating GPUVM lists, - * VM lock must also be held (write) to prevent concurrent VMA modifications. - * This is satisfied at both call sites: - * - xe_vma_destroy(): holds vm->lock write - * - madvise_purgeable(): holds vm->lock write (from madvise ioctl path) - * - * Return: nothing - */ -void xe_bo_recompute_purgeable_state(struct xe_bo *bo) -{ - enum xe_bo_vmas_purge_state vma_state; - - if (!bo) - return; - - xe_bo_assert_held(bo); - - /* - * Once purged, always purged. Cannot transition back to WILLNEED. - * This matches i915 semantics where purged BOs are permanently invalid. - */ - if (bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED) - return; - - vma_state = xe_bo_all_vmas_dontneed(bo); - - if (vma_state != (enum xe_bo_vmas_purge_state)bo->madv_purgeable && - vma_state != XE_BO_VMAS_STATE_NO_VMAS) - xe_bo_set_purgeable_state(bo, (enum xe_madv_purgeable_state)vma_state); -} - /** * madvise_purgeable - Handle purgeable buffer object advice * @xe: XE device @@ -359,12 +218,6 @@ static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm, /* BO must be locked before modifying madv state */ xe_bo_assert_held(bo); - /* Skip shared dma-bufs - no PTEs to zap */ - if (xe_bo_is_dmabuf_shared(bo)) { - vmas[i]->skip_invalidation = true; - continue; - } - /* * Once purged, always purged. Cannot transition back to WILLNEED. * This matches i915 semantics where purged BOs are permanently invalid. @@ -377,13 +230,14 @@ static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm, switch (op->purge_state_val.val) { case DRM_XE_VMA_PURGEABLE_STATE_WILLNEED: - vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_WILLNEED; vmas[i]->skip_invalidation = true; - - xe_bo_recompute_purgeable_state(bo); + /* Only act on a real DONTNEED -> WILLNEED transition. */ + if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_DONTNEED) { + vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_WILLNEED; + xe_bo_willneed_get_locked(bo); + } break; case DRM_XE_VMA_PURGEABLE_STATE_DONTNEED: - vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_DONTNEED; /* * Don't zap PTEs at DONTNEED time -- pages are still * alive. The zap happens in xe_bo_move_notify() right @@ -391,7 +245,11 @@ static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm, */ vmas[i]->skip_invalidation = true; - xe_bo_recompute_purgeable_state(bo); + /* Only act on a real WILLNEED -> DONTNEED transition. */ + if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) { + vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_DONTNEED; + xe_bo_willneed_put_locked(bo); + } break; default: /* Should never hit - values validated in madvise_args_are_sane() */ diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.h b/drivers/gpu/drm/xe/xe_vm_madvise.h index 39acd2689ca0..a3078f634c7e 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.h +++ b/drivers/gpu/drm/xe/xe_vm_madvise.h @@ -13,6 +13,4 @@ struct xe_bo; int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file); -void xe_bo_recompute_purgeable_state(struct xe_bo *bo); - #endif From 1c76fa517214cbdca458076b4dcb4a3e0b87347f Mon Sep 17 00:00:00 2001 From: Sanjay Yadav Date: Wed, 22 Apr 2026 18:25:03 +0530 Subject: [PATCH 08/36] drm/xe: Convert stolen memory over to ttm_range_manager Stolen memory requires physically contiguous allocations for display scanout and compressed framebuffers. The stolen memory manager was sharing the gpu_buddy allocator backend with the VRAM manager, but buddy manages non-contiguous power-of-two blocks making it a poor fit. Stolen memory also has fundamentally different allocation patterns: - Allocation sizes are not power-of-two. Since buddy rounds up to the next power-of-two block size, a ~17MB request can fail even with ~22MB free, because the free space is fragmented across non-fitting power-of-two blocks. - Hardware restrictions prevent using the first 4K page of stolen for certain allocations (e.g., FBC). The display code sets fpfn=1 to enforce this, but when fpfn != 0, gpu_buddy enables GPU_BUDDY_RANGE_ALLOCATION mode which disables the try_harder coalescing path, further reducing allocation success. This combination caused FBC compressed framebuffer (CFB) allocation failures on platforms like NVL/PTL. In case of NVL where stolen memory is ~56MB and the initial plane framebuffer consumes ~34MB at probe time, leaving ~22MB for subsequent allocations. Use ttm_range_man_init_nocheck() to set up a drm_mm-backed TTM resource manager for stolen memory. This reuses the TTM core's ttm_range_manager callbacks, avoiding duplicate implementations. Tested on NVL with a 4K DP display: stolen_mm shows a single ~22MB contiguous free hole after initial plane framebuffer allocation, and FBC successfully allocates its CFB from that region. The corresponding IGT was previously skipped and now passes. v2: - Clarify that stolen memory requires contiguous allocations (Matt B) - Properly handle xe_ttm_resource_visible() for stolen instead of unconditionally returning true (Matt A) v3: - Rebase - Fix xe_display_bo_fbdev_prefer_stolen() to compare in pages, since ttm_range_manager stores stolen->size in pages not bytes (Matt A) v4: - Add kernel-doc for struct xe_ttm_stolen_mgr (Matt B) Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/work_items/7631 Cc: Maarten Lankhorst Cc: Matthew Brost Suggested-by: Matthew Auld Assisted-by: GitHub Copilot:claude-sonnet-4.6 Signed-off-by: Sanjay Yadav Acked-by: Maarten Lankhorst Acked-by: Vinod Govindapillai Reviewed-by: Matthew Auld Link: https://patch.msgid.link/20260422125502.3088222-2-sanjay.kumar.yadav@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/display/xe_display_bo.c | 2 +- drivers/gpu/drm/xe/xe_bo.c | 16 ++++-- drivers/gpu/drm/xe/xe_device_types.h | 3 + drivers/gpu/drm/xe/xe_res_cursor.h | 14 ++++- drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c | 64 ++++++++-------------- drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h | 12 ++++ drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 11 ++-- 7 files changed, 67 insertions(+), 55 deletions(-) diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.c b/drivers/gpu/drm/xe/display/xe_display_bo.c index dc0d78ff2d79..7fbac223b097 100644 --- a/drivers/gpu/drm/xe/display/xe_display_bo.c +++ b/drivers/gpu/drm/xe/display/xe_display_bo.c @@ -138,7 +138,7 @@ bool xe_display_bo_fbdev_prefer_stolen(struct xe_device *xe, unsigned int size) * important and we should probably use that space with FBC or other * features. */ - return stolen->size >= size * 2; + return stolen->size >= (size * 2) >> PAGE_SHIFT; } static struct drm_gem_object *xe_display_bo_fbdev_create(struct drm_device *drm, int size) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index eaa3a4ee9111..4c80bac67622 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -586,11 +586,17 @@ static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt) kfree(tt); } -static bool xe_ttm_resource_visible(struct ttm_resource *mem) +static bool xe_ttm_resource_visible(struct xe_device *xe, struct ttm_resource *mem) { - struct xe_ttm_vram_mgr_resource *vres = - to_xe_ttm_vram_mgr_resource(mem); + struct xe_ttm_vram_mgr_resource *vres; + if (mem->mem_type == XE_PL_STOLEN) { + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; + + return mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe); + } + + vres = to_xe_ttm_vram_mgr_resource(mem); return vres->used_visible_size == mem->size; } @@ -608,7 +614,7 @@ bool xe_bo_is_visible_vram(struct xe_bo *bo) if (drm_WARN_ON(bo->ttm.base.dev, !xe_bo_is_vram(bo))) return false; - return xe_ttm_resource_visible(bo->ttm.resource); + return xe_ttm_resource_visible(xe_bo_device(bo), bo->ttm.resource); } static int xe_ttm_io_mem_reserve(struct ttm_device *bdev, @@ -624,7 +630,7 @@ static int xe_ttm_io_mem_reserve(struct ttm_device *bdev, case XE_PL_VRAM1: { struct xe_vram_region *vram = xe_map_resource_to_region(mem); - if (!xe_ttm_resource_visible(mem)) + if (!xe_ttm_resource_visible(xe, mem)) return -EINVAL; mem->bus.offset = mem->start << PAGE_SHIFT; diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 89437de3001a..32dd2ffbc796 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -42,6 +42,7 @@ struct xe_ggtt; struct xe_i2c; struct xe_pat_ops; struct xe_pxp; +struct xe_ttm_stolen_mgr; struct xe_vram_region; /** @@ -276,6 +277,8 @@ struct xe_device { struct ttm_resource_manager sys_mgr; /** @mem.shrinker: system memory shrinker. */ struct xe_shrinker *shrinker; + /** @mem.stolen_mgr: stolen memory manager. */ + struct xe_ttm_stolen_mgr *stolen_mgr; } mem; /** @sriov: device level virtualization data */ diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h index 5f4ab08c0686..0522caafd89d 100644 --- a/drivers/gpu/drm/xe/xe_res_cursor.h +++ b/drivers/gpu/drm/xe/xe_res_cursor.h @@ -101,7 +101,15 @@ static inline void xe_res_first(struct ttm_resource *res, cur->mem_type = res->mem_type; switch (cur->mem_type) { - case XE_PL_STOLEN: + case XE_PL_STOLEN: { + /* res->start is in pages (ttm_range_manager). */ + cur->start = (res->start << PAGE_SHIFT) + start; + cur->size = size; + cur->remaining = size; + cur->node = NULL; + cur->mm = NULL; + break; + } case XE_PL_VRAM0: case XE_PL_VRAM1: { struct gpu_buddy_block *block; @@ -289,6 +297,10 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size) switch (cur->mem_type) { case XE_PL_STOLEN: + /* Just advance within the contiguous region. */ + cur->start += size; + cur->size = cur->remaining; + break; case XE_PL_VRAM0: case XE_PL_VRAM1: start = size - cur->size; diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c index 27c9d72222cf..5e9070739e65 100644 --- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c @@ -19,30 +19,11 @@ #include "xe_device.h" #include "xe_gt_printk.h" #include "xe_mmio.h" -#include "xe_res_cursor.h" #include "xe_sriov.h" #include "xe_ttm_stolen_mgr.h" -#include "xe_ttm_vram_mgr.h" #include "xe_vram.h" #include "xe_wa.h" -struct xe_ttm_stolen_mgr { - struct xe_ttm_vram_mgr base; - - /* PCI base offset */ - resource_size_t io_base; - /* GPU base offset */ - resource_size_t stolen_base; - - void __iomem *mapping; -}; - -static inline struct xe_ttm_stolen_mgr * -to_stolen_mgr(struct ttm_resource_manager *man) -{ - return container_of(man, struct xe_ttm_stolen_mgr, base.manager); -} - /** * xe_ttm_stolen_cpu_access_needs_ggtt() - If we can't directly CPU access * stolen, can we then fallback to mapping through the GGTT. @@ -210,12 +191,19 @@ static u64 detect_stolen(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr) #endif } +static void xe_ttm_stolen_mgr_fini(struct drm_device *dev, void *arg) +{ + struct xe_device *xe = to_xe_device(dev); + + ttm_range_man_fini_nocheck(&xe->ttm, XE_PL_STOLEN); +} + int xe_ttm_stolen_mgr_init(struct xe_device *xe) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); struct xe_ttm_stolen_mgr *mgr; u64 stolen_size, io_size; - int err; + int ret; mgr = drmm_kzalloc(&xe->drm, sizeof(*mgr), GFP_KERNEL); if (!mgr) @@ -244,12 +232,12 @@ int xe_ttm_stolen_mgr_init(struct xe_device *xe) if (mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe)) io_size = stolen_size; - err = __xe_ttm_vram_mgr_init(xe, &mgr->base, XE_PL_STOLEN, stolen_size, - io_size, PAGE_SIZE); - if (err) { - drm_dbg_kms(&xe->drm, "Stolen mgr init failed: %i\n", err); - return err; - } + ret = ttm_range_man_init_nocheck(&xe->ttm, XE_PL_STOLEN, false, + stolen_size >> PAGE_SHIFT); + if (ret) + return ret; + + xe->mem.stolen_mgr = mgr; drm_dbg_kms(&xe->drm, "Initialized stolen memory support with %llu bytes\n", stolen_size); @@ -257,36 +245,32 @@ int xe_ttm_stolen_mgr_init(struct xe_device *xe) if (io_size) mgr->mapping = devm_ioremap_wc(&pdev->dev, mgr->io_base, io_size); - return 0; + return drmm_add_action_or_reset(&xe->drm, xe_ttm_stolen_mgr_fini, mgr); } u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset) { struct xe_device *xe = xe_bo_device(bo); - struct ttm_resource_manager *ttm_mgr = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); - struct xe_ttm_stolen_mgr *mgr = to_stolen_mgr(ttm_mgr); - struct xe_res_cursor cur; + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; XE_WARN_ON(!mgr->io_base); if (xe_ttm_stolen_cpu_access_needs_ggtt(xe)) return mgr->io_base + xe_bo_ggtt_addr(bo) + offset; - xe_res_first(bo->ttm.resource, offset, 4096, &cur); - return mgr->io_base + cur.start; + /* Range allocator: res->start is in pages. */ + return mgr->io_base + (bo->ttm.resource->start << PAGE_SHIFT) + offset; } static int __xe_ttm_stolen_io_mem_reserve_bar2(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr, struct ttm_resource *mem) { - struct xe_res_cursor cur; - if (!mgr->io_base) return -EIO; - xe_res_first(mem, 0, 4096, &cur); - mem->bus.offset = cur.start; + /* Range allocator always produces contiguous allocations. */ + mem->bus.offset = mem->start << PAGE_SHIFT; drm_WARN_ON(&xe->drm, !(mem->placement & TTM_PL_FLAG_CONTIGUOUS)); @@ -329,8 +313,7 @@ static int __xe_ttm_stolen_io_mem_reserve_stolen(struct xe_device *xe, int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem) { - struct ttm_resource_manager *ttm_mgr = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); - struct xe_ttm_stolen_mgr *mgr = ttm_mgr ? to_stolen_mgr(ttm_mgr) : NULL; + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; if (!mgr || !mgr->io_base) return -EIO; @@ -343,8 +326,5 @@ int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem) u64 xe_ttm_stolen_gpu_offset(struct xe_device *xe) { - struct xe_ttm_stolen_mgr *mgr = - to_stolen_mgr(ttm_manager_type(&xe->ttm, XE_PL_STOLEN)); - - return mgr->stolen_base; + return xe->mem.stolen_mgr->stolen_base; } diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h index 8e877d1e839b..0675106d535b 100644 --- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h +++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h @@ -12,6 +12,18 @@ struct ttm_resource; struct xe_bo; struct xe_device; +/** + * struct xe_ttm_stolen_mgr - Xe TTM stolen memory manager + */ +struct xe_ttm_stolen_mgr { + /** @io_base: PCI base offset for CPU I/O access */ + resource_size_t io_base; + /** @stolen_base: GPU base offset */ + resource_size_t stolen_base; + /** @mapping: I/O memory mapping for CPU access */ + void __iomem *mapping; +}; + int xe_ttm_stolen_mgr_init(struct xe_device *xe); int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem); bool xe_ttm_stolen_cpu_access_needs_ggtt(struct xe_device *xe); diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 6ba47996bc7c..9dad5cf5b50a 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -299,14 +299,13 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, u64 default_page_size) { struct ttm_resource_manager *man = &mgr->manager; + const char *name; int err; - if (mem_type != XE_PL_STOLEN) { - const char *name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; - man->cg = drmm_cgroup_register_region(&xe->drm, name, size); - if (IS_ERR(man->cg)) - return PTR_ERR(man->cg); - } + name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; + man->cg = drmm_cgroup_register_region(&xe->drm, name, size); + if (IS_ERR(man->cg)) + return PTR_ERR(man->cg); man->func = &xe_ttm_vram_mgr_func; mgr->mem_type = mem_type; From 09b399842907565a64e351fb22da790b4c673ffb Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Thu, 7 May 2026 14:00:15 -0700 Subject: [PATCH 09/36] drm/xe: Make decision to use Xe2-style blitter instructions a feature flag The blitter engines' MEM_COPY and MEM_SET instructions were added as part of the same hardware change that introduced service copy engines (i.e., BCS1-BCS8) which is why the driver checks for service copy engine presence when deciding whether to use these instructions or the older XY_* instructions. However when making this decision the driver should consider which engines are part of the hardware architecture, not which engines are present/usable on the current device. For graphics IP versions that architecturally include service copy engines (i.e., everything Xe2 and later, plus PVC's Xe_HPC) we should use MEM_SET and MEM_COPY even in if all of the service copy engines wind up getting fused off. I.e., we need to decide based on whether the platform's graphics descriptor contains these engines, rather than whether the usable engine mask contains them. This logic got broken when gt->info.__engine_mask was removed, although in practice that mistake has been harmless so far because there haven't been any hardware SKUs that fuse off all of the service copy engines yet. Replace the incorrect has_service_copy_support() function with a GT feature flag that tracks more accurately whether the new blitter instructions are usable. In addition to fixing incorrect logic if all service copies are fused off, the flag also makes it more obvious what the calling code is trying to do; previously it wasn't terribly obvious why "has service copy engines" was being used as the condition for using different instructions on all copy engine types. The new feature flag is named 'has_xe2_blt_instructions' because we expect this flag to be set for all Xe2 and later platforms (i.e., everything officially supported by the Xe driver). Technically there's also one Xe1-era platform (PVC) that supports these engines/instructions and will set this flag, but this still seems to be the most clear and understandable name for the flag. Fixes: 61549a2ee594 ("drm/xe: Drop __engine_mask") Cc: Balasubramani Vivekanandan Reviewed-by: Balasubramani Vivekanandan Link: https://patch.msgid.link/20260507-xe2_copy-v1-1-26506381b821@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_gt_types.h | 7 +++++++ drivers/gpu/drm/xe/xe_migrate.c | 18 ++---------------- drivers/gpu/drm/xe/xe_pci.c | 9 +++++++++ 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index 7351aadd238e..e5588c88800a 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -144,6 +144,13 @@ struct xe_gt { u8 id; /** @info.has_indirect_ring_state: GT has indirect ring state support */ u8 has_indirect_ring_state:1; + /** + * @info.has_xe2_blt_instructions: GT supports Xe2-style MEM_SET + * and MEM_COPY blitter functionality. Note that despite the + * name, some Xe1 platforms may also support this "Xe2-style" + * feature. + */ + u8 has_xe2_blt_instructions:1; /** * @info.num_geometry_xecore_fuse_regs: Number of 32b-bit fuse * registers the geometry XeCore mask spans. diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index a87fbc1e9fb1..f3c2ef269ba8 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -1525,23 +1525,9 @@ static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, bb->len += len; } -static bool has_service_copy_support(struct xe_gt *gt) -{ - /* - * What we care about is whether the architecture was designed with - * service copy functionality (specifically the new MEM_SET / MEM_COPY - * instructions) so check the architectural engine list rather than the - * actual list since these instructions are usable on BCS0 even if - * all of the actual service copy engines (BCS1-BCS8) have been fused - * off. - */ - return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8, - XE_HW_ENGINE_BCS1); -} - static u32 emit_clear_cmd_len(struct xe_gt *gt) { - if (has_service_copy_support(gt)) + if (gt->info.has_xe2_blt_instructions) return PVC_MEM_SET_CMD_LEN_DW; else return XY_FAST_COLOR_BLT_DW; @@ -1550,7 +1536,7 @@ static u32 emit_clear_cmd_len(struct xe_gt *gt) static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u32 size, u32 pitch, bool is_vram) { - if (has_service_copy_support(gt)) + if (gt->info.has_xe2_blt_instructions) emit_clear_link_copy(gt, bb, src_ofs, size, pitch); else emit_clear_main_copy(gt, bb, src_ofs, size, pitch, diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 41435f84aeb2..12d3be7f9f6c 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -849,6 +849,15 @@ static struct xe_gt *alloc_primary_gt(struct xe_tile *tile, gt->info.num_geometry_xecore_fuse_regs = graphics_desc->num_geometry_xecore_fuse_regs; gt->info.num_compute_xecore_fuse_regs = graphics_desc->num_compute_xecore_fuse_regs; + /* + * Even if the service copy engines wind up being fused off, their + * presence in the IP descriptor indicates that the platform supports + * Xe2-style MEM_SET and MEM_COPY functionality. + */ + if (graphics_desc->hw_engine_mask & GENMASK(XE_HW_ENGINE_BCS8, + XE_HW_ENGINE_BCS1)) + gt->info.has_xe2_blt_instructions = true; + /* * Before media version 13, the media IP was part of the primary GT * so we need to add the media engines to the primary GT's engine list. From 00cf90125ba6f914dd36bf36af82156519ac03a8 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Fri, 8 May 2026 12:44:28 -0700 Subject: [PATCH 10/36] drm/xe/multi_queue: Remove redundant assignment in guc_exec_queue_run_job The 'killed_or_banned_or_wedged = true' assignment is redundant since the variable is never read after that point. Assisted-by: GitHub Copilot:claude-sonnet-4.6 Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260508194428.61819-5-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index b1222b42174c..104e65913dff 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -1216,10 +1216,8 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) if (xe_exec_queue_is_multi_queue_secondary(q)) { struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); - if (exec_queue_killed_or_banned_or_wedged(primary)) { - killed_or_banned_or_wedged = true; + if (exec_queue_killed_or_banned_or_wedged(primary)) goto run_job_out; - } if (!exec_queue_registered(primary)) register_exec_queue(primary, GUC_CONTEXT_NORMAL); From 5dd993ef9d5e70b05adac92b3fb2f079b0a3fe1a Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Fri, 8 May 2026 12:44:29 -0700 Subject: [PATCH 11/36] drm/xe/multi_queue: Refactor CGP_SYNC send path Factor the repeated CGP_SYNC action build-and-send sequence into a new helper guc_exec_queue_send_cgp_sync(). Drop the redundant guc parameter from __register_exec_queue_group() since it can be derived via exec_queue_to_guc(q). Remove xe_guc_exec_queue_group_add() which is now identical to the helper and replace its call site directly. No functional change. Assisted-by: GitHub Copilot:claude-sonnet-4.6 Signed-off-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260508194428.61819-6-niranjana.vishwanathapura@intel.com --- drivers/gpu/drm/xe/xe_guc_submit.c | 65 +++++++++++------------------- 1 file changed, 23 insertions(+), 42 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 104e65913dff..4171eff4e8ad 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -852,10 +852,27 @@ static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc, xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1); } -static void __register_exec_queue_group(struct xe_guc *guc, - struct xe_exec_queue *q, +static void guc_exec_queue_send_cgp_sync(struct xe_exec_queue *q) +{ +#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; + int len = 0; + + action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; + action[len++] = group->primary->guc->id; + + xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); +#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE + + xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); +} + +static void __register_exec_queue_group(struct xe_exec_queue *q, struct guc_ctxt_registration_info *info) { + struct xe_guc *guc = exec_queue_to_guc(q); #define MAX_MULTI_QUEUE_REG_SIZE (8) u32 action[MAX_MULTI_QUEUE_REG_SIZE]; int len = 0; @@ -880,29 +897,6 @@ static void __register_exec_queue_group(struct xe_guc *guc, xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); } -static void xe_guc_exec_queue_group_add(struct xe_guc *guc, - struct xe_exec_queue *q) -{ -#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) - u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; - int len = 0; - - xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_multi_queue_secondary(q)); - - action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; - action[len++] = q->multi_queue.group->primary->guc->id; - - xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); -#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE - - /* - * The above XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC do expect a - * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response - * from guc. - */ - xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); -} - static void __register_mlrc_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q, struct guc_ctxt_registration_info *info) @@ -1028,7 +1022,7 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) set_exec_queue_registered(q); trace_xe_exec_queue_register(q); if (xe_exec_queue_is_multi_queue_primary(q)) - __register_exec_queue_group(guc, q, &info); + __register_exec_queue_group(q, &info); else if (xe_exec_queue_is_parallel(q)) __register_mlrc_exec_queue(guc, q, &info); else if (!xe_exec_queue_is_multi_queue_secondary(q)) @@ -1038,7 +1032,7 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) init_policies(guc, q); if (xe_exec_queue_is_multi_queue_secondary(q)) - xe_guc_exec_queue_group_add(guc, q); + guc_exec_queue_send_cgp_sync(q); } static u32 wq_space_until_wrap(struct xe_exec_queue *q) @@ -1887,21 +1881,8 @@ static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sche { struct xe_exec_queue *q = msg->private_data; - if (guc_exec_queue_allowed_to_change_state(q)) { -#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_exec_queue_group *group = q->multi_queue.group; - u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; - int len = 0; - - action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; - action[len++] = group->primary->guc->id; - - xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); -#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE - - xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); - } + if (guc_exec_queue_allowed_to_change_state(q)) + guc_exec_queue_send_cgp_sync(q); kfree(msg); } From 71477b7e702c88f214866a3dc400812a5da59905 Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Wed, 15 Apr 2026 11:27:17 -0700 Subject: [PATCH 12/36] drm/xe/eustall: Return ENODEV from read if EU stall registers get reset If a reset (GT or engine) happens during EU stall data sampling, all the EU stall registers can get reset to 0. This will result in EU stall data buffers' read and write pointer register values to be out of sync with the cached values. This will result in read() returning invalid data. To prevent this, check the value of a EU stall base register. If it is zero, it indicates a reset may have happened that wiped the register to zero. If this happens, return ENODEV from read() upon which the user space should disable and enable EU stall data sampling or close the fd and open a new fd for a new EU stall data collection session. This patch has been tested by running two IGT tests simultaneously xe_eu_stall and xe_exec_reset. Reviewed-by: Ashutosh Dixit Acked-by: Felix Degrood Signed-off-by: Harish Chegondi Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/6935698d70b6c7c347ff8b138209d601030c2c9f.1776200094.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/xe_eu_stall.c | 40 +++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index dddcdd0bb7a3..297be3c42b20 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -44,6 +44,7 @@ struct per_xecore_buf { struct xe_eu_stall_data_stream { bool pollin; bool enabled; + bool reset_detected; int wait_num_reports; int sampling_rate_mult; wait_queue_head_t poll_wq; @@ -428,9 +429,20 @@ static bool eu_stall_data_buf_poll(struct xe_eu_stall_data_stream *stream) set_bit(xecore, stream->data_drop.mask); xecore_buf->write = write_ptr; } + /* If a GT or engine reset happens during EU stall sampling, + * all EU stall registers get reset to 0 and the cached values of + * the EU stall data buffers' read pointers are out of sync with + * the register values. This causes invalid data to be returned + * from read(). To prevent this, check the value of a EU stall base + * register. If it is zero, there has been a reset. + */ + if (unlikely(!xe_gt_mcr_unicast_read_any(gt, XEHPC_EUSTALL_BASE))) + stream->reset_detected = true; + + stream->pollin = min_data_present || stream->reset_detected; mutex_unlock(&stream->xecore_buf_lock); - return min_data_present; + return stream->pollin; } static void clear_dropped_eviction_line_bit(struct xe_gt *gt, u16 group, u16 instance) @@ -544,6 +556,15 @@ static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *st int ret = 0; mutex_lock(&stream->xecore_buf_lock); + /* If EU stall registers got reset due to a GT/engine reset, + * continuing with the read() will return invalid data to + * the user space. Just return -ENODEV instead. + */ + if (unlikely(stream->reset_detected)) { + xe_gt_dbg(gt, "EU stall base register has been reset\n"); + mutex_unlock(&stream->xecore_buf_lock); + return -ENODEV; + } if (bitmap_weight(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS)) { if (!stream->data_drop.reported_to_user) { stream->data_drop.reported_to_user = true; @@ -554,7 +575,6 @@ static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *st } stream->data_drop.reported_to_user = false; } - for_each_dss_steering(xecore, gt, group, instance) { ret = xe_eu_stall_data_buf_read(stream, buf, count, &total_size, gt, group, instance, xecore); @@ -609,7 +629,8 @@ static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf, * We don't want to block the next read() when there is data in the buffer * now, but couldn't be accommodated in the small user buffer. */ - stream->pollin = false; + if (!stream->reset_detected) + stream->pollin = false; return ret; } @@ -692,6 +713,7 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream) xecore_buf->write = write_ptr; xecore_buf->read = write_ptr; } + stream->reset_detected = false; stream->data_drop.reported_to_user = false; bitmap_zero(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS); @@ -717,13 +739,13 @@ static void eu_stall_data_buf_poll_work_fn(struct work_struct *work) container_of(work, typeof(*stream), buf_poll_work.work); struct xe_gt *gt = stream->gt; - if (eu_stall_data_buf_poll(stream)) { - stream->pollin = true; + if (eu_stall_data_buf_poll(stream)) wake_up(&stream->poll_wq); - } - queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq, - &stream->buf_poll_work, - msecs_to_jiffies(POLL_PERIOD_MS)); + + if (!stream->reset_detected) + queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq, + &stream->buf_poll_work, + msecs_to_jiffies(POLL_PERIOD_MS)); } static int xe_eu_stall_stream_init(struct xe_eu_stall_data_stream *stream, From 404734f1fef84f8bec5008fd48a16c49b76e0367 Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Thu, 7 May 2026 09:20:18 -0700 Subject: [PATCH 13/36] drm/xe/lrc: Use 64 bit ctx timestamp in the LRC snapshot Use the 64 bit value when available for the context timestamp in the LRC snapshot. Suggested-by: Matthew Brost Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260507162016.3888309-14-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/xe_lrc.c | 4 ++-- drivers/gpu/drm/xe/xe_lrc.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index fdfe2ed5f683..570ff9f55849 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -2462,7 +2462,7 @@ struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc) snapshot->replay_offset = 0; snapshot->replay_size = lrc->replay_size; snapshot->lrc_snapshot = NULL; - snapshot->ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(lrc)); + snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc); snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); return snapshot; } @@ -2515,7 +2515,7 @@ void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start); drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno); drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); - drm_printf(p, "\tTimestamp: 0x%08x\n", snapshot->ctx_timestamp); + drm_printf(p, "\tTimestamp: 0x%016llx\n", snapshot->ctx_timestamp); drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); if (!snapshot->lrc_snapshot) diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index 5440663183f6..bef7af9a578b 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -37,7 +37,7 @@ struct xe_lrc_snapshot { } tail; u32 start_seqno; u32 seqno; - u32 ctx_timestamp; + u64 ctx_timestamp; u32 ctx_job_timestamp; }; From e98cf36bf5b5043c5172041e2d24d40139c7f0e9 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 7 May 2026 09:20:19 -0700 Subject: [PATCH 14/36] drm/xe: Add timestamp_ms to LRC snapshot Add a timestamp in milliseconds to the LRC snapshot to make it easier to reason about how long the LRC has been running and the average duration of each job. Signed-off-by: Matthew Brost Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260507162016.3888309-15-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/xe_lrc.c | 4 ++++ drivers/gpu/drm/xe/xe_lrc.h | 1 + 2 files changed, 5 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 570ff9f55849..66682a678cc2 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -23,6 +23,7 @@ #include "xe_drm_client.h" #include "xe_exec_queue_types.h" #include "xe_gt.h" +#include "xe_gt_clock.h" #include "xe_gt_printk.h" #include "xe_hw_fence.h" #include "xe_map.h" @@ -2463,6 +2464,8 @@ struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc) snapshot->replay_size = lrc->replay_size; snapshot->lrc_snapshot = NULL; snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc); + snapshot->ctx_timestamp_ms = + xe_gt_clock_interval_to_ms(lrc->gt, xe_lrc_ctx_timestamp(lrc)); snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); return snapshot; } @@ -2516,6 +2519,7 @@ void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno); drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); drm_printf(p, "\tTimestamp: 0x%016llx\n", snapshot->ctx_timestamp); + drm_printf(p, "\tTimestamp ms: %llu\n", snapshot->ctx_timestamp_ms); drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); if (!snapshot->lrc_snapshot) diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index bef7af9a578b..d280e6572398 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -38,6 +38,7 @@ struct xe_lrc_snapshot { u32 start_seqno; u32 seqno; u64 ctx_timestamp; + u64 ctx_timestamp_ms; u32 ctx_job_timestamp; }; From d243ef6a39c6e712694fac14c337710941f5f260 Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Thu, 7 May 2026 09:20:20 -0700 Subject: [PATCH 15/36] drm/xe/lrc: Refactor xe_lrc_timestamp to simplify logic Use a context_active() helper and simplify the timestamp logic. Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260507162016.3888309-16-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/xe_lrc.c | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 66682a678cc2..b31525bd2a4c 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -2585,6 +2585,11 @@ static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) return 0; } +static bool context_active(struct xe_lrc *lrc) +{ + return xe_lrc_ctx_timestamp(lrc) == CONTEXT_ACTIVE; +} + /** * xe_lrc_timestamp() - Current ctx timestamp * @lrc: Pointer to the lrc. @@ -2597,33 +2602,23 @@ static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) */ u64 xe_lrc_timestamp(struct xe_lrc *lrc) { - u64 lrc_ts, reg_ts, new_ts = lrc->ctx_timestamp; - u32 engine_id; + u64 reg_ts, new_ts = lrc->ctx_timestamp; - lrc_ts = xe_lrc_ctx_timestamp(lrc); /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ - if (IS_SRIOV_VF(lrc_to_xe(lrc))) { - new_ts = lrc_ts; - goto done; - } + if (IS_SRIOV_VF(lrc_to_xe(lrc))) + return xe_lrc_ctx_timestamp(lrc); - if (lrc_ts == CONTEXT_ACTIVE) { - engine_id = xe_lrc_engine_id(lrc); - if (!get_ctx_timestamp(lrc, engine_id, ®_ts)) - new_ts = reg_ts; - - /* read lrc again to ensure context is still active */ - lrc_ts = xe_lrc_ctx_timestamp(lrc); - } + if (context_active(lrc) && + !get_ctx_timestamp(lrc, xe_lrc_engine_id(lrc), ®_ts)) + new_ts = reg_ts; /* - * If context switched out, just use the lrc_ts. Note that this needs to - * be a separate if condition. + * If context swicthed out while we were here, just return the latest + * LRC CTX TIMESTAMP value. */ - if (lrc_ts != CONTEXT_ACTIVE) - new_ts = lrc_ts; + if (!context_active(lrc)) + return xe_lrc_ctx_timestamp(lrc); -done: return new_ts; } From 11ea979e19001a772117431213245cd06d810354 Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Thu, 7 May 2026 09:20:21 -0700 Subject: [PATCH 16/36] drm/xe/multi_queue: Refactor check for multi queue support for engine class xe exec queue code is using a check to see if a class of engines support multi queue. This check is also needed by other code, so move it to xe_gt and export it for others. Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260507162016.3888309-17-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.c | 7 +------ drivers/gpu/drm/xe/xe_gt.h | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 071b8c41df43..62a75c8fe72f 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -852,11 +852,6 @@ static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue * return 0; } -static inline bool xe_exec_queue_supports_multi_queue(struct xe_exec_queue *q) -{ - return q->gt->info.multi_queue_engine_class_mask & BIT(q->class); -} - static int xe_exec_queue_group_validate(struct xe_device *xe, struct xe_exec_queue *q, u32 primary_id) { @@ -931,7 +926,7 @@ static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queu static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q, u64 value) { - if (XE_IOCTL_DBG(xe, !xe_exec_queue_supports_multi_queue(q))) + if (XE_IOCTL_DBG(xe, !xe_gt_supports_multi_queue(q->gt, q->class))) return -ENODEV; if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe))) diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h index de7e47763411..4150aa594f05 100644 --- a/drivers/gpu/drm/xe/xe_gt.h +++ b/drivers/gpu/drm/xe/xe_gt.h @@ -155,4 +155,19 @@ static inline bool xe_gt_recovery_pending(struct xe_gt *gt) xe_gt_sriov_vf_recovery_pending(gt); } +/** + * xe_gt_supports_multi_queue() - Check if gt supports multi queue for the + * specified engine class. + * + * @gt: the GT object + * @class: hwe class type + * + * Return: true if the hw engine class supports multi queue, else false + */ +static inline bool xe_gt_supports_multi_queue(const struct xe_gt *gt, + enum xe_engine_class class) +{ + return gt->info.multi_queue_engine_class_mask & BIT(class); +} + #endif From 0612f582dc23c386e5ee9312085a3eacb87be256 Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Thu, 7 May 2026 09:20:22 -0700 Subject: [PATCH 17/36] drm/xe/multi_queue: Store primary LRC and position info in LRC For an LRC belonging to the secondary queue, in order to check if its context group is active, we need to check the LRC of the primary queue. In addition to that we want to compare the secondary queue position to CSMQDEBUG register to check if the queue itself is active. To do so, store primary LRC and position information in the LRC. A note on references involved: - In general the Queue takes a ref on its LRC. - In addition, for multi-queue, a. Primary Queue takes a ref for each Secondary LRC. b. Each Secondary Queue takes a ref to the Primary Queue In the current patch, each LRC in the queue group is storing a pointer to primary LRC. Both primary and secondary LRCs are freed only when primary queue is destroyed. At this time, all secondary queues are already destroyed, so there is no one using secondary LRCs. We should be good without taking any additional references. Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260507162016.3888309-18-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/xe_exec_queue.c | 7 ++++++- drivers/gpu/drm/xe/xe_lrc.h | 5 +++++ drivers/gpu/drm/xe/xe_lrc_types.h | 8 ++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 62a75c8fe72f..1b5ca3ce578a 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -275,8 +275,12 @@ static void xe_exec_queue_set_lrc(struct xe_exec_queue *q, struct xe_lrc *lrc, u { xe_assert(gt_to_xe(q->gt), idx < q->width); - scoped_guard(spinlock, &q->lrc_lookup_lock) + scoped_guard(spinlock, &q->lrc_lookup_lock) { q->lrc[idx] = lrc; + if (xe_exec_queue_is_multi_queue(q)) + q->lrc[idx]->multi_queue.primary_lrc = + q->multi_queue.group->primary->lrc[0]; + } } /** @@ -907,6 +911,7 @@ static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q } q->multi_queue.pos = pos; + q->lrc[0]->multi_queue.pos = pos; return 0; } diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index d280e6572398..557dce004d48 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -91,6 +91,11 @@ static inline size_t xe_lrc_ring_size(void) return SZ_16K; } +static inline bool xe_lrc_is_multi_queue(struct xe_lrc *lrc) +{ + return lrc->multi_queue.primary_lrc; +} + size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class); size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class); u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc); diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h index 5a718f759ed6..0a5c13ec2ad7 100644 --- a/drivers/gpu/drm/xe/xe_lrc_types.h +++ b/drivers/gpu/drm/xe/xe_lrc_types.h @@ -63,6 +63,14 @@ struct xe_lrc { /** @ctx_timestamp: readout value of CTX_TIMESTAMP on last update */ u64 ctx_timestamp; + + /** @multi_queue: Multi queue LRC related information */ + struct { + /** @multi_queue.primary_lrc: Primary lrc of this multi-queue group*/ + struct xe_lrc *primary_lrc; + /** @multi_queue.pos: Position of LRC within the multi-queue group */ + u8 pos; + } multi_queue; }; struct xe_lrc_snapshot; From be50e7dbd1cd3e752221f41f548ed1c77760dcee Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Thu, 7 May 2026 09:20:23 -0700 Subject: [PATCH 18/36] drm/xe/multi_queue: Add helpers to access CS QUEUE TIMESTAMP from lrc In secondary queue LRCs, the QUEUE TIMESTAMP register is saved and restored allowing us to view the individual queue run times. Add helpers to read this value from the LRC. BSpec: 73988 Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260507162016.3888309-19-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/regs/xe_lrc_layout.h | 3 ++ drivers/gpu/drm/xe/xe_lrc.c | 57 +++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_lrc.h | 2 + drivers/gpu/drm/xe/xe_lrc_types.h | 3 ++ 4 files changed, 65 insertions(+) diff --git a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h index b5eff383902c..4ab86fc369fd 100644 --- a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h +++ b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h @@ -34,6 +34,9 @@ #define CTX_CS_INT_VEC_REG 0x5a #define CTX_CS_INT_VEC_DATA (CTX_CS_INT_VEC_REG + 1) +#define CTX_QUEUE_TIMESTAMP (0xd0 + 1) +#define CTX_QUEUE_TIMESTAMP_UDW (0xd2 + 1) + #define INDIRECT_CTX_RING_HEAD (0x02 + 1) #define INDIRECT_CTX_RING_TAIL (0x04 + 1) #define INDIRECT_CTX_RING_START (0x06 + 1) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index b31525bd2a4c..0dc50627e1b7 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -777,6 +777,16 @@ static u32 __xe_lrc_ctx_timestamp_udw_offset(struct xe_lrc *lrc) return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP_UDW * sizeof(u32); } +static u32 __xe_lrc_queue_timestamp_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP * sizeof(u32); +} + +static u32 __xe_lrc_queue_timestamp_udw_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP_UDW * sizeof(u32); +} + static inline u32 __xe_lrc_indirect_ring_offset(struct xe_lrc *lrc) { u32 offset = xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE - @@ -826,6 +836,8 @@ DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo) DECL_MAP_ADDR_HELPERS(parallel, lrc->bo) DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo) DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo) +DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo) +DECL_MAP_ADDR_HELPERS(queue_timestamp_udw, lrc->bo) #undef DECL_MAP_ADDR_HELPERS @@ -874,6 +886,29 @@ static u64 xe_lrc_ctx_timestamp(struct xe_lrc *lrc) return (u64)udw << 32 | ldw; } +/** + * xe_lrc_queue_timestamp() - Read queue timestamp value + * @lrc: Pointer to the lrc. + * + * Returns: queue timestamp value + */ +static u64 xe_lrc_queue_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + u32 ldw, udw = 0; + + xe_assert(xe, xe_lrc_is_multi_queue(lrc)); + + map = __xe_lrc_queue_timestamp_map(lrc); + ldw = xe_map_read32(xe, &map); + + map = __xe_lrc_queue_timestamp_udw_map(lrc); + udw = xe_map_read32(xe, &map); + + return (u64)udw << 32 | ldw; +} + /** * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address * @lrc: Pointer to the lrc. @@ -1538,6 +1573,18 @@ static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct if (lrc_to_xe(lrc)->info.has_64bit_timestamp) xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP_UDW, 0); + /* + * Note: It's possible that this LRC may belong to an exec_queue that is + * not part of a multi-queue group. That said, it doesn't hurt to set + * this field anyways since any class that supports multi-queue will + * have these LRC fields defined. + */ + if (xe_gt_supports_multi_queue(gt, hwe->class)) { + lrc->queue_timestamp = 0; + xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP, 0); + xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP_UDW, 0); + } + if (xe->info.has_asid && vm) xe_lrc_write_ctx_reg(lrc, CTX_ASID, vm->usm.asid); @@ -2466,6 +2513,14 @@ struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc) snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc); snapshot->ctx_timestamp_ms = xe_gt_clock_interval_to_ms(lrc->gt, xe_lrc_ctx_timestamp(lrc)); + if (xe_lrc_is_multi_queue(lrc)) { + snapshot->queue_timestamp = xe_lrc_queue_timestamp(lrc); + snapshot->queue_timestamp_ms = + xe_gt_clock_interval_to_ms(lrc->gt, snapshot->queue_timestamp); + } else { + snapshot->queue_timestamp = 0; + snapshot->queue_timestamp_ms = 0; + } snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); return snapshot; } @@ -2520,6 +2575,8 @@ void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); drm_printf(p, "\tTimestamp: 0x%016llx\n", snapshot->ctx_timestamp); drm_printf(p, "\tTimestamp ms: %llu\n", snapshot->ctx_timestamp_ms); + drm_printf(p, "\tQueue Timestamp: 0x%016llx\n", snapshot->queue_timestamp); + drm_printf(p, "\tQueue Timestamp ms: %llu\n", snapshot->queue_timestamp_ms); drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); if (!snapshot->lrc_snapshot) diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index 557dce004d48..0a3a611391ee 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -39,6 +39,8 @@ struct xe_lrc_snapshot { u32 seqno; u64 ctx_timestamp; u64 ctx_timestamp_ms; + u64 queue_timestamp; + u64 queue_timestamp_ms; u32 ctx_job_timestamp; }; diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h index 0a5c13ec2ad7..53ef48feebfc 100644 --- a/drivers/gpu/drm/xe/xe_lrc_types.h +++ b/drivers/gpu/drm/xe/xe_lrc_types.h @@ -64,6 +64,9 @@ struct xe_lrc { /** @ctx_timestamp: readout value of CTX_TIMESTAMP on last update */ u64 ctx_timestamp; + /** @queue_timestamp: value of QUEUE_TIMESTAMP on last update */ + u64 queue_timestamp; + /** @multi_queue: Multi queue LRC related information */ struct { /** @multi_queue.primary_lrc: Primary lrc of this multi-queue group*/ From e3a9c46fc3a29b8d7f62a7bd73bf20975037dc82 Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Thu, 7 May 2026 09:20:24 -0700 Subject: [PATCH 19/36] drm/xe/lrc: Refactor out engine id to hwe conversion We need to define more helpers that read engine ID specific register, so move that logic outside of get_ctx_timestamp(). Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260507162016.3888309-20-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/xe_lrc.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 0dc50627e1b7..0fe9738cb360 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -2617,17 +2617,27 @@ void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot) kfree(snapshot); } -static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) +static struct xe_hw_engine *engine_id_to_hwe(struct xe_gt *gt, u32 engine_id) { u16 class = REG_FIELD_GET(ENGINE_CLASS_ID, engine_id); u16 instance = REG_FIELD_GET(ENGINE_INSTANCE_ID, engine_id); + struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, class, instance, false); + + if (xe_gt_WARN_ONCE(gt, !hwe || xe_hw_engine_is_reserved(hwe), + "Unexpected engine class:instance %d:%d for utilization\n", + class, instance)) + return NULL; + + return hwe; +} + +static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) +{ struct xe_hw_engine *hwe; u64 val; - hwe = xe_gt_hw_engine(lrc->gt, class, instance, false); - if (xe_gt_WARN_ONCE(lrc->gt, !hwe || xe_hw_engine_is_reserved(hwe), - "Unexpected engine class:instance %d:%d for context utilization\n", - class, instance)) + hwe = engine_id_to_hwe(lrc->gt, engine_id); + if (!hwe) return -1; if (lrc_to_xe(lrc)->info.has_64bit_timestamp) From bccf190ea78c042f0ae452fe4f0b7ab32963164b Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Thu, 7 May 2026 09:20:25 -0700 Subject: [PATCH 20/36] drm/xe/multi_queue: Capture queue run times for active queues If a queue is currently active on the CS, query the QUEUE TIMESTAMP register to get an up to date value of the runtime. To do so, ensure that the primary queue is active and then check if the secondary queue is executing on the CS. Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260507162016.3888309-21-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/regs/xe_engine_regs.h | 4 + drivers/gpu/drm/xe/xe_lrc.c | 120 ++++++++++++++++++----- 2 files changed, 101 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h index c4c879a9e555..94033982e694 100644 --- a/drivers/gpu/drm/xe/regs/xe_engine_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h @@ -170,6 +170,10 @@ #define GFX_DISABLE_LEGACY_MODE REG_BIT(3) #define RING_CSMQDEBUG(base) XE_REG((base) + 0x2b0) +#define CURRENT_ACTIVE_QUEUE_ID_MASK REG_GENMASK(7, 0) + +#define RING_QUEUE_TIMESTAMP(base) XE_REG((base) + 0x4c0) +#define RING_QUEUE_TIMESTAMP_UDW(base) XE_REG((base) + 0x4c0 + 4) #define RING_TIMESTAMP(base) XE_REG((base) + 0x358) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 0fe9738cb360..2e7316c2207b 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -21,6 +21,7 @@ #include "xe_configfs.h" #include "xe_device.h" #include "xe_drm_client.h" +#include "xe_exec_queue.h" #include "xe_exec_queue_types.h" #include "xe_gt.h" #include "xe_gt_clock.h" @@ -2652,22 +2653,69 @@ static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) return 0; } +static u64 get_queue_timestamp(struct xe_hw_engine *hwe) +{ + return xe_mmio_read64_2x32(&hwe->gt->mmio, + RING_QUEUE_TIMESTAMP(hwe->mmio_base)); +} + +static u32 get_multi_queue_active_queue_id(struct xe_hw_engine *hwe) +{ + u32 val = xe_mmio_read32(&hwe->gt->mmio, + RING_CSMQDEBUG(hwe->mmio_base)); + + return REG_FIELD_GET(CURRENT_ACTIVE_QUEUE_ID_MASK, val); +} + static bool context_active(struct xe_lrc *lrc) { return xe_lrc_ctx_timestamp(lrc) == CONTEXT_ACTIVE; } -/** - * xe_lrc_timestamp() - Current ctx timestamp - * @lrc: Pointer to the lrc. - * - * Return latest ctx timestamp. With support for active contexts, the - * calculation may be slightly racy, so follow a read-again logic to ensure that - * the context is still active before returning the right timestamp. - * - * Returns: New ctx timestamp value - */ -u64 xe_lrc_timestamp(struct xe_lrc *lrc) +static u64 xe_lrc_multi_queue_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct xe_lrc *primary_lrc = lrc->multi_queue.primary_lrc; + struct xe_hw_engine *hwe; + u64 reg_queue_ts = lrc->queue_timestamp; + + if (IS_SRIOV_VF(xe)) + return xe_lrc_queue_timestamp(lrc); + + xe_assert(xe, primary_lrc); + + /* WA BB populates CONTEXT_ACTIVE cookie for primary context only */ + if (!context_active(primary_lrc)) + return xe_lrc_queue_timestamp(lrc); + + /* WA BB populates engine id in PPHWSP of primary context only */ + hwe = engine_id_to_hwe(primary_lrc->gt, xe_lrc_engine_id(primary_lrc)); + if (!hwe) + return xe_lrc_queue_timestamp(lrc); + + if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos) + return xe_lrc_queue_timestamp(lrc); + + /* queue is active, so store the queue timestamp register */ + reg_queue_ts = get_queue_timestamp(hwe); + + /* double check queue and primary queue are both still active */ + if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos || + !context_active(primary_lrc)) + return xe_lrc_queue_timestamp(lrc); + + return reg_queue_ts; +} + +static u64 xe_lrc_update_multi_queue_timestamp(struct xe_lrc *lrc, u64 *old_ts) +{ + *old_ts = lrc->queue_timestamp; + lrc->queue_timestamp = xe_lrc_multi_queue_timestamp(lrc); + + return lrc->queue_timestamp; +} + +static u64 xe_lrc_context_timestamp(struct xe_lrc *lrc) { u64 reg_ts, new_ts = lrc->ctx_timestamp; @@ -2689,26 +2737,52 @@ u64 xe_lrc_timestamp(struct xe_lrc *lrc) return new_ts; } -/** - * xe_lrc_update_timestamp() - Update ctx timestamp - * @lrc: Pointer to the lrc. - * @old_ts: Old timestamp value - * - * Populate @old_ts current saved ctx timestamp, read new ctx timestamp and - * update saved value. - * - * Returns: New ctx timestamp value - */ -u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts) +static u64 xe_lrc_update_context_timestamp(struct xe_lrc *lrc, u64 *old_ts) { *old_ts = lrc->ctx_timestamp; - lrc->ctx_timestamp = xe_lrc_timestamp(lrc); + lrc->ctx_timestamp = xe_lrc_context_timestamp(lrc); trace_xe_lrc_update_timestamp(lrc, *old_ts); return lrc->ctx_timestamp; } +/** + * xe_lrc_timestamp() - Current lrc timestamp + * @lrc: Pointer to the lrc. + * + * Return latest lrc timestamp. With support for active contexts/queues, the + * calculation may be slightly racy, so follow a read-again logic to ensure that + * the context/queue is still active before returning the right timestamp. + * + * Returns: New lrc timestamp value + */ +u64 xe_lrc_timestamp(struct xe_lrc *lrc) +{ + if (xe_lrc_is_multi_queue(lrc)) + return xe_lrc_multi_queue_timestamp(lrc); + else + return xe_lrc_context_timestamp(lrc); +} + +/** + * xe_lrc_update_timestamp() - Update lrc timestamp + * @lrc: Pointer to the lrc. + * @old_ts: Old timestamp value + * + * Populate @old_ts with current saved lrc timestamp, read new lrc timestamp and + * update saved value. + * + * Returns: New lrc timestamp value + */ +u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts) +{ + if (xe_lrc_is_multi_queue(lrc)) + return xe_lrc_update_multi_queue_timestamp(lrc, old_ts); + else + return xe_lrc_update_context_timestamp(lrc, old_ts); +} + /** * xe_lrc_ring_is_idle() - LRC is idle * @lrc: Pointer to the lrc. From 1f30396de0c95d10859bce34f56d8b5b7a7d4bb8 Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Thu, 7 May 2026 09:20:26 -0700 Subject: [PATCH 21/36] drm/xe/multi_queue: Add trace event for the multi queue timestamp Add a trace event for multi queue timestamp capture. Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260507162016.3888309-22-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/xe_lrc.c | 2 ++ drivers/gpu/drm/xe/xe_trace_lrc.h | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 2e7316c2207b..a4292a11391d 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -2712,6 +2712,8 @@ static u64 xe_lrc_update_multi_queue_timestamp(struct xe_lrc *lrc, u64 *old_ts) *old_ts = lrc->queue_timestamp; lrc->queue_timestamp = xe_lrc_multi_queue_timestamp(lrc); + trace_xe_lrc_update_queue_timestamp(lrc, *old_ts); + return lrc->queue_timestamp; } diff --git a/drivers/gpu/drm/xe/xe_trace_lrc.h b/drivers/gpu/drm/xe/xe_trace_lrc.h index d525cbee1e34..5c4cfa0c1fe9 100644 --- a/drivers/gpu/drm/xe/xe_trace_lrc.h +++ b/drivers/gpu/drm/xe/xe_trace_lrc.h @@ -12,6 +12,7 @@ #include #include +#include "xe_exec_queue_types.h" #include "xe_gt_types.h" #include "xe_lrc.h" #include "xe_lrc_types.h" @@ -42,6 +43,32 @@ TRACE_EVENT(xe_lrc_update_timestamp, __get_str(device_id)) ); +TRACE_EVENT(xe_lrc_update_queue_timestamp, + TP_PROTO(struct xe_lrc *lrc, uint64_t old), + TP_ARGS(lrc, old), + TP_STRUCT__entry( + __field(struct xe_lrc *, lrc) + __field(struct xe_lrc *, primary_lrc) + __field(u64, old) + __field(u64, new) + __string(name, lrc->fence_ctx.name) + __string(device_id, __dev_name_lrc(lrc)) + ), + + TP_fast_assign( + __entry->lrc = lrc; + __entry->primary_lrc = lrc->multi_queue.primary_lrc; + __entry->old = old; + __entry->new = lrc->queue_timestamp; + __assign_str(name); + __assign_str(device_id); + ), + TP_printk("lrc=%p primary_lrc=%p lrc->name=%s old=%llu new=%llu device_id:%s", + __entry->lrc, __entry->primary_lrc, __get_str(name), + __entry->old, __entry->new, + __get_str(device_id)) +); + #endif /* This part must be outside protection */ From 1445cb708ee7f01989375098086e31fe77c60a02 Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Thu, 7 May 2026 09:20:27 -0700 Subject: [PATCH 22/36] drm/xe/multi_queue: Use QUEUE_TIMESTAMP as job timestamp for multi-queue Each queue in a multi queue group has a dedicated timestamp counter. Use this QUEUE TIMESTAMP register to capture the start timestamp for the job. Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260507162016.3888309-23-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/xe_ring_ops.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index cfeb4fc7d217..39a670e91ba7 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -269,8 +269,12 @@ static u32 get_ppgtt_flag(struct xe_sched_job *job) static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc, u32 *dw, int i) { + const struct xe_reg reg = xe_lrc_is_multi_queue(lrc) ? + RING_QUEUE_TIMESTAMP(0) : + RING_CTX_TIMESTAMP(0); + dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; - dw[i++] = RING_CTX_TIMESTAMP(0).addr; + dw[i++] = reg.addr; dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); dw[i++] = 0; @@ -281,7 +285,7 @@ static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc, if (IS_SRIOV_VF(xe)) { dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; - dw[i++] = RING_CTX_TIMESTAMP(0).addr; + dw[i++] = reg.addr; dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc); dw[i++] = 0; } From b619bbcda1389cc54c2a5c5a7561c3ca3b439c9c Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Thu, 7 May 2026 09:20:28 -0700 Subject: [PATCH 23/36] drm/xe/multi_queue: Whitelist QUEUE_TIMESTAMP register In a multi-queue use case, when a job is running on the secondary queue, the CTX_TIMESTAMP does not reflect the queues run ticks. Instead, we use the QUEUE TIMESTAMP to check how long the job ran. For user space to see the run ticks for a secondary queue, whitelist the QUEUE_TIMESTAMP register. Compute PR: https://github.com/intel/compute-runtime/pull/923 Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260507162016.3888309-24-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/xe/xe_reg_whitelist.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c index 8cc313182968..fb65940848d7 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.c +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c @@ -9,6 +9,7 @@ #include "regs/xe_gt_regs.h" #include "regs/xe_oa_regs.h" #include "xe_device.h" +#include "xe_gt.h" #include "xe_gt_types.h" #include "xe_gt_printk.h" #include "xe_platform_types.h" @@ -33,6 +34,13 @@ static bool match_has_mert(const struct xe_device *xe, return xe_device_has_mert((struct xe_device *)xe); } +static bool match_multi_queue_class(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_gt_supports_multi_queue(gt, hwe->class); +} + static const struct xe_rtp_entry_sr register_whitelist[] = { { XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), @@ -54,6 +62,12 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { RING_FORCE_TO_NONPRIV_ACCESS_RD, XE_RTP_ACTION_FLAG(ENGINE_BASE))) }, + { XE_RTP_NAME("allow_read_queue_timestamp"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3500, 3511), FUNC(match_multi_queue_class)), + XE_RTP_ACTIONS(WHITELIST(RING_QUEUE_TIMESTAMP(0), + RING_FORCE_TO_NONPRIV_ACCESS_RD, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, { XE_RTP_NAME("16014440446"), XE_RTP_RULES(PLATFORM(PVC)), XE_RTP_ACTIONS(WHITELIST(XE_REG(0x4400), From af1f2ad0c59fe4e2f924c526f66e968289d77971 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Fri, 8 May 2026 11:26:36 +0100 Subject: [PATCH 24/36] drm/xe/dma-buf: handle empty bo and UAF races MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There look to be some nasty races here when triggering the invalidate_mappings hook: 1) We do xe_bo_alloc() followed by the attach, before the actual full bo init step in xe_dma_buf_init_obj(). However the bo is visible on the attachments list after the attach. This is bad since exporter driver, say amdgpu, can at any time call back into our invalidate_mappings hook, with an empty/bogus bo, leading to potential bugs/crashes. 2) Similar to 1) but here we get a UAF, when the invalidate_mappings hook is triggered. For example, we get as far as xe_bo_init_locked() but this fails in some way. But here the bo will be freed on error, but we still have it attached from dma-buf pov, so if the invalidate_mappings is now triggered then the bo we access is gone and we trigger UAF and more bugs/crashes. To fix this, move the attach step until after we actually have a fully set up buffer object. Note that the bo is not published to userspace until later, so not sure what the comment "Don't publish the bo until we have a valid attachment", is referring to. We have at least two different customers reporting hitting a NULL ptr deref in evict_flags when importing something from amdgpu, followed by triggering the evict flow. Hit rate is also pretty low, which would hint at some kind of race, so something like 1) or 2) might explain this. v2: - Shuffle the order of the ops slightly (no functional change) - Improve the comment to better explain the ordering (Matt B) Assisted-by: Gemini:gemini-3 #debug Link: https://gitlab.freedesktop.org/drm/xe/kernel/-/work_items/7903 Link: https://gitlab.freedesktop.org/drm/xe/kernel/-/work_items/4055 Fixes: dd08ebf6c352 ("drm/xe: Introduce a new DRM driver for Intel GPUs") Signed-off-by: Matthew Auld Cc: Thomas Hellström Cc: Matthew Brost Cc: # v6.8+ Reviewed-by: Matthew Brost Acked-by: Thomas Hellström Link: https://patch.msgid.link/20260508102635.149172-3-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_dma_buf.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index 855d32ba314d..f7b47e9d1a4c 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -377,15 +377,25 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, } } - /* - * Don't publish the bo until we have a valid attachment, and a - * valid attachment needs the bo address. So pre-create a bo before - * creating the attachment and publish. - */ bo = xe_bo_alloc(); if (IS_ERR(bo)) return ERR_CAST(bo); + /* + * xe_dma_buf_init_obj() takes ownership of the raw bo, so do not touch + * on fail, since it will already take care of cleanup. On success we + * still need to drop the ref, if something later fails. + * + * In addition this needs to happen before the attach, since + * it will create a new attachment for this, and add it to the list of + * attachments, at which point it is globally visible, and at any point + * the export side can call into on invalidate_mappings callback, which + * require a working object. + */ + obj = xe_dma_buf_init_obj(dev, bo, dma_buf); + if (IS_ERR(obj)) + return obj; + attach_ops = &xe_dma_buf_attach_ops; #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) if (test) @@ -398,21 +408,12 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, goto out_err; } - /* - * xe_dma_buf_init_obj() takes ownership of bo on both success - * and failure, so we must not touch bo after this call. - */ - obj = xe_dma_buf_init_obj(dev, bo, dma_buf); - if (IS_ERR(obj)) { - dma_buf_detach(dma_buf, attach); - return obj; - } get_dma_buf(dma_buf); obj->import_attach = attach; return obj; out_err: - xe_bo_free(bo); + xe_bo_put(bo); return obj; } From 479669418253e0f27f8cf5db01a731352ea592e7 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Fri, 8 May 2026 11:26:37 +0100 Subject: [PATCH 25/36] drm/xe/dma-buf: fix UAF with retry loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retry doesn't work here, since bo will be freed on error, leading to UAF. However, now that we do the alloc & init before the attach, we can now combine this as one unit and have the init do the alloc for us. This should make the retry safe. Reported by Sashiko. v2: Fix up the error unwind (CI) Closes: https://sashiko.dev/#/patchset/20260506184332.86743-2-matthew.auld%40intel.com Fixes: eb289a5f6cc6 ("drm/xe: Convert xe_dma_buf.c for exhaustive eviction") Signed-off-by: Matthew Auld Cc: Thomas Hellström Cc: Matthew Brost Cc: # v6.18+ Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20260508102635.149172-4-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_dma_buf.c | 49 ++++++++------------------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index f7b47e9d1a4c..8a920e58245c 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -278,16 +278,8 @@ struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags) return ERR_PTR(ret); } -/* - * Takes ownership of @storage: on success it is transferred to the returned - * drm_gem_object; on failure it is freed before returning the error. - * This matches the contract of xe_bo_init_locked() which frees @storage on - * its error paths, so callers need not (and must not) free @storage after - * this call. - */ static struct drm_gem_object * -xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, - struct dma_buf *dma_buf) +xe_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) { struct dma_resv *resv = dma_buf->resv; struct xe_device *xe = to_xe_device(dev); @@ -298,10 +290,8 @@ xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, int ret = 0; dummy_obj = drm_gpuvm_resv_object_alloc(&xe->drm); - if (!dummy_obj) { - xe_bo_free(storage); + if (!dummy_obj) return ERR_PTR(-ENOMEM); - } dummy_obj->resv = resv; xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, ret) { @@ -310,8 +300,7 @@ xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, if (ret) break; - /* xe_bo_init_locked() frees storage on error */ - bo = xe_bo_init_locked(xe, storage, NULL, resv, NULL, dma_buf->size, + bo = xe_bo_init_locked(xe, NULL, NULL, resv, NULL, dma_buf->size, 0, /* Will require 1way or 2way for vm_bind */ ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec); drm_exec_retry_on_contention(&exec); @@ -362,7 +351,6 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, const struct dma_buf_attach_ops *attach_ops; struct dma_buf_attachment *attach; struct drm_gem_object *obj; - struct xe_bo *bo; if (dma_buf->ops == &xe_dmabuf_ops) { obj = dma_buf->priv; @@ -377,22 +365,14 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, } } - bo = xe_bo_alloc(); - if (IS_ERR(bo)) - return ERR_CAST(bo); - /* - * xe_dma_buf_init_obj() takes ownership of the raw bo, so do not touch - * on fail, since it will already take care of cleanup. On success we - * still need to drop the ref, if something later fails. - * - * In addition this needs to happen before the attach, since - * it will create a new attachment for this, and add it to the list of - * attachments, at which point it is globally visible, and at any point - * the export side can call into on invalidate_mappings callback, which - * require a working object. + * This needs to happen before the attach, since it will create a new + * attachment for this, and add it to the list of attachments, at which + * point it is globally visible, and at any point the export side can + * call into on invalidate_mappings callback, which require a working + * object. */ - obj = xe_dma_buf_init_obj(dev, bo, dma_buf); + obj = xe_dma_buf_create_obj(dev, dma_buf); if (IS_ERR(obj)) return obj; @@ -402,20 +382,15 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, attach_ops = test->attach_ops; #endif - attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base); + attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, obj); if (IS_ERR(attach)) { - obj = ERR_CAST(attach); - goto out_err; + xe_bo_put(gem_to_xe_bo(obj)); + return ERR_CAST(attach); } get_dma_buf(dma_buf); obj->import_attach = attach; return obj; - -out_err: - xe_bo_put(bo); - - return obj; } #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) From 15e31865ce3329e07fe634f4a75bceb3a4ad778a Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Thu, 30 Apr 2026 09:14:56 -0700 Subject: [PATCH 26/36] drm/xe/oa: Refactor oa_unit_supports_oa_format Minor refactor of oa_unit_supports_oa_format to implement Wa_14026746987. Reviewed-by: Umesh Nerlige Ramappa Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20260430161459.2892545-2-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_oa.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 5de5bf19240a..b767dccfc4fc 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1934,16 +1934,18 @@ static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent) return div_u64(nom + den - 1, den); } -static bool oa_unit_supports_oa_format(struct xe_oa_open_param *param, int type) +static bool oa_unit_supports_oa_format(struct xe_oa *oa, struct xe_oa_open_param *param) { + const struct xe_oa_format *f = &oa->oa_formats[param->oa_format]; + switch (param->oa_unit->type) { case DRM_XE_OA_UNIT_TYPE_OAG: - return type == DRM_XE_OA_FMT_TYPE_OAG || type == DRM_XE_OA_FMT_TYPE_OAR || - type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC; + return f->type == DRM_XE_OA_FMT_TYPE_OAG || f->type == DRM_XE_OA_FMT_TYPE_OAR || + f->type == DRM_XE_OA_FMT_TYPE_OAC || f->type == DRM_XE_OA_FMT_TYPE_PEC; case DRM_XE_OA_UNIT_TYPE_OAM: case DRM_XE_OA_UNIT_TYPE_OAM_SAG: case DRM_XE_OA_UNIT_TYPE_MERT: - return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; + return f->type == DRM_XE_OA_FMT_TYPE_OAM || f->type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; default: return false; } @@ -2083,8 +2085,7 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f goto err_exec_q; f = &oa->oa_formats[param.oa_format]; - if (!param.oa_format || !f->size || - !oa_unit_supports_oa_format(¶m, f->type)) { + if (!param.oa_format || !f->size || !oa_unit_supports_oa_format(oa, ¶m)) { drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n", param.oa_format, f->type, f->size, param.hwe->class); ret = -EINVAL; From dc4f643c1c2f6929f78b7f002d281411b81952e8 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Thu, 30 Apr 2026 09:14:57 -0700 Subject: [PATCH 27/36] drm/xe/oa: MERTOA Wa_14026746987 Wa_14026746987 implies that only XE_OAM_FORMAT_MPEC8u32_B8_C8 and not XE_OAM_FORMAT_MPEC8u64_B8_C8 can be supported for MERTOA unit. Reviewed-by: Umesh Nerlige Ramappa Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20260430161459.2892545-3-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_device_wa_oob.rules | 1 + drivers/gpu/drm/xe/xe_oa.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_device_wa_oob.rules b/drivers/gpu/drm/xe/xe_device_wa_oob.rules index 92371c490529..0793b9e63549 100644 --- a/drivers/gpu/drm/xe/xe_device_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_device_wa_oob.rules @@ -5,3 +5,4 @@ 14022085890 SUBPLATFORM(BATTLEMAGE, G21) 14026539277 PLATFORM(NOVALAKE_P), PLATFORM_STEP(A0, B0) 14026633728 PLATFORM(CRESCENTISLAND) +14026746987 PLATFORM(CRESCENTISLAND) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index b767dccfc4fc..13458050907a 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1942,9 +1942,12 @@ static bool oa_unit_supports_oa_format(struct xe_oa *oa, struct xe_oa_open_param case DRM_XE_OA_UNIT_TYPE_OAG: return f->type == DRM_XE_OA_FMT_TYPE_OAG || f->type == DRM_XE_OA_FMT_TYPE_OAR || f->type == DRM_XE_OA_FMT_TYPE_OAC || f->type == DRM_XE_OA_FMT_TYPE_PEC; + case DRM_XE_OA_UNIT_TYPE_MERT: + if (XE_DEVICE_WA(oa->xe, 14026746987)) + return param->oa_format == XE_OAM_FORMAT_MPEC8u32_B8_C8; + fallthrough; case DRM_XE_OA_UNIT_TYPE_OAM: case DRM_XE_OA_UNIT_TYPE_OAM_SAG: - case DRM_XE_OA_UNIT_TYPE_MERT: return f->type == DRM_XE_OA_FMT_TYPE_OAM || f->type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; default: return false; From 8555fb84749c16d799caad0c5997c183bcefd0da Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Thu, 30 Apr 2026 09:14:58 -0700 Subject: [PATCH 28/36] drm/xe/oa: Add val arg to xe_oa_is_valid_config_reg Add val arg to xe_oa_is_valid_config_reg so that register values can also be verified, in addition to register address. Value verification is needed to implement MERTOA Wa_14026779378. Reviewed-by: Umesh Nerlige Ramappa Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20260430161459.2892545-4-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/xe_oa.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 13458050907a..7e1e8a86d19c 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -2249,7 +2249,7 @@ static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr) return xe_oa_reg_in_range_table(addr, gen12_oa_mux_regs); } -static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr) +static bool xe_oa_is_valid_config_reg(struct xe_oa *oa, u32 addr, u32 val) { return xe_oa_is_valid_flex_addr(oa, addr) || xe_oa_is_valid_b_counter_addr(oa, addr) || @@ -2257,7 +2257,7 @@ static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr) } static struct xe_oa_reg * -xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr), +xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr, u32 val), u32 __user *regs, u32 n_regs) { struct xe_oa_reg *oa_regs; @@ -2275,16 +2275,16 @@ xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr), if (err) goto addr_err; - if (!is_valid(oa, addr)) { - drm_dbg(&oa->xe->drm, "Invalid oa_reg address: %X\n", addr); - err = -EINVAL; - goto addr_err; - } - err = get_user(value, regs + 1); if (err) goto addr_err; + if (!is_valid(oa, addr, value)) { + drm_dbg(&oa->xe->drm, "Invalid oa_reg addr/value: %#x %#x\n", addr, value); + err = -EINVAL; + goto addr_err; + } + oa_regs[i].addr = XE_REG(addr); oa_regs[i].value = value; @@ -2383,7 +2383,7 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid)); oa_config->regs_len = arg->n_regs; - regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg_addr, + regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg, u64_to_user_ptr(arg->regs_ptr), arg->n_regs); if (IS_ERR(regs)) { From 160c1af4fbb66ad887825eda5601eccd386015f8 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Thu, 30 Apr 2026 09:14:59 -0700 Subject: [PATCH 29/36] drm/xe/oa: MERTOA Wa_14026779378 Do not allow SYS_MEM_LAT_MEASURE_EN bit to be set in SYS_MEM_LAT_MEASURE register. Doing so can cause memory corruption. Reviewed-by: Umesh Nerlige Ramappa Signed-off-by: Ashutosh Dixit Link: https://patch.msgid.link/20260430161459.2892545-5-ashutosh.dixit@intel.com --- drivers/gpu/drm/xe/regs/xe_oa_regs.h | 3 +++ drivers/gpu/drm/xe/xe_device_wa_oob.rules | 1 + drivers/gpu/drm/xe/xe_oa.c | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/drivers/gpu/drm/xe/regs/xe_oa_regs.h b/drivers/gpu/drm/xe/regs/xe_oa_regs.h index 04a729e610aa..aa66af7e99fe 100644 --- a/drivers/gpu/drm/xe/regs/xe_oa_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_oa_regs.h @@ -6,6 +6,9 @@ #ifndef __XE_OA_REGS__ #define __XE_OA_REGS__ +#define SYS_MEM_LAT_MEASURE XE_REG(0x145194) +#define SYS_MEM_LAT_MEASURE_EN REG_BIT(31) + #define RPM_CONFIG1 XE_REG(0xd04) #define GT_NOA_ENABLE REG_BIT(9) diff --git a/drivers/gpu/drm/xe/xe_device_wa_oob.rules b/drivers/gpu/drm/xe/xe_device_wa_oob.rules index 0793b9e63549..d8dc41851425 100644 --- a/drivers/gpu/drm/xe/xe_device_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_device_wa_oob.rules @@ -6,3 +6,4 @@ 14026539277 PLATFORM(NOVALAKE_P), PLATFORM_STEP(A0, B0) 14026633728 PLATFORM(CRESCENTISLAND) 14026746987 PLATFORM(CRESCENTISLAND) +14026779378 PLATFORM(CRESCENTISLAND) diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 7e1e8a86d19c..7c9071abb44f 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -2251,6 +2251,10 @@ static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr) static bool xe_oa_is_valid_config_reg(struct xe_oa *oa, u32 addr, u32 val) { + if (XE_DEVICE_WA(oa->xe, 14026779378) && + addr == SYS_MEM_LAT_MEASURE.addr && val & SYS_MEM_LAT_MEASURE_EN) + return false; + return xe_oa_is_valid_flex_addr(oa, addr) || xe_oa_is_valid_b_counter_addr(oa, addr) || xe_oa_is_valid_mux_addr(oa, addr); From 1d3e2a655898f06d15932a8a914654d443e79029 Mon Sep 17 00:00:00 2001 From: Shekhar Chauhan Date: Tue, 12 May 2026 11:25:08 +0530 Subject: [PATCH 30/36] drm/xe/devcoredump: Drop a FIXME in devcoredump The FIXME says that xe_engine_snapshot_print.. is accessing persistent driver data, unlike what the FIXME says that it does. Drop the FIXME since the current code is not going to access the hardware while dumping. More details about this patch: https://patchwork.freedesktop.org/patch/703884/?series=161407&rev=1 The starting two feedbacks make sense and the original patch is wrong in adding those changes, but the last feedback is the one which highlights the point. Signed-off-by: Shekhar Chauhan Reviewed-by: Balasubramani Vivekanandan Link: https://patch.msgid.link/20260512055508.1380191-1-shekhar.chauhan@intel.com Signed-off-by: Balasubramani Vivekanandan --- drivers/gpu/drm/xe/xe_guc_capture.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c index bc49e40165a3..21f7caf9ea08 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.c +++ b/drivers/gpu/drm/xe/xe_guc_capture.c @@ -1841,12 +1841,6 @@ void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm str_yes_no(snapshot->kernel_reserved)); for (type = GUC_STATE_CAPTURE_TYPE_GLOBAL; type < GUC_STATE_CAPTURE_TYPE_MAX; type++) { - /* - * FIXME: During devcoredump print we should avoid accessing the - * driver pointers for gt or engine. Printing should be done only - * using the snapshot captured. Here we are accessing the gt - * pointer. It should be fixed. - */ list = xe_guc_capture_get_reg_desc_list(gt, GUC_CAPTURE_LIST_INDEX_PF, type, capture_class, false); snapshot_print_by_list_order(snapshot, p, type, list); From 388768fe17386fe82ebac1869a2e084380b6d2c7 Mon Sep 17 00:00:00 2001 From: Balasubramani Vivekanandan Date: Mon, 11 May 2026 18:07:48 +0530 Subject: [PATCH 31/36] drm/xe: Refactor emit_clear_main_copy Implement a function which returns the length of XY_FAST_COLOR_BLT instruction instead of hardcoding it inside the emit_clear_main_copy. In future platforms, the length of this instruction is expected to change and this patch helps in preparing for it. Reviewed-by: Matt Roper Link: https://patch.msgid.link/20260511123746.616662-6-balasubramani.vivekanandan@intel.com Signed-off-by: Balasubramani Vivekanandan --- .../gpu/drm/xe/instructions/xe_gpu_commands.h | 1 - drivers/gpu/drm/xe/xe_migrate.c | 16 ++++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h index 885fcf211e6d..4546c8f10516 100644 --- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h @@ -20,7 +20,6 @@ #define XY_FAST_COLOR_BLT_CMD (2 << 29 | 0x44 << 22) #define XY_FAST_COLOR_BLT_DEPTH_32 (2 << 19) -#define XY_FAST_COLOR_BLT_DW 16 #define XY_FAST_COLOR_BLT_MOCS_MASK GENMASK(27, 22) #define XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK GENMASK(27, 24) #define XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT 31 diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index f3c2ef269ba8..6ffd50050e3e 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -1484,15 +1484,21 @@ static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs bb->len += len; } +static u32 blt_fast_color_cmd_len(struct xe_device *xe) +{ + if (GRAPHICS_VERx100(xe) >= 1250) + return 16; + else + return 11; +} + static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u32 size, u32 pitch, bool is_vram) { struct xe_device *xe = gt_to_xe(gt); u32 *cs = bb->cs + bb->len; - u32 len = XY_FAST_COLOR_BLT_DW; + u32 len = blt_fast_color_cmd_len(xe); - if (GRAPHICS_VERx100(xe) < 1250) - len = 11; *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | (len - 2); @@ -1527,10 +1533,12 @@ static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, static u32 emit_clear_cmd_len(struct xe_gt *gt) { + struct xe_device *xe = gt_to_xe(gt); + if (gt->info.has_xe2_blt_instructions) return PVC_MEM_SET_CMD_LEN_DW; else - return XY_FAST_COLOR_BLT_DW; + return blt_fast_color_cmd_len(xe); } static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, From 65be72013baf9c9bbaa859b659269b31deccc8c9 Mon Sep 17 00:00:00 2001 From: Balasubramani Vivekanandan Date: Mon, 11 May 2026 18:07:49 +0530 Subject: [PATCH 32/36] drm/xe: Refactor emit_clear_link_copy Implement a function to return the length of the MEM_SET instruction. This is to prepare for future platforms where the length of MEM_SET instruction is expected to change. Reviewed-by: Matt Roper Link: https://patch.msgid.link/20260511123746.616662-7-balasubramani.vivekanandan@intel.com Signed-off-by: Balasubramani Vivekanandan --- drivers/gpu/drm/xe/instructions/xe_gpu_commands.h | 1 - drivers/gpu/drm/xe/xe_migrate.c | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h index 4546c8f10516..edd204b8dfe5 100644 --- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h @@ -37,7 +37,6 @@ #define MEM_COPY_DST_MOCS_INDEX_MASK GENMASK(6, 3) #define PVC_MEM_SET_CMD (2 << 29 | 0x5b << 22) -#define PVC_MEM_SET_CMD_LEN_DW 7 #define PVC_MEM_SET_MATRIX REG_BIT(17) #define PVC_MEM_SET_DATA_FIELD GENMASK(31, 24) /* Bspec lists field as [6:0], but index alone is from [6:1] */ diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 6ffd50050e3e..4f9be41b2f64 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -1461,12 +1461,17 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off return fence; } +static u32 blt_mem_set_cmd_len(struct xe_device *xe) +{ + return 7; +} + static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u32 size, u32 pitch) { struct xe_device *xe = gt_to_xe(gt); u32 *cs = bb->cs + bb->len; - u32 len = PVC_MEM_SET_CMD_LEN_DW; + u32 len = blt_mem_set_cmd_len(xe); *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2); *cs++ = pitch - 1; @@ -1536,7 +1541,7 @@ static u32 emit_clear_cmd_len(struct xe_gt *gt) struct xe_device *xe = gt_to_xe(gt); if (gt->info.has_xe2_blt_instructions) - return PVC_MEM_SET_CMD_LEN_DW; + return blt_mem_set_cmd_len(xe); else return blt_fast_color_cmd_len(xe); } From 66cbb3c856bc67be893f81961e1b671dc102f19b Mon Sep 17 00:00:00 2001 From: Balasubramani Vivekanandan Date: Mon, 11 May 2026 18:07:50 +0530 Subject: [PATCH 33/36] drm/xe: Refactor emit_xy_fast_copy and emit_mem_copy functions To perform copy, based on whether the platform supports service copy engines, either MEM_COPY or XY_FAST_COPY_BLT instruction is used. Length of both the instructions is same today and so they use a common define EMIT_COPY_DW. This is not true for the future platforms. Implement separate functions which return the length of the instruction to help in preparing for it. Reviewed-by: Matt Roper Link: https://patch.msgid.link/20260511123746.616662-8-balasubramani.vivekanandan@intel.com Signed-off-by: Balasubramani Vivekanandan --- .../gpu/drm/xe/instructions/xe_gpu_commands.h | 2 +- drivers/gpu/drm/xe/tests/xe_migrate.c | 2 +- drivers/gpu/drm/xe/xe_migrate.c | 32 +++++++++++++++---- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h index edd204b8dfe5..18d0fde8c98f 100644 --- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h @@ -30,7 +30,7 @@ #define XY_FAST_COPY_BLT_D1_DST_TILE4 REG_BIT(30) #define XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK GENMASK(23, 20) -#define MEM_COPY_CMD (2 << 29 | 0x5a << 22 | 0x8) +#define MEM_COPY_CMD (2 << 29 | 0x5a << 22) #define MEM_COPY_PAGE_COPY_MODE REG_BIT(19) #define MEM_COPY_MATRIX_COPY REG_BIT(17) #define MEM_COPY_SRC_MOCS_INDEX_MASK GENMASK(31, 28) diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c index 50a97705e0ac..3c1be809be82 100644 --- a/drivers/gpu/drm/xe/tests/xe_migrate.c +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -421,7 +421,7 @@ static struct dma_fence *blt_copy(struct xe_tile *tile, avail_pts, avail_pts); /* Add copy commands size here */ - batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + + batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) + ((xe_device_has_flat_ccs(xe) && copy_only_ccs) ? EMIT_COPY_CCS_DW : 0); bb = xe_bb_new(gt, batch_size, xe->info.has_usm); diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 4f9be41b2f64..9428dd5e7760 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -728,7 +728,22 @@ static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, bb->len = cs - bb->cs; } -#define EMIT_COPY_DW 10 +static u32 blt_fast_copy_cmd_len(struct xe_device *xe) +{ + return 10; +} + +static u32 blt_mem_copy_cmd_len(struct xe_device *xe) +{ + return 10; +} + +static u32 emit_copy_cmd_len(struct xe_device *xe) +{ + return (xe->info.has_mem_copy_instr) ? blt_mem_copy_cmd_len(xe) : + blt_fast_copy_cmd_len(xe); +} + static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u64 dst_ofs, unsigned int size, unsigned int pitch) @@ -736,6 +751,7 @@ static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, struct xe_device *xe = gt_to_xe(gt); u32 mocs = 0; u32 tile_y = 0; + u32 len; xe_gt_assert(gt, !(pitch & 3)); xe_gt_assert(gt, size / pitch <= S16_MAX); @@ -748,7 +764,8 @@ static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, if (GRAPHICS_VERx100(xe) >= 1250) tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4; - bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2); + len = blt_fast_copy_cmd_len(xe); + bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (len - 2); bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs; bb->cs[bb->len++] = 0; bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; @@ -765,6 +782,7 @@ static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u64 dst_ofs, unsigned int size, unsigned int pitch) { u32 mode, copy_type, width; + u32 len; xe_gt_assert(gt, IS_ALIGNED(size, pitch)); xe_gt_assert(gt, pitch <= U16_MAX); @@ -790,7 +808,9 @@ static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, xe_gt_assert(gt, width <= U16_MAX); - bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type; + len = blt_mem_copy_cmd_len(gt_to_xe(gt)); + + bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type | (len - 2); bb->cs[bb->len++] = width - 1; bb->cs[bb->len++] = size / pitch - 1; /* ignored by hw for page-copy/linear above */ bb->cs[bb->len++] = pitch - 1; @@ -967,7 +987,7 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m, } /* Add copy commands size here */ - batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + + batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) + ((needs_ccs_emit ? EMIT_COPY_CCS_DW : 0)); bb = xe_bb_new(gt, batch_size, usm); @@ -1406,7 +1426,7 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off batch_size += pte_update_size(m, 0, sysmem, &sysmem_it, &vram_L0, &sysmem_L0_ofs, &sysmem_L0_pt, 0, avail_pts, avail_pts); - batch_size += EMIT_COPY_DW; + batch_size += emit_copy_cmd_len(xe); bb = xe_bb_new(gt, batch_size, usm); if (IS_ERR(bb)) { @@ -2216,7 +2236,7 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, xe_assert(xe, npages * PAGE_SIZE <= MAX_PREEMPTDISABLE_TRANSFER); batch_size += pte_update_cmd_size(npages << PAGE_SHIFT); - batch_size += EMIT_COPY_DW; + batch_size += emit_copy_cmd_len(xe); bb = xe_bb_new(gt, batch_size, use_usm_batch); if (IS_ERR(bb)) { From 21d5a871f57909dc4d8e4f5d3bf92f9ccf2597b2 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Sun, 10 May 2026 22:56:05 +0200 Subject: [PATCH 34/36] drm/xe: Drop unused ggtt_balloon field During recent GGTT refactoring we missed to drop now unused field from the xe_tile. Drop it now. Fixes: e904c56ba6e0 ("drm/xe: Rewrite GGTT VF initialization") Signed-off-by: Michal Wajdeczko Reviewed-by: Maarten Lankhorst Link: https://patch.msgid.link/20260510205605.642-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_tile_types.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_tile_types.h b/drivers/gpu/drm/xe/xe_tile_types.h index 33932fd547d7..0048100ccb72 100644 --- a/drivers/gpu/drm/xe/xe_tile_types.h +++ b/drivers/gpu/drm/xe/xe_tile_types.h @@ -106,8 +106,6 @@ struct xe_tile { struct xe_lmtt lmtt; } pf; struct { - /** @sriov.vf.ggtt_balloon: GGTT regions excluded from use. */ - struct xe_ggtt_node *ggtt_balloon[2]; /** @sriov.vf.self_config: VF configuration data */ struct xe_tile_sriov_vf_selfconfig self_config; } vf; From 284f4cae4579eed9dd4406f18a6c1becc69f8931 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Mon, 11 May 2026 19:28:37 +0200 Subject: [PATCH 35/36] drm/xe/memirq: Update interrupt handler logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To workaround some corner case hardware limitations, new programming note for the memory based interrupt handler suggests to assume that some status bytes, like GT_MI_USER_INTERRUPT and GUC_INTR_GUC2HOST, are always set. Update our interrupt handler to follow the new rules. Bspec: 53672 Fixes: a6581ebe7685 ("drm/xe/vf: Introduce Memory Based Interrupts Handler") Signed-off-by: Michal Wajdeczko Cc: Rodrigo Vivi Cc: Matthew Brost Reviewed-by: Michał Winiarski Link: https://patch.msgid.link/20260511172838.2299-2-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_memirq.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c index 811e07136efb..579af47edc61 100644 --- a/drivers/gpu/drm/xe/xe_memirq.c +++ b/drivers/gpu/drm/xe/xe_memirq.c @@ -427,13 +427,25 @@ static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector, return __memirq_received(memirq, vector, offset, name, true); } +static void memirq_assume_received(struct xe_memirq *memirq, const char *source, + u16 offset, const char *status) +{ + memirq_debug(memirq, "ASSUME %s %s(%u)\n", source, status, offset); +} + static void memirq_dispatch_engine(struct xe_memirq *memirq, struct iosys_map *status, struct xe_hw_engine *hwe) { memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr); - if (memirq_received(memirq, status, ilog2(GT_MI_USER_INTERRUPT), hwe->name)) - xe_hw_engine_handle_irq(hwe, GT_MI_USER_INTERRUPT); + /* + * The programming note says to assume that GT_MI_USER_INTERRUPT is always + * set. Check and clear related status byte just for a debug. + */ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ) && + !memirq_received(memirq, status, ilog2(GT_MI_USER_INTERRUPT), hwe->name)) + memirq_assume_received(memirq, hwe->name, ilog2(GT_MI_USER_INTERRUPT), "USER"); + xe_hw_engine_handle_irq(hwe, GT_MI_USER_INTERRUPT); } static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *status, @@ -443,8 +455,14 @@ static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *stat memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr); - if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name)) - xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST); + /* + * The programming note says to assume that GUC_INTR_GUC2HOST is always + * set. Check and clear related status byte just for a debug. + */ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ) && + !memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name)) + memirq_assume_received(memirq, name, ilog2(GUC_INTR_GUC2HOST), "GUC2HOST"); + xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST); /* * This is a software interrupt that must be cleared after it's consumed From 2ddedd4b7b7c329dd65358025cba8652675bec3d Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Mon, 11 May 2026 19:28:38 +0200 Subject: [PATCH 36/36] drm/xe/memirq: Enable GT_MI_USER_INTERRUPT only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We only expect and handle the GT_MI_USER_INTERRUPT from the engines, there is no point in enabling other interrupts, like GT_CONTEXT_SWITCH_INTERRUPT, if we don't intent to handle them. Signed-off-by: Michal Wajdeczko Reviewed-by: Michał Winiarski Link: https://patch.msgid.link/20260511172838.2299-3-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_memirq.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c index 579af47edc61..3848ff81c1f9 100644 --- a/drivers/gpu/drm/xe/xe_memirq.c +++ b/drivers/gpu/drm/xe/xe_memirq.c @@ -212,7 +212,11 @@ static int memirq_alloc_pages(struct xe_memirq *memirq) static void memirq_set_enable(struct xe_memirq *memirq, bool enable) { - iosys_map_wr(&memirq->mask, 0, u32, enable ? GENMASK(15, 0) : 0); + /* + * We only care about the GT_MI_USER_INTERRUPT from the engines and + * the GuC does not look at the ENABLE mask at all. + */ + iosys_map_wr(&memirq->mask, 0, u32, enable ? GT_MI_USER_INTERRUPT : 0); memirq->enabled = enable; }