From aef2ca9353c2f26dbacfb3b8e6f33fecfbf2e67d Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Wed, 19 Aug 2026 15:23:56 +0800 Subject: [PATCH 01/18] drm/amdgpu/mes: fix the inconsistent indenting for mes_userq_map() Fix the inconsistent indenting warning for mes_userq_map(). Fixes: d0827dda8fa7 ("drm/amdgpu/mes: refactor the amdgpu_mes_alloc/free_proc|gang()") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202608190252.8XCa0HqR-lkp@intel.com/ Signed-off-by: Prike Liang Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c index fae709f134bb..14a5abe42d1f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c @@ -146,7 +146,7 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue) queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr; if (mes->use_rs64mem) { - if (!uq_mgr->proc_ctx_allocated) { + if (!uq_mgr->proc_ctx_allocated) { r = amdgpu_mes_alloc_proc_ctx_index(mes, &uq_mgr->proc_ctx_array_index); if (r) { DRM_ERROR("Failed to allocate userq process index err:%d\n", r); From d36fbf82189319e9af564c93900d30e55e87e7e0 Mon Sep 17 00:00:00 2001 From: Jesse Zhang Date: Fri, 14 Aug 2026 15:01:44 +0800 Subject: [PATCH 02/18] drm/amdgpu/userq: lock and validate wptr BOs before reading their GPU offset on restore On resume, amdgpu_userq_vm_validate_and_restore_queue() updates each queue's wptr GPU address via amdgpu_bo_gpu_offset(). WPTR BOs are VM-mapped, but each BO has its own reservation object and is not implicitly covered by the VM validation path here. This can leave offset reads without proper BO locking/placement state and trigger WARN_ONs. ------------[ cut here ]------------ WARNING: amdgpu_object.c:1486 at amdgpu_bo_gpu_offset+0x75/0xa0 [amdgpu], CPU#3: kworker/3:1/116 Workqueue: events amdgpu_userq_restore_worker [amdgpu] RIP: 0010:amdgpu_bo_gpu_offset+0x75/0xa0 [amdgpu] Call Trace: amdgpu_userq_vm_validate_and_restore_queue+0x629/0x960 [amdgpu] amdgpu_userq_restore_worker+0xa6/0x180 [amdgpu] process_scheduled_works+0xa6/0x460 worker_thread+0x13c/0x290 kthread+0xfb/0x140 ret_from_fork+0x1b6/0x2b0 ret_from_fork_asm+0x1a/0x30 ---[ end trace 0000000000000000 ]--- ------------[ cut here ]------------ WARNING: amdgpu_object.c:1485 at amdgpu_bo_gpu_offset+0x9a/0xa0 [amdgpu], CPU#2: kworker/2:1/127 Workqueue: events amdgpu_userq_restore_worker [amdgpu] RIP: 0010:amdgpu_bo_gpu_offset+0x9a/0xa0 [amdgpu] Add each queue's WPTR BO to the drm_exec ww context and validate it to its allowed placement before the later offset update. v2: - Clarify that WPTR BOs are VM-mapped (fix incorrect "not part of VM" wording). (Christian) - Describe both parts of the fix: lock BO reservations in drm_exec and validate BO placement before offset reads. Acked-by: Alex Deucher Signed-off-by: Jesse Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 24adad7be251..0a816b3c5ff9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -1047,6 +1047,30 @@ amdgpu_userq_vm_validate_and_restore_queue(struct amdgpu_userq_mgr *uq_mgr) drm_exec_retry_on_contention(&exec); if (unlikely(ret)) goto unlock_all; + + /* + * WPTR BOs are VM-mapped, but each BO has its own reservation + * object. Lock them into this drm_exec ww context so the later + * amdgpu_bo_gpu_offset() reads are done with the BO resv locked. + */ + xa_for_each(&uq_mgr->userq_xa, tmp_key, queue) { + struct ttm_operation_ctx wptr_ctx = { false, false }; + + bo = queue->wptr_obj.obj; + if (!bo) + continue; + + ret = drm_exec_prepare_obj(&exec, &bo->tbo.base, + TTM_NUM_MOVE_FENCES + 1); + drm_exec_retry_on_contention(&exec); + if (unlikely(ret)) + goto unlock_all; + + amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &wptr_ctx); + if (unlikely(ret)) + goto unlock_all; + } } if (invalidated) { From 6760f5cb12d2366ddd58a2d8637f7583d73f596b Mon Sep 17 00:00:00 2001 From: Bob Zhou Date: Wed, 19 Aug 2026 14:23:49 +0800 Subject: [PATCH 03/18] drm/amdgpu: avoid force-completing uninitialized UVD rings uvd_v7_0_sw_init() does not initialize the UVD decode ring for an SR-IOV VF. However, amdgpu_uvd_resume() unconditionally force-completes the decode ring when restoring its fence sequence. Skip fence completion when the fence driver is not initialized. Fixes: 0a33b11d26c6 ("drm/amdgpu: mark force completed fences with -ECANCELED") Cc: stable@vger.kernel.org Signed-off-by: Bob Zhou Acked-by: Leo Liu Acked-by: Frank Min Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 228a405a94c4..ecd7caa95d4b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -517,7 +517,8 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev) } memset_io(ptr, 0, size); /* to restore uvd fence seq */ - amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring, NULL); + if (adev->uvd.inst[i].ring.fence_drv.initialized) + amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring, NULL); } } return 0; From 290e0be2abb7d9f1473d12ef2ed3220b071c42c3 Mon Sep 17 00:00:00 2001 From: David Belanger Date: Fri, 14 Aug 2026 12:57:18 -0400 Subject: [PATCH 04/18] drm/kfd: Add CU occupancy support to GFX11 Port changes from GFX9 to GFX11 mostly as-is. Minor changes to register access code. Assisted-by: Claude:Sonnet-4-6 Signed-off-by: David Belanger Acked-by: Alex Deucher Reviewed-by: Sreekant Somasekharan Signed-off-by: Alex Deucher --- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c index 724beb96ed1a..04fab30ab5dd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c @@ -807,6 +807,153 @@ static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev, return 0; } +static void lock_spi_csq_mutexes(struct amdgpu_device *adev) +{ + mutex_lock(&adev->srbm_mutex); + mutex_lock(&adev->grbm_idx_mutex); + +} + +static void unlock_spi_csq_mutexes(struct amdgpu_device *adev) +{ + mutex_unlock(&adev->grbm_idx_mutex); + mutex_unlock(&adev->srbm_mutex); +} + +/** + * get_wave_count: Read device registers to get number of waves in flight for + * a particular queue. The method also returns the doorbell offset associated + * with the queue. + * + * @adev: Handle of device whose registers are to be read + * @queue_idx: Index of queue in the queue-map bit-field + * @queue_cnt: Stores the wave count and doorbell offset for an active queue + * @inst: xcc's instance number on a multi-XCC setup + */ +static void get_wave_count(struct amdgpu_device *adev, int queue_idx, + struct kfd_cu_occupancy *queue_cnt, uint32_t inst) +{ + int pipe_idx; + int queue_slot; + unsigned int reg_val; + unsigned int wave_cnt; + /* + * Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID + * parameters to read out waves in flight. Get doorbell offset if there are + * non-zero waves in flight. + */ + pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe; + queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe; + soc21_grbm_select(adev, 1, pipe_idx, queue_slot, 0); + reg_val = RREG32_SOC15_IP(GC, SOC15_REG_OFFSET(GC, 0, + regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot); + wave_cnt = reg_val & SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK; + if (wave_cnt != 0) { + queue_cnt->wave_cnt += wave_cnt; + queue_cnt->doorbell_off = + (RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL) & + CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK) >> + CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; + } +} + +/** + * kgd_gfx_v11_get_cu_occupancy: Reads relevant registers associated with each + * shader engine and aggregates the number of waves that are in flight for the + * process whose pasid is provided as a parameter. The process could have ZERO + * or more queues running and submitting waves to compute units. + * + * @adev: Handle of device from which to get number of waves in flight + * @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset + * for comparison later. + * @max_waves_per_cu: Output parameter updated with maximum number of waves + * possible per Compute Unit + * @inst: xcc's instance number on a multi-XCC setup + * + * Note: It's possible that the device has too many queues (oversubscription) + * in which case a VMID could be remapped to a different PASID. This could lead + * to an inaccurate wave count. Following is a high-level sequence: + * Time T1: vmid = getVmid(); vmid is associated with Pasid P1 + * Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2 + * In the sequence above wave count obtained from time T1 will be incorrectly + * lost or added to total wave count. + * + * The registers that provide the waves in flight are: + * + * SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a + * queue is slotted, OFF if there is no queue. A process could have ZERO or + * more queues slotted and submitting waves to be run on compute units. Even + * when there is a queue it is possible there could be zero wave fronts, this + * can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem + * command + * + * For each bit that is ON from above: + * + * Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the + * number of waves that are in flight for the queue at specified index. The + * index ranges from 0 to 7. + * + * If non-zero waves are in flight, store the corresponding doorbell offset + * of the queue, along with the wave count. + * + * Determine if the queue belongs to the process by comparing the doorbell + * offset against the process's queues. If it matches, aggregate the wave + * count for the process. + * + * Reading registers referenced above involves programming GRBM appropriately + */ +static void kgd_gfx_v11_get_cu_occupancy(struct amdgpu_device *adev, + struct kfd_cu_occupancy *cu_occupancy, + int *max_waves_per_cu, uint32_t inst) +{ + int qidx; + int se_idx; + int se_cnt; + int queue_map; + int max_queue_cnt; + DECLARE_BITMAP(cp_queue_bitmap, AMDGPU_MAX_QUEUES); + + lock_spi_csq_mutexes(adev); + soc21_grbm_select(adev, 1, 0, 0, 0); + + /* + * Iterate through the shader engines and arrays of the device + * to get number of waves in flight + */ + bitmap_complement(cp_queue_bitmap, adev->gfx.mec_bitmap[0].queue_bitmap, + AMDGPU_MAX_QUEUES); + max_queue_cnt = adev->gfx.mec.num_pipe_per_mec * + adev->gfx.mec.num_queue_per_pipe; + se_cnt = adev->gfx.config.max_shader_engines; + for (se_idx = 0; se_idx < se_cnt; se_idx++) { + amdgpu_gfx_select_se_sh(adev, se_idx, 0, 0xffffffff, inst); + queue_map = RREG32_SOC15(GC, 0, + regSPI_CSQ_WF_ACTIVE_STATUS); + + for (qidx = 0; qidx < max_queue_cnt; qidx++) { + /* Skip queues that are not associated with + * compute functions + */ + if (!test_bit(qidx, cp_queue_bitmap)) + continue; + + if (!(queue_map & (1 << qidx))) + continue; + + /* Get number of waves in flight and aggregate them */ + get_wave_count(adev, qidx, &cu_occupancy[qidx], inst); + } + } + + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, inst); + soc21_grbm_select(adev, 0, 0, 0, 0); + unlock_spi_csq_mutexes(adev); + + /* Update the output parameters and return */ + *max_waves_per_cu = adev->gfx.cu_info.simd_per_cu * + adev->gfx.cu_info.max_waves_per_simd; +} + const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .program_sh_mem_settings = program_sh_mem_settings_v11, .set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11, @@ -832,5 +979,6 @@ const struct kfd2kgd_calls gfx_v11_kfd2kgd = { .clear_address_watch = kgd_gfx_v11_clear_address_watch, .hqd_get_pq_addr = kgd_gfx_v11_hqd_get_pq_addr, .hqd_reset = kgd_gfx_v11_hqd_reset, + .get_cu_occupancy = kgd_gfx_v11_get_cu_occupancy, .hqd_sdma_get_doorbell = kgd_gfx_v11_hqd_sdma_get_doorbell }; From fb1e65a80dd9167e8dfdfdfef178d1d012d8bb58 Mon Sep 17 00:00:00 2001 From: David Belanger Date: Fri, 14 Aug 2026 13:25:56 -0400 Subject: [PATCH 05/18] drm/kfd: Add CU occupancy support to GFX12 Port changes from GFX9 to GFX12 mostly as-is. Minor changes to register access code. Assisted-by: Claude:Sonnet-4-6 Signed-off-by: David Belanger Acked-by: Alex Deucher Reviewed-by: Sreekant Somasekharan Signed-off-by: Alex Deucher --- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c | 150 +++++++++++++++++- 1 file changed, 149 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c index e11ba3e91841..62b9db64368e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c @@ -368,6 +368,153 @@ static uint32_t kgd_gfx_v12_hqd_sdma_get_doorbell(struct amdgpu_device *adev, return 0; } +static void lock_spi_csq_mutexes(struct amdgpu_device *adev) +{ + mutex_lock(&adev->srbm_mutex); + mutex_lock(&adev->grbm_idx_mutex); + +} + +static void unlock_spi_csq_mutexes(struct amdgpu_device *adev) +{ + mutex_unlock(&adev->grbm_idx_mutex); + mutex_unlock(&adev->srbm_mutex); +} + +/** + * get_wave_count: Read device registers to get number of waves in flight for + * a particular queue. The method also returns the doorbell offset associated + * with the queue. + * + * @adev: Handle of device whose registers are to be read + * @queue_idx: Index of queue in the queue-map bit-field + * @queue_cnt: Stores the wave count and doorbell offset for an active queue + * @inst: xcc's instance number on a multi-XCC setup + */ +static void get_wave_count(struct amdgpu_device *adev, int queue_idx, + struct kfd_cu_occupancy *queue_cnt, uint32_t inst) +{ + int pipe_idx; + int queue_slot; + unsigned int reg_val; + unsigned int wave_cnt; + /* + * Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID + * parameters to read out waves in flight. Get doorbell offset if there are + * non-zero waves in flight. + */ + pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe; + queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe; + soc24_grbm_select(adev, 1, pipe_idx, queue_slot, 0); + reg_val = RREG32_SOC15_IP(GC, SOC15_REG_OFFSET(GC, 0, + regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot); + wave_cnt = reg_val & SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK; + if (wave_cnt != 0) { + queue_cnt->wave_cnt += wave_cnt; + queue_cnt->doorbell_off = + (RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL) & + CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK) >> + CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; + } +} + +/** + * kgd_gfx_v12_get_cu_occupancy: Reads relevant registers associated with each + * shader engine and aggregates the number of waves that are in flight for the + * process whose pasid is provided as a parameter. The process could have ZERO + * or more queues running and submitting waves to compute units. + * + * @adev: Handle of device from which to get number of waves in flight + * @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset + * for comparison later. + * @max_waves_per_cu: Output parameter updated with maximum number of waves + * possible per Compute Unit + * @inst: xcc's instance number on a multi-XCC setup + * + * Note: It's possible that the device has too many queues (oversubscription) + * in which case a VMID could be remapped to a different PASID. This could lead + * to an inaccurate wave count. Following is a high-level sequence: + * Time T1: vmid = getVmid(); vmid is associated with Pasid P1 + * Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2 + * In the sequence above wave count obtained from time T1 will be incorrectly + * lost or added to total wave count. + * + * The registers that provide the waves in flight are: + * + * SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a + * queue is slotted, OFF if there is no queue. A process could have ZERO or + * more queues slotted and submitting waves to be run on compute units. Even + * when there is a queue it is possible there could be zero wave fronts, this + * can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem + * command + * + * For each bit that is ON from above: + * + * Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the + * number of waves that are in flight for the queue at specified index. The + * index ranges from 0 to 7. + * + * If non-zero waves are in flight, store the corresponding doorbell offset + * of the queue, along with the wave count. + * + * Determine if the queue belongs to the process by comparing the doorbell + * offset against the process's queues. If it matches, aggregate the wave + * count for the process. + * + * Reading registers referenced above involves programming GRBM appropriately + */ +static void kgd_gfx_v12_get_cu_occupancy(struct amdgpu_device *adev, + struct kfd_cu_occupancy *cu_occupancy, + int *max_waves_per_cu, uint32_t inst) +{ + int qidx; + int se_idx; + int se_cnt; + int queue_map; + int max_queue_cnt; + DECLARE_BITMAP(cp_queue_bitmap, AMDGPU_MAX_QUEUES); + + lock_spi_csq_mutexes(adev); + soc24_grbm_select(adev, 1, 0, 0, 0); + + /* + * Iterate through the shader engines and arrays of the device + * to get number of waves in flight + */ + bitmap_complement(cp_queue_bitmap, adev->gfx.mec_bitmap[0].queue_bitmap, + AMDGPU_MAX_QUEUES); + max_queue_cnt = adev->gfx.mec.num_pipe_per_mec * + adev->gfx.mec.num_queue_per_pipe; + se_cnt = adev->gfx.config.max_shader_engines; + for (se_idx = 0; se_idx < se_cnt; se_idx++) { + amdgpu_gfx_select_se_sh(adev, se_idx, 0, 0xffffffff, inst); + queue_map = RREG32_SOC15(GC, 0, + regSPI_CSQ_WF_ACTIVE_STATUS); + + for (qidx = 0; qidx < max_queue_cnt; qidx++) { + /* Skip queues that are not associated with + * compute functions + */ + if (!test_bit(qidx, cp_queue_bitmap)) + continue; + + if (!(queue_map & (1 << qidx))) + continue; + + /* Get number of waves in flight and aggregate them */ + get_wave_count(adev, qidx, &cu_occupancy[qidx], inst); + } + } + + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, inst); + soc24_grbm_select(adev, 0, 0, 0, 0); + unlock_spi_csq_mutexes(adev); + + /* Update the output parameters and return */ + *max_waves_per_cu = adev->gfx.cu_info.simd_per_cu * + adev->gfx.cu_info.max_waves_per_simd; +} + const struct kfd2kgd_calls gfx_v12_kfd2kgd = { .init_interrupts = init_interrupts_v12, .hqd_dump = hqd_dump_v12, @@ -381,5 +528,6 @@ const struct kfd2kgd_calls gfx_v12_kfd2kgd = { .set_wave_launch_mode = kgd_gfx_v12_set_wave_launch_mode, .set_address_watch = kgd_gfx_v12_set_address_watch, .clear_address_watch = kgd_gfx_v12_clear_address_watch, - .hqd_sdma_get_doorbell = kgd_gfx_v12_hqd_sdma_get_doorbell + .hqd_sdma_get_doorbell = kgd_gfx_v12_hqd_sdma_get_doorbell, + .get_cu_occupancy = kgd_gfx_v12_get_cu_occupancy, }; From fb62f7f031155fe6c1095d2fc921654ad51f87c2 Mon Sep 17 00:00:00 2001 From: David Belanger Date: Fri, 14 Aug 2026 13:26:17 -0400 Subject: [PATCH 06/18] drm/kfd: Add CU occupancy support to GFX12.1 Port changes from GFX9 to GFX12.1 mostly as-is. Minor changes to register access code. Assisted-by: Claude:Sonnet 4.6 Signed-off-by: David Belanger Reviewed-by: Sreekant Somasekharan Signed-off-by: Alex Deucher --- .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c | 150 +++++++++++++++++- 1 file changed, 149 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c index 38ca1aea33b2..070001fd34b0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c @@ -371,6 +371,153 @@ static uint32_t kgd_gfx_v12_1_hqd_sdma_get_doorbell(struct amdgpu_device *adev, return 0; } +static void lock_spi_csq_mutexes(struct amdgpu_device *adev) +{ + mutex_lock(&adev->srbm_mutex); + mutex_lock(&adev->grbm_idx_mutex); + +} + +static void unlock_spi_csq_mutexes(struct amdgpu_device *adev) +{ + mutex_unlock(&adev->grbm_idx_mutex); + mutex_unlock(&adev->srbm_mutex); +} + +/** + * get_wave_count: Read device registers to get number of waves in flight for + * a particular queue. The method also returns the doorbell offset associated + * with the queue. + * + * @adev: Handle of device whose registers are to be read + * @queue_idx: Index of queue in the queue-map bit-field + * @queue_cnt: Stores the wave count and doorbell offset for an active queue + * @inst: xcc's instance number on a multi-XCC setup + */ +static void get_wave_count(struct amdgpu_device *adev, int queue_idx, + struct kfd_cu_occupancy *queue_cnt, uint32_t inst) +{ + int pipe_idx; + int queue_slot; + unsigned int reg_val; + unsigned int wave_cnt; + /* + * Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID + * parameters to read out waves in flight. Get doorbell offset if there are + * non-zero waves in flight. + */ + pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe; + queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe; + amdgpu_gfx_select_me_pipe_q(adev, 1, pipe_idx, queue_slot, 0, inst); + reg_val = RREG32_SOC15_IP(GC, SOC15_REG_OFFSET(GC, GET_INST(GC, inst), + regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot); + wave_cnt = reg_val & SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK; + if (wave_cnt != 0) { + queue_cnt->wave_cnt += wave_cnt; + queue_cnt->doorbell_off = + (RREG32_SOC15(GC, GET_INST(GC, inst), regCP_HQD_PQ_DOORBELL_CONTROL) & + CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK) >> + CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; + } +} + +/** + * kgd_gfx_v12_1_get_cu_occupancy: Reads relevant registers associated with + * each shader engine and aggregates the number of waves that are in flight + * for the process whose pasid is provided as a parameter. The process could + * have ZERO or more queues running and submitting waves to compute units. + * + * @adev: Handle of device from which to get number of waves in flight + * @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset + * for comparison later. + * @max_waves_per_cu: Output parameter updated with maximum number of waves + * possible per Compute Unit + * @inst: xcc's instance number on a multi-XCC setup + * + * Note: It's possible that the device has too many queues (oversubscription) + * in which case a VMID could be remapped to a different PASID. This could lead + * to an inaccurate wave count. Following is a high-level sequence: + * Time T1: vmid = getVmid(); vmid is associated with Pasid P1 + * Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2 + * In the sequence above wave count obtained from time T1 will be incorrectly + * lost or added to total wave count. + * + * The registers that provide the waves in flight are: + * + * SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a + * queue is slotted, OFF if there is no queue. A process could have ZERO or + * more queues slotted and submitting waves to be run on compute units. Even + * when there is a queue it is possible there could be zero wave fronts, this + * can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem + * command + * + * For each bit that is ON from above: + * + * Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the + * number of waves that are in flight for the queue at specified index. The + * index ranges from 0 to 7. + * + * If non-zero waves are in flight, store the corresponding doorbell offset + * of the queue, along with the wave count. + * + * Determine if the queue belongs to the process by comparing the doorbell + * offset against the process's queues. If it matches, aggregate the wave + * count for the process. + * + * Reading registers referenced above involves programming GRBM appropriately + */ +static void kgd_gfx_v12_1_get_cu_occupancy(struct amdgpu_device *adev, + struct kfd_cu_occupancy *cu_occupancy, + int *max_waves_per_cu, uint32_t inst) +{ + int qidx; + int se_idx; + int se_cnt; + int queue_map; + int max_queue_cnt; + DECLARE_BITMAP(cp_queue_bitmap, AMDGPU_MAX_QUEUES); + + lock_spi_csq_mutexes(adev); + amdgpu_gfx_select_me_pipe_q(adev, 1, 0, 0, 0, inst); + + /* + * Iterate through the shader engines and arrays of the device + * to get number of waves in flight + */ + bitmap_complement(cp_queue_bitmap, adev->gfx.mec_bitmap[0].queue_bitmap, + AMDGPU_MAX_QUEUES); + max_queue_cnt = adev->gfx.mec.num_pipe_per_mec * + adev->gfx.mec.num_queue_per_pipe; + se_cnt = adev->gfx.config.max_shader_engines; + for (se_idx = 0; se_idx < se_cnt; se_idx++) { + amdgpu_gfx_select_se_sh(adev, se_idx, 0, 0xffffffff, inst); + queue_map = RREG32_SOC15(GC, GET_INST(GC, inst), + regSPI_CSQ_WF_ACTIVE_STATUS); + + for (qidx = 0; qidx < max_queue_cnt; qidx++) { + /* Skip queues that are not associated with + * compute functions + */ + if (!test_bit(qidx, cp_queue_bitmap)) + continue; + + if (!(queue_map & (1 << qidx))) + continue; + + /* Get number of waves in flight and aggregate them */ + get_wave_count(adev, qidx, &cu_occupancy[qidx], inst); + } + } + + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, inst); + amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, inst); + unlock_spi_csq_mutexes(adev); + + /* Update the output parameters and return */ + *max_waves_per_cu = adev->gfx.cu_info.simd_per_cu * + adev->gfx.cu_info.max_waves_per_simd; +} + const struct kfd2kgd_calls gfx_v12_1_kfd2kgd = { .init_interrupts = init_interrupts_v12_1, .hqd_dump = hqd_dump_v12_1, @@ -384,5 +531,6 @@ const struct kfd2kgd_calls gfx_v12_1_kfd2kgd = { .set_wave_launch_mode = kgd_gfx_v12_1_set_wave_launch_mode, .set_address_watch = kgd_gfx_v12_1_set_address_watch, .clear_address_watch = kgd_gfx_v12_1_clear_address_watch, - .hqd_sdma_get_doorbell = kgd_gfx_v12_1_hqd_sdma_get_doorbell + .hqd_sdma_get_doorbell = kgd_gfx_v12_1_hqd_sdma_get_doorbell, + .get_cu_occupancy = kgd_gfx_v12_1_get_cu_occupancy }; From 2ee9836545e690c9e66ec203445d7705959fd7a8 Mon Sep 17 00:00:00 2001 From: David Rosca Date: Thu, 20 Aug 2026 16:32:51 +0200 Subject: [PATCH 07/18] drm/amdgpu: Fix VCE 3 ring align_mask The largest frame is 20 dwords, so 0xf mask is too small. This was always wrong, but we were lucky with the VCE_CMD_END commands inserted after fence and vm_flush. Fixes: 8897ea8c761b ("drm/amdgpu: Implement insert_end for VCE 3") Cc: stable@vger.kernel.org Acked-by: Alex Deucher Signed-off-by: David Rosca Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/vce_v3_0.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index a9497e2e07f7..4dbbeaf97ad1 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -891,7 +891,7 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_phys_funcs = { static const struct amdgpu_ring_funcs vce_v3_0_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCE, - .align_mask = 0xf, + .align_mask = 0x1f, .nop = VCE_CMD_NO_OP, .support_64bit_ptrs = false, .no_user_fence = true, From 40ba09e11188d1b7f79d51fc28aca5ea45e0c138 Mon Sep 17 00:00:00 2001 From: Sunday Clement Date: Thu, 6 Aug 2026 10:59:34 -0400 Subject: [PATCH 08/18] drm/amdkfd: Reject zero-sized AQL queue allocations after size halving KFD_IOC_ALLOC_MEMORY_OF_GPU with flag KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM and size=1 triggers the AQL wraparound workaround (size >>= 1), reducing size to 0. The resulting zero passes through PAGE_ALIGN(0) = 0 without validation, bypassing the per-process VRAM quota check in reserve_mem_limit() (vram_used + 0 > vram_available is always false). The fix adds post-halving zero-size validation in the primary allocation path (amdgpu_amdkfd_gpuvm.c). The check happens after size halving but before reserve_mem_limit(), and uses err_alignment_size error path to properly clean up the allocated kgd_mem structure and mutex. Cc: stable@vger.kernel.org Signed-off-by: Sunday Clement Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 7 +++++++ drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 34481ee7065a..d66881684ee5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1795,6 +1795,12 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( size >>= 1; aligned_size = PAGE_ALIGN(size); + /* reject AQL queue with size < 2 */ + if (!aligned_size) { + ret = -EINVAL; + goto err_alignment_size; + } + (*mem)->alloc_flags = flags; amdgpu_sync_create(&(*mem)->sync); @@ -1886,6 +1892,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id); err_reserve_limit: amdgpu_sync_free(&(*mem)->sync); +err_alignment_size: mutex_destroy(&(*mem)->lock); if (gobj) drm_gem_object_put(gobj); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 6fd18488d5cf..7fcfc150a7fc 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1200,7 +1200,8 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, if (flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM) size >>= 1; - atomic64_add(PAGE_ALIGN(size), &pdd->vram_usage); + size = PAGE_ALIGN(size); + atomic64_add(size, &pdd->vram_usage); } mutex_unlock(&p->mutex); From b30900566642ceb2c9e12b56c2afec28d0fd91a0 Mon Sep 17 00:00:00 2001 From: Xiang Liu Date: Fri, 21 Aug 2026 17:41:57 +0800 Subject: [PATCH 09/18] drm/amdgpu: clamp the isolation index for rings outside a partition adev->isolation[] has one slot per partition, but a ring that is not assigned to one keeps AMDGPU_XCP_NO_PARTITION, which is ~0, so indexing the array with it is out of bounds. SDMA submissions hit this on both the isolation enforcement and the VM flush path and trip UBSAN. Fall back to the first slot the way the cleaner shader path already does, and stop taking the address before the ring type check that makes it relevant. Cc: stable@vger.kernel.org Signed-off-by: Xiang Liu Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 5 ++++- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 019581577603..44bed0ba64a3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -6687,8 +6687,8 @@ struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev, struct amdgpu_ring *ring, struct amdgpu_job *job) { - struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id]; struct drm_sched_fence *f = job->base.s_fence; + struct amdgpu_isolation *isolation; struct dma_fence *dep; void *owner; int r; @@ -6701,6 +6701,9 @@ struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev, ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE) return NULL; + isolation = &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ? + 0 : ring->xcp_id]; + /* * All submissions where enforce isolation is false are handled as if * they come from a single client. Use ~0l as the owner to distinct it diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index a3758c654dd4..aedf72c2333e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -776,7 +776,9 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool *emit_gds_needed) { struct amdgpu_device *adev = ring->adev; - struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id]; + struct amdgpu_isolation *isolation = + &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ? + 0 : ring->xcp_id]; unsigned vmhub = ring->vm_hub; struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; From 52536ce677a3470c0e5323b940791efb33975450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timur=20Krist=C3=B3f?= Date: Fri, 21 Aug 2026 23:50:58 +0200 Subject: [PATCH 10/18] drm/amd/display: Fix HPD consideration for VGA/LVDS connectors on DCE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a refactor that landed in Linux 7.0, DC now crashes when it is initialized on GPUs that have a VGA or LVDS connector. This is because these connectors have no HPD so the hpd_gpio is NULL and therefore DC takes the code path meant for DCN 4.2+ which sets irq_source_hpd = 255 that causes the subsequent code to try to register the HPD interrupt, which fails, and causes a crash. This commit should be backported to Linux 7.0 and newer. Cc: stable@vger.kernel.org Cc: Dmytro Laktyushkin Cc: Roman Li Closes: https://gitlab.freedesktop.org/drm/amd/-/work_items/5490 Fixes: def3488eb0fd ("drm/amd/display: refactor HPD to increase flexibility") Signed-off-by: Timur Kristóf Reviewed-by: Mario Limonciello (AMD) Link: https://patch.msgid.link/20260821215059.312868-1-timur.kristof@gmail.com Signed-off-by: Mario Limonciello Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/link/link_factory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c index 89265b083935..7146f8356233 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c @@ -639,7 +639,7 @@ static bool construct_phy(struct dc_link *link, DC_LOG_DC("BIOS object table - hpd_gpio id: %d", enc_init_data.hpd_gpio->id); DC_LOG_DC("BIOS object table - hpd_gpio en: %d", enc_init_data.hpd_gpio->en); - } else { + } else if (link->ctx->dce_version > DCN_VERSION_4_01) { struct graphics_object_hpd_info hpd_info; if (link->ctx->dc_bios->funcs->get_hpd_info(link->ctx->dc_bios, link->link_id, &hpd_info) == BP_RESULT_OK) { From 9e8bcfde0039238e904b4e720e66b7f4fbf82c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timur=20Krist=C3=B3f?= Date: Fri, 21 Aug 2026 23:50:59 +0200 Subject: [PATCH 11/18] drm/amd/display: Log details when failing to register HPD IRQ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should help diagnose HPD IRQ related issues in the future. Signed-off-by: Timur Kristóf Reviewed-by: Mario Limonciello (AMD) Link: https://patch.msgid.link/20260821215059.312868-2-timur.kristof@gmail.com Signed-off-by: Mario Limonciello Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c index 9be63996b062..d0239a3de2e1 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c @@ -1735,7 +1735,9 @@ int amdgpu_dm_register_hpd_handlers(struct amdgpu_device *adev) if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || int_params.irq_source < DC_IRQ_SOURCE_HPD1 || int_params.irq_source > DC_IRQ_SOURCE_HPD6) { - drm_err(adev_to_drm(adev), "Failed to register hpd irq!\n"); + drm_err(adev_to_drm(adev), + "Failed to register hpd irq %u for %s!\n", + int_params.irq_source, connector->name); return -EINVAL; } @@ -1753,7 +1755,9 @@ int amdgpu_dm_register_hpd_handlers(struct amdgpu_device *adev) if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || int_params.irq_source < DC_IRQ_SOURCE_HPD1RX || int_params.irq_source > DC_IRQ_SOURCE_HPD6RX) { - drm_err(adev_to_drm(adev), "Failed to register hpd rx irq!\n"); + drm_err(adev_to_drm(adev), + "Failed to register hpd rx irq %u for %s!\n", + int_params.irq_source, connector->name); return -EINVAL; } From 960c4a8069bfd352c48cc88592618f1ebe24c69e Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Sun, 23 Aug 2026 15:22:54 -0500 Subject: [PATCH 12/18] drm/amdkfd: Fix error path at svm_migrate_copy_to_ram If page migration from device to sys ram fails for some reasons driver needs release and unlock allocated system pages. To do that driver should use page physical address, or pfn, then get struct page*. Current driver uses dma address(for adev) that is not correct with IOMMU enabled, or even in general. The patch releases and unlocks allocated system pages based on where migration failed by struct page* of sys ram pages. Also dma_unmap correspodent system ram pages at error path. Cc: stable@vger.kernel.org Signed-off-by: Xiaogang Chen Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 45 ++++++++++++++++-------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index f5af1dd3b70e..a6bb41fdc8c1 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -258,15 +258,6 @@ svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr) return page; } -static void svm_migrate_put_sys_page(unsigned long addr) -{ - struct page *page; - - page = pfn_to_page(addr >> PAGE_SHIFT); - unlock_page(page); - put_page(page); -} - static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate) { unsigned long mpages = 0; @@ -591,9 +582,10 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, dma_addr_t *scratch, u64 npages) { struct device *dev = adev->dev; - u64 *src; + struct page *dpage = NULL; dma_addr_t *dst; - struct page *dpage; + u64 *src; + u64 i = 0, j; u64 addr; int r = 0; @@ -647,6 +639,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, r = dma_mapping_error(dev, dst[i]); if (r) { dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r); + dst[i] = 0; goto out_oom; } @@ -654,17 +647,39 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, dst[i] >> PAGE_SHIFT, page_to_pfn(dpage)); migrate->dst[i] = migrate_pfn(page_to_pfn(dpage)); + + dpage = NULL; j++; } - r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j, - FROM_VRAM_TO_RAM, mfence); - + if (j > 0) + r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j, + FROM_VRAM_TO_RAM, mfence); out_oom: if (r) { pr_debug("failed %d copy to ram\n", r); + + /* first release current dpage when dma_map_page fail */ + if (dpage) { + unlock_page(dpage); + put_page(dpage); + } + + /* release previous allocated sys pages and unmap dma address */ while (i--) { - svm_migrate_put_sys_page(dst[i]); + + if (dst[i]) { + dma_unmap_page(dev, dst[i], PAGE_SIZE, + DMA_BIDIRECTIONAL); + dst[i] = 0; + } + + dpage = migrate_pfn_to_page(migrate->dst[i]); + if (!dpage) + continue; + + unlock_page(dpage); + put_page(dpage); migrate->dst[i] = 0; } } From 520e345ffe05aabef1db82beda4288afb1757ff2 Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Sun, 23 Aug 2026 15:47:15 -0500 Subject: [PATCH 13/18] drm/amdkfd: Fix the case that vm range is hole at svm_migrate_copy_to_vram When migration vm range is hole at cpu side(MIGRATE_PFN_MIGRATE set + MIGRATE_PFN_VALID unset) driver still allocates device pages. There is no dma map of src pages and migration. j is 0 and svm_migrate_copy_memory_gart() will return an uninitialized r. That can trigger out_free_vram_pages to drop all VRAM just set up. Initialize r and only call the last svm_migrate_copy_memory_gart if j > 0. Current code postponed the last page to the final copy. This patch flushes on the last page when reach to the end of current drm_buddy_block; avoids another svm_migrate_copy_memory_gart. Cc: stable@vger.kernel.org Signed-off-by: Xiaogang Chen Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index a6bb41fdc8c1..253365a8257e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -134,7 +134,7 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys, u64 gart_s, gart_d; struct dma_fence *next; u64 size; - int r; + int r = 0; ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); entity = &adev->mman.move_entities[0]; @@ -284,7 +284,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, dma_addr_t *src; u64 *dst; u64 i, j; - int r; + int r = 0; pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start, prange->last, ttm_res_offset); @@ -310,6 +310,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, DMA_BIDIRECTIONAL); r = dma_mapping_error(dev, src[i]); if (r) { + src[i] = 0; dev_err(dev, "%s: fail %d dma_map_page\n", __func__, r); goto out_free_vram_pages; @@ -334,7 +335,8 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n", src[i] >> PAGE_SHIFT, page_to_pfn(spage)); - if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) { + /* accumulated j + 1 pages reach end of current drm_buddy_block */ + if (j + 1 >= (cursor.size >> PAGE_SHIFT)) { r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j + 1, FROM_RAM_TO_VRAM, @@ -348,7 +350,8 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, } } - r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j, + if (j > 0) + r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j, FROM_RAM_TO_VRAM, mfence); out_free_vram_pages: From a04ea08ddb516f9f21f17574b6a2e7b540dfadf8 Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Fri, 7 Aug 2026 09:46:22 +0800 Subject: [PATCH 14/18] drm/amdgpu/userq: fix lock missing for userq fence error set amdgpu_userq_fence_driver() and amdgpu_userq_fence_driver_destroy() don't acquire the dma_fence spinlock, so locking the dma_fence lock before test the signaled state and set error state to avoid missing lock assert error. Signed-off-by: Prike Liang Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index a33dbe978798..4b023e024d9f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -191,14 +191,15 @@ void amdgpu_userq_fence_driver_destroy(struct kref *ref) struct dma_fence *f; spin_lock_irqsave(&fence_drv->fence_list_lock, flags); + lockdep_assert_held(&fence_drv->fence_list_lock); list_for_each_entry_safe(fence, tmp, &fence_drv->fences, link) { f = &fence->base; - - if (!dma_fence_is_signaled(f)) { + spin_lock(dma_fence_spinlock(f)); + if (!dma_fence_is_signaled_locked(f)) { dma_fence_set_error(f, -ECANCELED); - dma_fence_signal(f); + dma_fence_signal_locked(f); } - + spin_unlock(dma_fence_spinlock(f)); list_del(&fence->link); dma_fence_put(f); } @@ -423,11 +424,16 @@ amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence, struct dma_fence *f; spin_lock_irqsave(&fence_drv->fence_list_lock, flags); - + lockdep_assert_held(&fence_drv->fence_list_lock); f = rcu_dereference_protected(&fence->base, lockdep_is_held(&fence_drv->fence_list_lock)); - if (f && !dma_fence_is_signaled_locked(f)) - dma_fence_set_error(f, error); + if (f) { + /* nest f->lock inside fence_list_lock */ + spin_lock(dma_fence_spinlock(f)); + if (!dma_fence_is_signaled_locked(f)) + dma_fence_set_error(f, error); + spin_unlock(dma_fence_spinlock(f)); + } spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags); } From 6aa530642f95d5c48aa336416f94a35e7949b647 Mon Sep 17 00:00:00 2001 From: Vladimir Marioukhine Date: Wed, 12 Aug 2026 12:58:12 -0400 Subject: [PATCH 15/18] drm/amdkfd: guard against NULL restore_mqd in CRIU queue restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both create_queue_cpsch() and create_queue_nocpsch() unconditionally call mqd_mgr->restore_mqd() when a CRIU restore is in progress (qd != NULL), with no NULL guard. On any system where restore_mqd is not implemented for the given queue type, a user holding CAP_CHECKPOINT_RESTORE can trigger a kernel NULL pointer dereference and panic the machine by issuing KFD_IOC_CRIU_OP_RESTORE with a crafted queue restore object. Note that checkpoint_mqd is likewise unimplemented on GFX12, so no legitimate CRIU image can reach this path — only a hand-crafted restore payload. Add a NULL guard for restore_mqd immediately after mqd_mgr is resolved, unwinding via the existing error labels and returning -EOPNOTSUPP if the callback is not implemented. This mirrors the existing checkpoint_mqd guard in checkpoint_mqd(). Fixes: 48f0bdf4e38e ("drm/amdkfd: Added MQD manager files for GFX12.") Cc: stable@vger.kernel.org Signed-off-by: Vladimir Marioukhine Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index a23384571193..4bc947c3bd0d 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -769,6 +769,11 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm, mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( q->properties.type)]; + if (qd && !mqd_mgr->restore_mqd) { + pr_debug("restore_mqd not implemented for this GPU\n"); + retval = -EOPNOTSUPP; + goto deallocate_vmid; + } if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { retval = allocate_hqd(dqm, q); if (retval) @@ -2236,6 +2241,11 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( q->properties.type)]; + if (qd && !mqd_mgr->restore_mqd) { + pr_debug("restore_mqd not implemented for this GPU\n"); + retval = -EOPNOTSUPP; + goto out_deallocate_doorbell; + } if (q->properties.type == KFD_QUEUE_TYPE_SDMA || q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) From fdc290ff4ab19c7e0dde36c4cd1e2771b61f6bf5 Mon Sep 17 00:00:00 2001 From: Srinivasan Shanmugam Date: Thu, 20 Aug 2026 15:29:43 +0530 Subject: [PATCH 16/18] drm/amd/display: Fix wrong bytes-per-pixel value for dml2_422_packed_10 The pixel format dml2_422_packed_10 needs BytePerPixelDETY set to 8.0/3. But it was accidentally placed in the wrong group that sets it to 4, so the correct value was never used. This caused wrong DET buffer size and bandwidth calculations whenever this format was used. Fix it by moving dml2_422_packed_10 out of the wrong group so it gets the correct value of 8.0/3. Fixes: 7f7d7ea1fa51 ("drm/amd/display: Add new sources for DCN6") Reported-by: Dan Carpenter Cc: Roman Li Cc: Alex Hung Cc: Tom Chung Cc: Aurabindo Pillai Signed-off-by: Srinivasan Shanmugam Reviewed-by: George Zhang Signed-off-by: Alex Deucher --- .../dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c index 05a99c4f761b..38ccf9dab31f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c +++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c @@ -74,7 +74,9 @@ void dcn5_calculate_byte_per_pixel_and_block_sizes( *BytePerPixelDETC = 0; *BytePerPixelY = 8; *BytePerPixelC = 0; - } else if (SourcePixelFormat == dml2_444_32 || SourcePixelFormat == dml2_rgbe || SourcePixelFormat == dml2_422_packed_10 || SourcePixelFormat == dml2_422_packed_12) { + } else if (SourcePixelFormat == dml2_444_32 || + SourcePixelFormat == dml2_rgbe || + SourcePixelFormat == dml2_422_packed_12) { *BytePerPixelDETY = 4; *BytePerPixelDETC = 0; *BytePerPixelY = 4; From 84298acf1c8be2b1b03c0339bf1eb63102c51728 Mon Sep 17 00:00:00 2001 From: Srinivasan Shanmugam Date: Thu, 20 Aug 2026 15:46:01 +0530 Subject: [PATCH 17/18] drm/amd/display: Fix redundant GPUVMEnable checks in dcn6 flip schedule Inside dcn6_calculate_flip_schedule(), GPUVMEnable is already checked in the outer if block. But the same GPUVMEnable is checked again in two inner if blocks inside it. Since GPUVMEnable is always true at that point, the inner else branches that assign meta_row_height are never reached. Remove the redundant inner GPUVMEnable checks and directly assign dpte_row_height, which is always the correct value here. Fixes: 7f7d7ea1fa51 ("drm/amd/display: Add new sources for DCN6") Reported-by: Dan Carpenter Cc: Roman Li Cc: Alex Hung Cc: Tom Chung Cc: Aurabindo Pillai Signed-off-by: Srinivasan Shanmugam Reviewed-by: George Zhang Signed-off-by: Alex Deucher --- .../src/dml2_core/dml2_core_dcn6_calcs_dchub.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c index cae6bee93fe3..a4e2f8604650 100644 --- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c +++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c @@ -569,20 +569,11 @@ void dcn6_calculate_flip_schedule( if (GPUVMEnable) { if (l->dual_plane) { - if (GPUVMEnable) { - l->min_row_height = dpte_row_height; - l->min_row_height_chroma = dpte_row_height_chroma; - } else { - l->min_row_height = meta_row_height; - l->min_row_height_chroma = meta_row_height_chroma; - } + l->min_row_height = dpte_row_height; + l->min_row_height_chroma = dpte_row_height_chroma; l->min_row_time = math_min2(l->min_row_height * LineTime / VRatio, l->min_row_height_chroma * LineTime / VRatioChroma); } else { - if (GPUVMEnable) - l->min_row_height = dpte_row_height; - else - l->min_row_height = meta_row_height; - + l->min_row_height = dpte_row_height; l->min_row_time = l->min_row_height * LineTime / VRatio; } DML_LOG_VERBOSE("DML::%s: min_row_time = %f\n", __func__, l->min_row_time); From 92a9eebd2a1f892fe482154d83f9f1626bc73d3b Mon Sep 17 00:00:00 2001 From: Linkai Gong Date: Wed, 19 Aug 2026 13:47:42 +0800 Subject: [PATCH 18/18] drm/amd/display: fix dc_lock leak on GPU reset error paths On GPU reset, dm_suspend() takes dc_lock and leaves it for dm_resume() to drop. If amdgpu_dm_commit_zero_streams() or dm_dmub_hw_init() fails, the function returns with the lock still held. The matching resume path is then skipped, so every later dc_lock take hangs. Release the cached DC state and unlock before returning the error. Fixes: 3cf7a0bc87f0 ("drm/amd/display: Catch failures for amdgpu_dm_commit_zero_streams()") Fixes: 2b6943df5413 ("drm/amd/display: Pass up errors for reset GPU that fails to init HW") Cc: stable@vger.kernel.org Signed-off-by: Linkai Gong Reviewed-by: Mario Limonciello Signed-off-by: Mario Limonciello Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 284aac4d96bc..ec483276d753 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1589,6 +1589,9 @@ static int dm_suspend(struct amdgpu_ip_block *ip_block) res = amdgpu_dm_commit_zero_streams(dm->dc); if (res != DC_OK) { drm_err(adev_to_drm(adev), "Failed to commit zero streams: %d\n", res); + dc_state_release(dm->cached_dc_state); + dm->cached_dc_state = NULL; + mutex_unlock(&dm->dc_lock); return -EINVAL; } @@ -1884,6 +1887,9 @@ static int dm_resume(struct amdgpu_ip_block *ip_block) r = dm_dmub_hw_init(adev); if (r) { drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r); + dc_state_release(dm->cached_dc_state); + dm->cached_dc_state = NULL; + mutex_unlock(&dm->dc_lock); return r; }