mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 08:15:07 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'amd-drm-next-7.3-2026-08-27' of https://gitlab.freedesktop.org/agd5f/linux into drm-next

Browse Source 
amd-drm-next-7.3-2026-08-27:

amdgpu:
- MES fixes
- Userq fixes
- UVD fix
- VCE 3 fix
- Enforce isolation fix
- HPD fix for VGA/LVDS
- DML fix
- DCN 6 fixes
- DC gpu reset fix

amdkfd:
- CU occupancy for GFX 11
- CU occupancy for GFX 12/12.1
- Queue bounds checking fix
- SVM fixes
- CRIU bounds checking fix

Signed-off-by: Dave Airlie <airlied@redhat.com>

From: Alex Deucher <alexander.deucher@amd.com>
Link: https://patch.msgid.link/20260827203610.3249084-1-alexander.deucher@amd.com
This commit is contained in:
Dave Airlie
2026-08-28 13:27:51 +10:00
parent 1dd31281a6 92a9eebd2a
commit a8b02aa825
19 changed files with 569 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c
View File

@@ -807,6 +807,153 @@ static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev,
return 0;
}
static void lock_spi_csq_mutexes(struct amdgpu_device *adev)
{
mutex_lock(&adev->srbm_mutex);
mutex_lock(&adev->grbm_idx_mutex);
}
static void unlock_spi_csq_mutexes(struct amdgpu_device *adev)
{
mutex_unlock(&adev->grbm_idx_mutex);
mutex_unlock(&adev->srbm_mutex);
}
/**
* get_wave_count: Read device registers to get number of waves in flight for
* a particular queue. The method also returns the doorbell offset associated
* with the queue.
*
* @adev: Handle of device whose registers are to be read
* @queue_idx: Index of queue in the queue-map bit-field
* @queue_cnt: Stores the wave count and doorbell offset for an active queue
* @inst: xcc's instance number on a multi-XCC setup
*/
static void get_wave_count(struct amdgpu_device *adev, int queue_idx,
struct kfd_cu_occupancy *queue_cnt, uint32_t inst)
{
int pipe_idx;
int queue_slot;
unsigned int reg_val;
unsigned int wave_cnt;
/*
* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID
* parameters to read out waves in flight. Get doorbell offset if there are
* non-zero waves in flight.
*/
pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe;
queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe;
soc21_grbm_select(adev, 1, pipe_idx, queue_slot, 0);
reg_val = RREG32_SOC15_IP(GC, SOC15_REG_OFFSET(GC, 0,
regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot);
wave_cnt = reg_val & SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK;
if (wave_cnt != 0) {
queue_cnt->wave_cnt += wave_cnt;
queue_cnt->doorbell_off =
(RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL) &
CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK) >>
CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
}
}
/**
* kgd_gfx_v11_get_cu_occupancy: Reads relevant registers associated with each
* shader engine and aggregates the number of waves that are in flight for the
* process whose pasid is provided as a parameter. The process could have ZERO
* or more queues running and submitting waves to compute units.
*
* @adev: Handle of device from which to get number of waves in flight
* @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset
* for comparison later.
* @max_waves_per_cu: Output parameter updated with maximum number of waves
* possible per Compute Unit
* @inst: xcc's instance number on a multi-XCC setup
*
* Note: It's possible that the device has too many queues (oversubscription)
* in which case a VMID could be remapped to a different PASID. This could lead
* to an inaccurate wave count. Following is a high-level sequence:
* Time T1: vmid = getVmid(); vmid is associated with Pasid P1
* Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2
* In the sequence above wave count obtained from time T1 will be incorrectly
* lost or added to total wave count.
*
* The registers that provide the waves in flight are:
*
* SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a
* queue is slotted, OFF if there is no queue. A process could have ZERO or
* more queues slotted and submitting waves to be run on compute units. Even
* when there is a queue it is possible there could be zero wave fronts, this
* can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem
* command
*
* For each bit that is ON from above:
*
* Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the
* number of waves that are in flight for the queue at specified index. The
* index ranges from 0 to 7.
*
* If non-zero waves are in flight, store the corresponding doorbell offset
* of the queue, along with the wave count.
*
* Determine if the queue belongs to the process by comparing the doorbell
* offset against the process's queues. If it matches, aggregate the wave
* count for the process.
*
* Reading registers referenced above involves programming GRBM appropriately
*/
static void kgd_gfx_v11_get_cu_occupancy(struct amdgpu_device *adev,
struct kfd_cu_occupancy *cu_occupancy,
int *max_waves_per_cu, uint32_t inst)
{
int qidx;
int se_idx;
int se_cnt;
int queue_map;
int max_queue_cnt;
DECLARE_BITMAP(cp_queue_bitmap, AMDGPU_MAX_QUEUES);
lock_spi_csq_mutexes(adev);
soc21_grbm_select(adev, 1, 0, 0, 0);
/*
* Iterate through the shader engines and arrays of the device
* to get number of waves in flight
*/
bitmap_complement(cp_queue_bitmap, adev->gfx.mec_bitmap[0].queue_bitmap,
AMDGPU_MAX_QUEUES);
max_queue_cnt = adev->gfx.mec.num_pipe_per_mec *
adev->gfx.mec.num_queue_per_pipe;
se_cnt = adev->gfx.config.max_shader_engines;
for (se_idx = 0; se_idx < se_cnt; se_idx++) {
amdgpu_gfx_select_se_sh(adev, se_idx, 0, 0xffffffff, inst);
queue_map = RREG32_SOC15(GC, 0,
regSPI_CSQ_WF_ACTIVE_STATUS);
for (qidx = 0; qidx < max_queue_cnt; qidx++) {
/* Skip queues that are not associated with
* compute functions
*/
if (!test_bit(qidx, cp_queue_bitmap))
continue;
if (!(queue_map & (1 << qidx)))
continue;
/* Get number of waves in flight and aggregate them */
get_wave_count(adev, qidx, &cu_occupancy[qidx], inst);
}
}
amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, inst);
soc21_grbm_select(adev, 0, 0, 0, 0);
unlock_spi_csq_mutexes(adev);
/* Update the output parameters and return */
*max_waves_per_cu = adev->gfx.cu_info.simd_per_cu *
adev->gfx.cu_info.max_waves_per_simd;
}
const struct kfd2kgd_calls gfx_v11_kfd2kgd = {
.program_sh_mem_settings = program_sh_mem_settings_v11,
.set_pasid_vmid_mapping = set_pasid_vmid_mapping_v11,
@@ -832,5 +979,6 @@ const struct kfd2kgd_calls gfx_v11_kfd2kgd = {
.clear_address_watch = kgd_gfx_v11_clear_address_watch,
.hqd_get_pq_addr = kgd_gfx_v11_hqd_get_pq_addr,
.hqd_reset = kgd_gfx_v11_hqd_reset,
.get_cu_occupancy = kgd_gfx_v11_get_cu_occupancy,
.hqd_sdma_get_doorbell = kgd_gfx_v11_hqd_sdma_get_doorbell
};

150
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c
View File

@@ -368,6 +368,153 @@ static uint32_t kgd_gfx_v12_hqd_sdma_get_doorbell(struct amdgpu_device *adev,
return 0;
}
static void lock_spi_csq_mutexes(struct amdgpu_device *adev)
{
mutex_lock(&adev->srbm_mutex);
mutex_lock(&adev->grbm_idx_mutex);
}
static void unlock_spi_csq_mutexes(struct amdgpu_device *adev)
{
mutex_unlock(&adev->grbm_idx_mutex);
mutex_unlock(&adev->srbm_mutex);
}
/**
* get_wave_count: Read device registers to get number of waves in flight for
* a particular queue. The method also returns the doorbell offset associated
* with the queue.
*
* @adev: Handle of device whose registers are to be read
* @queue_idx: Index of queue in the queue-map bit-field
* @queue_cnt: Stores the wave count and doorbell offset for an active queue
* @inst: xcc's instance number on a multi-XCC setup
*/
static void get_wave_count(struct amdgpu_device *adev, int queue_idx,
struct kfd_cu_occupancy *queue_cnt, uint32_t inst)
{
int pipe_idx;
int queue_slot;
unsigned int reg_val;
unsigned int wave_cnt;
/*
* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID
* parameters to read out waves in flight. Get doorbell offset if there are
* non-zero waves in flight.
*/
pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe;
queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe;
soc24_grbm_select(adev, 1, pipe_idx, queue_slot, 0);
reg_val = RREG32_SOC15_IP(GC, SOC15_REG_OFFSET(GC, 0,
regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot);
wave_cnt = reg_val & SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK;
if (wave_cnt != 0) {
queue_cnt->wave_cnt += wave_cnt;
queue_cnt->doorbell_off =
(RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL) &
CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK) >>
CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
}
}
/**
* kgd_gfx_v12_get_cu_occupancy: Reads relevant registers associated with each
* shader engine and aggregates the number of waves that are in flight for the
* process whose pasid is provided as a parameter. The process could have ZERO
* or more queues running and submitting waves to compute units.
*
* @adev: Handle of device from which to get number of waves in flight
* @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset
* for comparison later.
* @max_waves_per_cu: Output parameter updated with maximum number of waves
* possible per Compute Unit
* @inst: xcc's instance number on a multi-XCC setup
*
* Note: It's possible that the device has too many queues (oversubscription)
* in which case a VMID could be remapped to a different PASID. This could lead
* to an inaccurate wave count. Following is a high-level sequence:
* Time T1: vmid = getVmid(); vmid is associated with Pasid P1
* Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2
* In the sequence above wave count obtained from time T1 will be incorrectly
* lost or added to total wave count.
*
* The registers that provide the waves in flight are:
*
* SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a
* queue is slotted, OFF if there is no queue. A process could have ZERO or
* more queues slotted and submitting waves to be run on compute units. Even
* when there is a queue it is possible there could be zero wave fronts, this
* can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem
* command
*
* For each bit that is ON from above:
*
* Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the
* number of waves that are in flight for the queue at specified index. The
* index ranges from 0 to 7.
*
* If non-zero waves are in flight, store the corresponding doorbell offset
* of the queue, along with the wave count.
*
* Determine if the queue belongs to the process by comparing the doorbell
* offset against the process's queues. If it matches, aggregate the wave
* count for the process.
*
* Reading registers referenced above involves programming GRBM appropriately
*/
static void kgd_gfx_v12_get_cu_occupancy(struct amdgpu_device *adev,
struct kfd_cu_occupancy *cu_occupancy,
int *max_waves_per_cu, uint32_t inst)
{
int qidx;
int se_idx;
int se_cnt;
int queue_map;
int max_queue_cnt;
DECLARE_BITMAP(cp_queue_bitmap, AMDGPU_MAX_QUEUES);
lock_spi_csq_mutexes(adev);
soc24_grbm_select(adev, 1, 0, 0, 0);
/*
* Iterate through the shader engines and arrays of the device
* to get number of waves in flight
*/
bitmap_complement(cp_queue_bitmap, adev->gfx.mec_bitmap[0].queue_bitmap,
AMDGPU_MAX_QUEUES);
max_queue_cnt = adev->gfx.mec.num_pipe_per_mec *
adev->gfx.mec.num_queue_per_pipe;
se_cnt = adev->gfx.config.max_shader_engines;
for (se_idx = 0; se_idx < se_cnt; se_idx++) {
amdgpu_gfx_select_se_sh(adev, se_idx, 0, 0xffffffff, inst);
queue_map = RREG32_SOC15(GC, 0,
regSPI_CSQ_WF_ACTIVE_STATUS);
for (qidx = 0; qidx < max_queue_cnt; qidx++) {
/* Skip queues that are not associated with
* compute functions
*/
if (!test_bit(qidx, cp_queue_bitmap))
continue;
if (!(queue_map & (1 << qidx)))
continue;
/* Get number of waves in flight and aggregate them */
get_wave_count(adev, qidx, &cu_occupancy[qidx], inst);
}
}
amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, inst);
soc24_grbm_select(adev, 0, 0, 0, 0);
unlock_spi_csq_mutexes(adev);
/* Update the output parameters and return */
*max_waves_per_cu = adev->gfx.cu_info.simd_per_cu *
adev->gfx.cu_info.max_waves_per_simd;
}
const struct kfd2kgd_calls gfx_v12_kfd2kgd = {
.init_interrupts = init_interrupts_v12,
.hqd_dump = hqd_dump_v12,
@@ -381,5 +528,6 @@ const struct kfd2kgd_calls gfx_v12_kfd2kgd = {
.set_wave_launch_mode = kgd_gfx_v12_set_wave_launch_mode,
.set_address_watch = kgd_gfx_v12_set_address_watch,
.clear_address_watch = kgd_gfx_v12_clear_address_watch,
.hqd_sdma_get_doorbell = kgd_gfx_v12_hqd_sdma_get_doorbell
.hqd_sdma_get_doorbell = kgd_gfx_v12_hqd_sdma_get_doorbell,
.get_cu_occupancy = kgd_gfx_v12_get_cu_occupancy,
};

150
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
View File

@@ -371,6 +371,153 @@ static uint32_t kgd_gfx_v12_1_hqd_sdma_get_doorbell(struct amdgpu_device *adev,
return 0;
}
static void lock_spi_csq_mutexes(struct amdgpu_device *adev)
{
mutex_lock(&adev->srbm_mutex);
mutex_lock(&adev->grbm_idx_mutex);
}
static void unlock_spi_csq_mutexes(struct amdgpu_device *adev)
{
mutex_unlock(&adev->grbm_idx_mutex);
mutex_unlock(&adev->srbm_mutex);
}
/**
* get_wave_count: Read device registers to get number of waves in flight for
* a particular queue. The method also returns the doorbell offset associated
* with the queue.
*
* @adev: Handle of device whose registers are to be read
* @queue_idx: Index of queue in the queue-map bit-field
* @queue_cnt: Stores the wave count and doorbell offset for an active queue
* @inst: xcc's instance number on a multi-XCC setup
*/
static void get_wave_count(struct amdgpu_device *adev, int queue_idx,
struct kfd_cu_occupancy *queue_cnt, uint32_t inst)
{
int pipe_idx;
int queue_slot;
unsigned int reg_val;
unsigned int wave_cnt;
/*
* Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID
* parameters to read out waves in flight. Get doorbell offset if there are
* non-zero waves in flight.
*/
pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe;
queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe;
amdgpu_gfx_select_me_pipe_q(adev, 1, pipe_idx, queue_slot, 0, inst);
reg_val = RREG32_SOC15_IP(GC, SOC15_REG_OFFSET(GC, GET_INST(GC, inst),
regSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot);
wave_cnt = reg_val & SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK;
if (wave_cnt != 0) {
queue_cnt->wave_cnt += wave_cnt;
queue_cnt->doorbell_off =
(RREG32_SOC15(GC, GET_INST(GC, inst), regCP_HQD_PQ_DOORBELL_CONTROL) &
CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK) >>
CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
}
}
/**
* kgd_gfx_v12_1_get_cu_occupancy: Reads relevant registers associated with
* each shader engine and aggregates the number of waves that are in flight
* for the process whose pasid is provided as a parameter. The process could
* have ZERO or more queues running and submitting waves to compute units.
*
* @adev: Handle of device from which to get number of waves in flight
* @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset
* for comparison later.
* @max_waves_per_cu: Output parameter updated with maximum number of waves
* possible per Compute Unit
* @inst: xcc's instance number on a multi-XCC setup
*
* Note: It's possible that the device has too many queues (oversubscription)
* in which case a VMID could be remapped to a different PASID. This could lead
* to an inaccurate wave count. Following is a high-level sequence:
* Time T1: vmid = getVmid(); vmid is associated with Pasid P1
* Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2
* In the sequence above wave count obtained from time T1 will be incorrectly
* lost or added to total wave count.
*
* The registers that provide the waves in flight are:
*
* SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a
* queue is slotted, OFF if there is no queue. A process could have ZERO or
* more queues slotted and submitting waves to be run on compute units. Even
* when there is a queue it is possible there could be zero wave fronts, this
* can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem
* command
*
* For each bit that is ON from above:
*
* Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the
* number of waves that are in flight for the queue at specified index. The
* index ranges from 0 to 7.
*
* If non-zero waves are in flight, store the corresponding doorbell offset
* of the queue, along with the wave count.
*
* Determine if the queue belongs to the process by comparing the doorbell
* offset against the process's queues. If it matches, aggregate the wave
* count for the process.
*
* Reading registers referenced above involves programming GRBM appropriately
*/
static void kgd_gfx_v12_1_get_cu_occupancy(struct amdgpu_device *adev,
struct kfd_cu_occupancy *cu_occupancy,
int *max_waves_per_cu, uint32_t inst)
{
int qidx;
int se_idx;
int se_cnt;
int queue_map;
int max_queue_cnt;
DECLARE_BITMAP(cp_queue_bitmap, AMDGPU_MAX_QUEUES);
lock_spi_csq_mutexes(adev);
amdgpu_gfx_select_me_pipe_q(adev, 1, 0, 0, 0, inst);
/*
* Iterate through the shader engines and arrays of the device
* to get number of waves in flight
*/
bitmap_complement(cp_queue_bitmap, adev->gfx.mec_bitmap[0].queue_bitmap,
AMDGPU_MAX_QUEUES);
max_queue_cnt = adev->gfx.mec.num_pipe_per_mec *
adev->gfx.mec.num_queue_per_pipe;
se_cnt = adev->gfx.config.max_shader_engines;
for (se_idx = 0; se_idx < se_cnt; se_idx++) {
amdgpu_gfx_select_se_sh(adev, se_idx, 0, 0xffffffff, inst);
queue_map = RREG32_SOC15(GC, GET_INST(GC, inst),
regSPI_CSQ_WF_ACTIVE_STATUS);
for (qidx = 0; qidx < max_queue_cnt; qidx++) {
/* Skip queues that are not associated with
* compute functions
*/
if (!test_bit(qidx, cp_queue_bitmap))
continue;
if (!(queue_map & (1 << qidx)))
continue;
/* Get number of waves in flight and aggregate them */
get_wave_count(adev, qidx, &cu_occupancy[qidx], inst);
}
}
amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, inst);
amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, inst);
unlock_spi_csq_mutexes(adev);
/* Update the output parameters and return */
*max_waves_per_cu = adev->gfx.cu_info.simd_per_cu *
adev->gfx.cu_info.max_waves_per_simd;
}
const struct kfd2kgd_calls gfx_v12_1_kfd2kgd = {
.init_interrupts = init_interrupts_v12_1,
.hqd_dump = hqd_dump_v12_1,
@@ -384,5 +531,6 @@ const struct kfd2kgd_calls gfx_v12_1_kfd2kgd = {
.set_wave_launch_mode = kgd_gfx_v12_1_set_wave_launch_mode,
.set_address_watch = kgd_gfx_v12_1_set_address_watch,
.clear_address_watch = kgd_gfx_v12_1_clear_address_watch,
.hqd_sdma_get_doorbell = kgd_gfx_v12_1_hqd_sdma_get_doorbell
.hqd_sdma_get_doorbell = kgd_gfx_v12_1_hqd_sdma_get_doorbell,
.get_cu_occupancy = kgd_gfx_v12_1_get_cu_occupancy
};

7
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
View File

@@ -1795,6 +1795,12 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
size >>= 1;
aligned_size = PAGE_ALIGN(size);
/* reject AQL queue with size < 2 */
if (!aligned_size) {
ret = -EINVAL;
goto err_alignment_size;
}
(*mem)->alloc_flags = flags;
amdgpu_sync_create(&(*mem)->sync);
@@ -1886,6 +1892,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id);
err_reserve_limit:
amdgpu_sync_free(&(*mem)->sync);
err_alignment_size:
mutex_destroy(&(*mem)->lock);
if (gobj)
drm_gem_object_put(gobj);

5
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
View File

@@ -6687,8 +6687,8 @@ struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev,
struct amdgpu_ring *ring,
struct amdgpu_job *job)
{
struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id];
struct drm_sched_fence *f = job->base.s_fence;
struct amdgpu_isolation *isolation;
struct dma_fence *dep;
void *owner;
int r;
@@ -6701,6 +6701,9 @@ struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev,
ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
return NULL;
isolation = &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ?
0 : ring->xcp_id];
/*
* All submissions where enforce isolation is false are handled as if
* they come from a single client. Use ~0l as the owner to distinct it

24
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
View File

@@ -1047,6 +1047,30 @@ amdgpu_userq_vm_validate_and_restore_queue(struct amdgpu_userq_mgr *uq_mgr)
drm_exec_retry_on_contention(&exec);
if (unlikely(ret))
goto unlock_all;
/*
* WPTR BOs are VM-mapped, but each BO has its own reservation
* object. Lock them into this drm_exec ww context so the later
* amdgpu_bo_gpu_offset() reads are done with the BO resv locked.
*/
xa_for_each(&uq_mgr->userq_xa, tmp_key, queue) {
struct ttm_operation_ctx wptr_ctx = { false, false };
bo = queue->wptr_obj.obj;
if (!bo)
continue;
ret = drm_exec_prepare_obj(&exec, &bo->tbo.base,
TTM_NUM_MOVE_FENCES + 1);
drm_exec_retry_on_contention(&exec);
if (unlikely(ret))
goto unlock_all;
amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
ret = ttm_bo_validate(&bo->tbo, &bo->placement, &wptr_ctx);
if (unlikely(ret))
goto unlock_all;
}
}
if (invalidated) {

20
drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
View File

@@ -191,14 +191,15 @@ void amdgpu_userq_fence_driver_destroy(struct kref *ref)
struct dma_fence *f;
spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
lockdep_assert_held(&fence_drv->fence_list_lock);
list_for_each_entry_safe(fence, tmp, &fence_drv->fences, link) {
f = &fence->base;
if (!dma_fence_is_signaled(f)) {
spin_lock(dma_fence_spinlock(f));
if (!dma_fence_is_signaled_locked(f)) {
dma_fence_set_error(f, -ECANCELED);
dma_fence_signal(f);
dma_fence_signal_locked(f);
}
spin_unlock(dma_fence_spinlock(f));
list_del(&fence->link);
dma_fence_put(f);
}
@@ -423,11 +424,16 @@ amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
struct dma_fence *f;
spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
lockdep_assert_held(&fence_drv->fence_list_lock);
f = rcu_dereference_protected(&fence->base,
lockdep_is_held(&fence_drv->fence_list_lock));
if (f && !dma_fence_is_signaled_locked(f))
dma_fence_set_error(f, error);
if (f) {
/* nest f->lock inside fence_list_lock */
spin_lock(dma_fence_spinlock(f));
if (!dma_fence_is_signaled_locked(f))
dma_fence_set_error(f, error);
spin_unlock(dma_fence_spinlock(f));
}
spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
}

3
drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
View File

@@ -517,7 +517,8 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
}
memset_io(ptr, 0, size);
/* to restore uvd fence seq */
amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring, NULL);
if (adev->uvd.inst[i].ring.fence_drv.initialized)
amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring, NULL);
}
}
return 0;

4
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
View File

@@ -776,7 +776,9 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
bool *emit_gds_needed)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id];
struct amdgpu_isolation *isolation =
&adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ?
0 : ring->xcp_id];
unsigned vmhub = ring->vm_hub;
struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];

2
drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
View File

@@ -146,7 +146,7 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
if (mes->use_rs64mem) {
if (!uq_mgr->proc_ctx_allocated) {
if (!uq_mgr->proc_ctx_allocated) {
r = amdgpu_mes_alloc_proc_ctx_index(mes, &uq_mgr->proc_ctx_array_index);
if (r) {
DRM_ERROR("Failed to allocate userq process index err:%d\n", r);

2
drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
View File

@@ -891,7 +891,7 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_phys_funcs = {
static const struct amdgpu_ring_funcs vce_v3_0_ring_vm_funcs = {
.type = AMDGPU_RING_TYPE_VCE,
.align_mask = 0xf,
.align_mask = 0x1f,
.nop = VCE_CMD_NO_OP,
.support_64bit_ptrs = false,
.no_user_fence = true,

3
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
View File

@@ -1200,7 +1200,8 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
if (flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM)
size >>= 1;
atomic64_add(PAGE_ALIGN(size), &pdd->vram_usage);
size = PAGE_ALIGN(size);
atomic64_add(size, &pdd->vram_usage);
}
mutex_unlock(&p->mutex);

10
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
View File

@@ -769,6 +769,11 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
q->properties.type)];
if (qd && !mqd_mgr->restore_mqd) {
pr_debug("restore_mqd not implemented for this GPU\n");
retval = -EOPNOTSUPP;
goto deallocate_vmid;
}
if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
retval = allocate_hqd(dqm, q);
if (retval)
@@ -2236,6 +2241,11 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
q->properties.type)];
if (qd && !mqd_mgr->restore_mqd) {
pr_debug("restore_mqd not implemented for this GPU\n");
retval = -EOPNOTSUPP;
goto out_deallocate_doorbell;
}
if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)

56
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
View File

@@ -134,7 +134,7 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
u64 gart_s, gart_d;
struct dma_fence *next;
u64 size;
int r;
int r = 0;
ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
entity = &adev->mman.move_entities[0];
@@ -258,15 +258,6 @@ svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr)
return page;
}
static void svm_migrate_put_sys_page(unsigned long addr)
{
struct page *page;
page = pfn_to_page(addr >> PAGE_SHIFT);
unlock_page(page);
put_page(page);
}
static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate)
{
unsigned long mpages = 0;
@@ -293,7 +284,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
dma_addr_t *src;
u64 *dst;
u64 i, j;
int r;
int r = 0;
pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start,
prange->last, ttm_res_offset);
@@ -319,6 +310,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
DMA_BIDIRECTIONAL);
r = dma_mapping_error(dev, src[i]);
if (r) {
src[i] = 0;
dev_err(dev, "%s: fail %d dma_map_page\n",
__func__, r);
goto out_free_vram_pages;
@@ -343,7 +335,8 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n",
src[i] >> PAGE_SHIFT, page_to_pfn(spage));
if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) {
/* accumulated j + 1 pages reach end of current drm_buddy_block */
if (j + 1 >= (cursor.size >> PAGE_SHIFT)) {
r = svm_migrate_copy_memory_gart(adev, src + i - j,
dst + i - j, j + 1,
FROM_RAM_TO_VRAM,
@@ -357,7 +350,8 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
}
}
r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
if (j > 0)
r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
FROM_RAM_TO_VRAM, mfence);
out_free_vram_pages:
@@ -591,9 +585,10 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
dma_addr_t *scratch, u64 npages)
{
struct device *dev = adev->dev;
u64 *src;
struct page *dpage = NULL;
dma_addr_t *dst;
struct page *dpage;
u64 *src;
u64 i = 0, j;
u64 addr;
int r = 0;
@@ -647,6 +642,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
r = dma_mapping_error(dev, dst[i]);
if (r) {
dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r);
dst[i] = 0;
goto out_oom;
}
@@ -654,17 +650,39 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
dpage = NULL;
j++;
}
r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j,
FROM_VRAM_TO_RAM, mfence);
if (j > 0)
r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j,
FROM_VRAM_TO_RAM, mfence);
out_oom:
if (r) {
pr_debug("failed %d copy to ram\n", r);
/* first release current dpage when dma_map_page fail */
if (dpage) {
unlock_page(dpage);
put_page(dpage);
}
/* release previous allocated sys pages and unmap dma address */
while (i--) {
svm_migrate_put_sys_page(dst[i]);
if (dst[i]) {
dma_unmap_page(dev, dst[i], PAGE_SIZE,
DMA_BIDIRECTIONAL);
dst[i] = 0;
}
dpage = migrate_pfn_to_page(migrate->dst[i]);
if (!dpage)
continue;
unlock_page(dpage);
put_page(dpage);
migrate->dst[i] = 0;
}
}

6
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
View File

@@ -1589,6 +1589,9 @@ static int dm_suspend(struct amdgpu_ip_block *ip_block)
res = amdgpu_dm_commit_zero_streams(dm->dc);
if (res != DC_OK) {
drm_err(adev_to_drm(adev), "Failed to commit zero streams: %d\n", res);
dc_state_release(dm->cached_dc_state);
dm->cached_dc_state = NULL;
mutex_unlock(&dm->dc_lock);
return -EINVAL;
}
@@ -1884,6 +1887,9 @@ static int dm_resume(struct amdgpu_ip_block *ip_block)
r = dm_dmub_hw_init(adev);
if (r) {
drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r);
dc_state_release(dm->cached_dc_state);
dm->cached_dc_state = NULL;
mutex_unlock(&dm->dc_lock);
return r;
}

8
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
View File

@@ -1735,7 +1735,9 @@ int amdgpu_dm_register_hpd_handlers(struct amdgpu_device *adev)
if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
int_params.irq_source < DC_IRQ_SOURCE_HPD1 ||
int_params.irq_source > DC_IRQ_SOURCE_HPD6) {
drm_err(adev_to_drm(adev), "Failed to register hpd irq!\n");
drm_err(adev_to_drm(adev),
"Failed to register hpd irq %u for %s!\n",
int_params.irq_source, connector->name);
return -EINVAL;
}
@@ -1753,7 +1755,9 @@ int amdgpu_dm_register_hpd_handlers(struct amdgpu_device *adev)
if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
int_params.irq_source < DC_IRQ_SOURCE_HPD1RX ||
int_params.irq_source > DC_IRQ_SOURCE_HPD6RX) {
drm_err(adev_to_drm(adev), "Failed to register hpd rx irq!\n");
drm_err(adev_to_drm(adev),
"Failed to register hpd rx irq %u for %s!\n",
int_params.irq_source, connector->name);
return -EINVAL;
}

4
drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c
View File

@@ -74,7 +74,9 @@ void dcn5_calculate_byte_per_pixel_and_block_sizes(
*BytePerPixelDETC = 0;
*BytePerPixelY = 8;
*BytePerPixelC = 0;
} else if (SourcePixelFormat == dml2_444_32 || SourcePixelFormat == dml2_rgbe || SourcePixelFormat == dml2_422_packed_10 || SourcePixelFormat == dml2_422_packed_12) {
} else if (SourcePixelFormat == dml2_444_32 ||
SourcePixelFormat == dml2_rgbe ||
SourcePixelFormat == dml2_422_packed_12) {
*BytePerPixelDETY = 4;
*BytePerPixelDETC = 0;
*BytePerPixelY = 4;

15
drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c
View File

@@ -569,20 +569,11 @@ void dcn6_calculate_flip_schedule(
if (GPUVMEnable) {
if (l->dual_plane) {
if (GPUVMEnable) {
l->min_row_height = dpte_row_height;
l->min_row_height_chroma = dpte_row_height_chroma;
} else {
l->min_row_height = meta_row_height;
l->min_row_height_chroma = meta_row_height_chroma;
}
l->min_row_height = dpte_row_height;
l->min_row_height_chroma = dpte_row_height_chroma;
l->min_row_time = math_min2(l->min_row_height * LineTime / VRatio, l->min_row_height_chroma * LineTime / VRatioChroma);
} else {
if (GPUVMEnable)
l->min_row_height = dpte_row_height;
else
l->min_row_height = meta_row_height;
l->min_row_height = dpte_row_height;
l->min_row_time = l->min_row_height * LineTime / VRatio;
}
DML_LOG_VERBOSE("DML::%s: min_row_time = %f\n", __func__, l->min_row_time);

2
drivers/gpu/drm/amd/display/dc/link/link_factory.c
View File

@@ -639,7 +639,7 @@ static bool construct_phy(struct dc_link *link,
DC_LOG_DC("BIOS object table - hpd_gpio id: %d", enc_init_data.hpd_gpio->id);
DC_LOG_DC("BIOS object table - hpd_gpio en: %d", enc_init_data.hpd_gpio->en);
} else {
} else if (link->ctx->dce_version > DCN_VERSION_4_01) {
struct graphics_object_hpd_info hpd_info;
if (link->ctx->dc_bios->funcs->get_hpd_info(link->ctx->dc_bios, link->link_id, &hpd_info) == BP_RESULT_OK) {