mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-07-25 09:41:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

drm/panthor: Add a GEM shrinker

Browse Source 
This implementation is losely based on the MSM shrinker, and it's
relying on the drm_gpuvm eviction/validation infrastructure.

Right now we only support swapout/eviction, but we could add an extra
flag to specify when buffer content doesn't need to be preserved to
avoid the swapout/swapin dance.

Locking is a bit of a nightmare, but using _trylock() all the way in
the reclaim path seems to make lockdep happy. And yes, we might be
missing opportunities to reclaim when the system is under heavy GPU
load/heavy memory pressure/heavy GPU VM activity, but that's better
than no reclaim at all.

v2:
- Move gpu_mapped_shared next to the mmapped LRU
- Add a bunch of missing is_[vm_bo,vma]_evicted() tests
- Only test mmap_count to check if a BO is mmaped
- Remove stale comment about shrinker not being a thing
- Allow pin_count to be non-zero in panthor_gem_swapin_locked()
- Fix panthor_gem_sync() to check for BO residency before doing the CPU sync
- Fix the value returned by panthor_gem_shrinker_count() in case some
  memory has been released
- Check drmm_mutex_init() ret code
- Explicitly mention that PANTHOR_GEM_UNRECLAIMABLE is the initial state
  of all BOs

v3:
- Make panthor_gem_try_evict() static
- Collect {A,R}-bs

v4:
- Update the reclaim_state in panthor_gem_mmap()
- Don't reclaim GPU-mapped BOs if can_block() returns false
- Skip evicited vm_bos in panthor_vm_update_bo_reclaim_lru_locked() to
  avoid spurious WARN_ON()s
- Explain why we have to do this
  select_evicted_vma/repopulate_evicted_vma dance

v5:
- Properly report the reclaimable size in panthor_gem_debugfs_print_bos()
- Check panthor_vm_lock_region() errors in
  panthor_vm_evict_bo_mappings_locked()
- Fix lock order inversion (dma_resv_wait_timeout() inside gpuva.lock)

v6:
- Don't remap if the unmapped VMA is evicted.
- Drop a stale comment in panthor_gem_dev_map_get_sgt_locked()
- s/PANTHOR_GEM_GPU_MAPPED_PRIVATE/PANTHOR_GEM_GPU_MAPPED_SINGLE_VM/
- s/PANTHOR_GEM_GPU_MAPPED_SHARED/PANTHOR_GEM_GPU_MAPPED_MULTI_VM/
- Just count the number of vm_bo to determine the reclaim state in
  is_gpu_mapped()
- Drop a redundant panthor_gem_backing_get_pages_locked() call in
  panthor_gem_swapin_locked()
- Add more comments to panthor_vm_evict_bo_mappings_locked() to
  convince Claude it's actually safe.
- Add a comment in panthor_gem_shrinker_count() to mention the race
  and hopefully clear Claude's concerns.
- Rework the "is-still-in-list" check we have in
  panthor_mmu_reclaim_priv_bos() to address Claude's concerns
- Don't call panthor_vm_unlock_region() if panthor_vm_lock_region()
  fails in remap_evicted_vma(). Was harmless, but confusing, as
  pointed out by Claude
- Fix can_block() to allow blocking on KSWAPD_RECLAIM-only in a kswapd
  context
- Fix a lockdep warning when the last ref on a BO is released in the
  shrinker path, where the resv lock can't be acquired other than with
  a try_lock()

v7:
- Skip drm_gpuvm_bo_deferred_cleanup() when repopulating VMAs to fix a
  deadlock

Signed-off-by: Akash Goel <akash.goel@arm.com>
Acked-by: Liviu Dudau <liviu.dudau@arm.com>
Reviewed-by: Steven Price <steven.price@arm.com>
Link: https://patch.msgid.link/20260401134854.2275433-11-boris.brezillon@collabora.com
Co-developed-by: Boris Brezillon <boris.brezillon@collabora.com>
Signed-off-by: Boris Brezillon <boris.brezillon@collabora.com>
This commit is contained in:
Akash Goel
2026-04-01 15:48:54 +02:00
committed by Boris Brezillon
parent 3218aae545
commit fb42964e2a
6 changed files with 1027 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
drivers/gpu/drm/panthor/panthor_device.c
View File

@@ -2,6 +2,7 @@
/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
/* Copyright 2023 Collabora ltd. */
/* Copyright 2025 ARM Limited. All rights reserved. */
#include <linux/clk.h>
#include <linux/mm.h>
@@ -122,6 +123,7 @@ void panthor_device_unplug(struct panthor_device *ptdev)
panthor_sched_unplug(ptdev);
panthor_fw_unplug(ptdev);
panthor_mmu_unplug(ptdev);
panthor_gem_shrinker_unplug(ptdev);
panthor_gpu_unplug(ptdev);
panthor_pwr_unplug(ptdev);
@@ -291,10 +293,14 @@ int panthor_device_init(struct panthor_device *ptdev)
if (ret)
goto err_unplug_gpu;
ret = panthor_mmu_init(ptdev);
ret = panthor_gem_shrinker_init(ptdev);
if (ret)
goto err_unplug_gpu;
ret = panthor_mmu_init(ptdev);
if (ret)
goto err_unplug_shrinker;
ret = panthor_fw_init(ptdev);
if (ret)
goto err_unplug_mmu;
@@ -326,6 +332,9 @@ int panthor_device_init(struct panthor_device *ptdev)
err_unplug_mmu:
panthor_mmu_unplug(ptdev);
err_unplug_shrinker:
panthor_gem_shrinker_unplug(ptdev);
err_unplug_gpu:
panthor_gpu_unplug(ptdev);

73
drivers/gpu/drm/panthor/panthor_device.h
View File

@@ -14,6 +14,7 @@
#include <linux/spinlock.h>
#include <drm/drm_device.h>
#include <drm/drm_gem.h>
#include <drm/drm_mm.h>
#include <drm/gpu_scheduler.h>
#include <drm/panthor_drm.h>
@@ -178,6 +179,78 @@ struct panthor_device {
/** @devfreq: Device frequency scaling management data. */
struct panthor_devfreq *devfreq;
/** @reclaim: Reclaim related stuff */
struct {
/** @reclaim.shrinker: Shrinker instance */
struct shrinker *shrinker;
/** @reclaim.lock: Lock protecting all LRUs */
struct mutex lock;
/**
* @reclaim.unused: BOs with unused pages
*
* Basically all buffers that got mmapped, vmapped or GPU mapped and
* then unmapped. There should be no contention on these buffers,
* making them ideal to reclaim.
*/
struct drm_gem_lru unused;
/**
* @reclaim.mmapped: mmap()-ed buffers
*
* Those are relatively easy to reclaim since we don't need user
* agreement, we can simply teardown the mapping and let it fault on
* the next access.
*/
struct drm_gem_lru mmapped;
/**
* @reclaim.gpu_mapped_shared: shared BO LRU list
*
* That's the most tricky BO type to reclaim, because it involves
* tearing down all mappings in all VMs where this BO is mapped,
* which increases the risk of contention and thus decreases the
* likeliness of success.
*/
struct drm_gem_lru gpu_mapped_shared;
/**
* @reclaim.vms: VM LRU list
*
* VMs that have reclaimable BOs only mapped to a single VM are placed
* in this LRU. Reclaiming such BOs implies waiting for VM idleness
* (no in-flight GPU jobs targeting this VM), meaning we can't reclaim
* those if we're in a context where we can't block/sleep.
*/
struct list_head vms;
/**
* @reclaim.gpu_mapped_count: Global counter of pages that are GPU mapped
*
* Allows us to get the number of reclaimable pages without walking
* the vms and gpu_mapped_shared LRUs.
*/
long gpu_mapped_count;
/**
* @reclaim.retry_count: Number of times we ran the shrinker without being
* able to reclaim stuff
*
* Used to stop scanning GEMs when too many attempts were made
* without progress.
*/
atomic_t retry_count;
#ifdef CONFIG_DEBUG_FS
/**
* @reclaim.nr_pages_reclaimed_on_last_scan: Number of pages reclaimed on the last
* shrinker scan
*/
unsigned long nr_pages_reclaimed_on_last_scan;
#endif
} reclaim;
/** @unplug: Device unplug related fields. */
struct {
/** @lock: Lock used to serialize unplug operations. */

508
drivers/gpu/drm/panthor/panthor_gem.c
View File

@@ -2,8 +2,10 @@
/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
/* Copyright 2023 Collabora ltd. */
/* Copyright 2025 Amazon.com, Inc. or its affiliates */
/* Copyright 2025 ARM Limited. All rights reserved. */
#include <linux/cleanup.h>
#include <linux/debugfs.h>
#include <linux/dma-buf.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
@@ -12,6 +14,8 @@
#include <drm/drm_debugfs.h>
#include <drm/drm_file.h>
#include <drm/drm_gpuvm.h>
#include <drm/drm_managed.h>
#include <drm/drm_prime.h>
#include <drm/drm_print.h>
#include <drm/panthor_drm.h>
@@ -114,10 +118,116 @@ should_map_wc(struct panthor_gem_object *bo)
return true;
}
static bool is_gpu_mapped(struct panthor_gem_object *bo,
enum panthor_gem_reclaim_state *state)
{
struct drm_gpuvm_bo *vm_bo;
u32 vm_count = 0;
drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) {
/* Skip evicted GPU mappings. */
if (vm_bo->evicted)
continue;
if (vm_count++) {
*state = PANTHOR_GEM_GPU_MAPPED_MULTI_VM;
break;
}
*state = PANTHOR_GEM_GPU_MAPPED_SINGLE_VM;
}
return vm_count > 0;
}
static enum panthor_gem_reclaim_state
panthor_gem_evaluate_reclaim_state_locked(struct panthor_gem_object *bo)
{
enum panthor_gem_reclaim_state gpu_mapped_state;
dma_resv_assert_held(bo->base.resv);
lockdep_assert_held(&bo->base.gpuva.lock);
/* If pages have not been allocated, there's nothing to reclaim. */
if (!bo->backing.pages)
return PANTHOR_GEM_UNRECLAIMABLE;
/* If memory is pinned, we prevent reclaim. */
if (refcount_read(&bo->backing.pin_count))
return PANTHOR_GEM_UNRECLAIMABLE;
if (is_gpu_mapped(bo, &gpu_mapped_state))
return gpu_mapped_state;
if (refcount_read(&bo->cmap.mmap_count))
return PANTHOR_GEM_MMAPPED;
return PANTHOR_GEM_UNUSED;
}
void panthor_gem_update_reclaim_state_locked(struct panthor_gem_object *bo,
enum panthor_gem_reclaim_state *old_statep)
{
struct panthor_device *ptdev = container_of(bo->base.dev, struct panthor_device, base);
enum panthor_gem_reclaim_state old_state = bo->reclaim_state;
enum panthor_gem_reclaim_state new_state;
bool was_gpu_mapped, is_gpu_mapped;
if (old_statep)
*old_statep = old_state;
new_state = panthor_gem_evaluate_reclaim_state_locked(bo);
if (new_state == old_state)
return;
was_gpu_mapped = old_state == PANTHOR_GEM_GPU_MAPPED_MULTI_VM ||
old_state == PANTHOR_GEM_GPU_MAPPED_SINGLE_VM;
is_gpu_mapped = new_state == PANTHOR_GEM_GPU_MAPPED_MULTI_VM ||
new_state == PANTHOR_GEM_GPU_MAPPED_SINGLE_VM;
if (is_gpu_mapped && !was_gpu_mapped)
ptdev->reclaim.gpu_mapped_count += bo->base.size >> PAGE_SHIFT;
else if (!is_gpu_mapped && was_gpu_mapped)
ptdev->reclaim.gpu_mapped_count -= bo->base.size >> PAGE_SHIFT;
switch (new_state) {
case PANTHOR_GEM_UNUSED:
drm_gem_lru_move_tail(&ptdev->reclaim.unused, &bo->base);
break;
case PANTHOR_GEM_MMAPPED:
drm_gem_lru_move_tail(&ptdev->reclaim.mmapped, &bo->base);
break;
case PANTHOR_GEM_GPU_MAPPED_SINGLE_VM:
panthor_vm_update_bo_reclaim_lru_locked(bo);
break;
case PANTHOR_GEM_GPU_MAPPED_MULTI_VM:
drm_gem_lru_move_tail(&ptdev->reclaim.gpu_mapped_shared, &bo->base);
break;
case PANTHOR_GEM_UNRECLAIMABLE:
drm_gem_lru_remove(&bo->base);
break;
default:
drm_WARN(&ptdev->base, true, "invalid GEM reclaim state (%d)\n", new_state);
break;
}
bo->reclaim_state = new_state;
}
static void
bo_assert_locked_or_gone(struct panthor_gem_object *bo)
{
/* If the refcount is zero, the BO is being freed, and we
* allow the lock to not be held in that particular case.
*/
if (kref_read(&bo->base.refcount))
dma_resv_assert_held(bo->base.resv);
}
static void
panthor_gem_backing_cleanup_locked(struct panthor_gem_object *bo)
{
dma_resv_assert_held(bo->base.resv);
bo_assert_locked_or_gone(bo);
if (!bo->backing.pages)
return;
@@ -157,28 +267,35 @@ static int panthor_gem_backing_pin_locked(struct panthor_gem_object *bo)
return 0;
ret = panthor_gem_backing_get_pages_locked(bo);
if (!ret)
if (!ret) {
refcount_set(&bo->backing.pin_count, 1);
mutex_lock(&bo->base.gpuva.lock);
panthor_gem_update_reclaim_state_locked(bo, NULL);
mutex_unlock(&bo->base.gpuva.lock);
}
return ret;
}
static void panthor_gem_backing_unpin_locked(struct panthor_gem_object *bo)
{
dma_resv_assert_held(bo->base.resv);
bo_assert_locked_or_gone(bo);
drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base));
if (refcount_dec_and_test(&bo->backing.pin_count)) {
/* We don't release anything when pin_count drops to zero.
* Pages stay there until an explicit cleanup is requested.
*/
mutex_lock(&bo->base.gpuva.lock);
panthor_gem_update_reclaim_state_locked(bo, NULL);
mutex_unlock(&bo->base.gpuva.lock);
}
}
static void
panthor_gem_dev_map_cleanup_locked(struct panthor_gem_object *bo)
{
dma_resv_assert_held(bo->base.resv);
bo_assert_locked_or_gone(bo);
if (!bo->dmap.sgt)
return;
@@ -200,9 +317,6 @@ panthor_gem_dev_map_get_sgt_locked(struct panthor_gem_object *bo)
if (bo->dmap.sgt)
return bo->dmap.sgt;
/* Pages stay around after they've been allocated. At least that stands
* until we add a shrinker.
*/
ret = panthor_gem_backing_get_pages_locked(bo);
if (ret)
return ERR_PTR(ret);
@@ -336,11 +450,19 @@ static void panthor_gem_free_object(struct drm_gem_object *obj)
if (drm_gem_is_imported(obj)) {
drm_prime_gem_destroy(obj, bo->dmap.sgt);
} else {
dma_resv_lock(obj->resv, NULL);
/* The last ref on the GEM object can be released
* by the shrinker, which can't block on the resv
* lock acquisition. In practice, even if we were
* taking the lock, it wouldn't block because we're
* the last piece of code having a visibility on
* this GEM, but lockdep can't see that, so we've
* just tought the _cleanup_locked() helpers about
* this "being freed" exception, and we call those
* without the lock held here.
*/
panthor_gem_vmap_cleanup_locked(bo);
panthor_gem_dev_map_cleanup_locked(bo);
panthor_gem_backing_cleanup_locked(bo);
dma_resv_unlock(obj->resv);
}
drm_gem_object_release(obj);
@@ -535,6 +657,41 @@ void panthor_gem_unpin(struct panthor_gem_object *bo)
dma_resv_unlock(bo->base.resv);
}
int panthor_gem_swapin_locked(struct panthor_gem_object *bo)
{
struct sg_table *sgt;
dma_resv_assert_held(bo->base.resv);
if (drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base)))
return -EINVAL;
sgt = panthor_gem_dev_map_get_sgt_locked(bo);
if (IS_ERR(sgt))
return PTR_ERR(sgt);
return 0;
}
static void panthor_gem_evict_locked(struct panthor_gem_object *bo)
{
dma_resv_assert_held(bo->base.resv);
lockdep_assert_held(&bo->base.gpuva.lock);
if (drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base)))
return;
if (drm_WARN_ON_ONCE(bo->base.dev, refcount_read(&bo->backing.pin_count)))
return;
if (drm_WARN_ON_ONCE(bo->base.dev, !bo->backing.pages))
return;
panthor_gem_dev_map_cleanup_locked(bo);
panthor_gem_backing_cleanup_locked(bo);
panthor_gem_update_reclaim_state_locked(bo, NULL);
}
static struct sg_table *panthor_gem_get_sg_table(struct drm_gem_object *obj)
{
struct panthor_gem_object *bo = to_panthor_bo(obj);
@@ -607,8 +764,12 @@ static int panthor_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *v
if (!refcount_inc_not_zero(&bo->cmap.mmap_count)) {
dma_resv_lock(obj->resv, NULL);
if (!refcount_inc_not_zero(&bo->cmap.mmap_count))
if (!refcount_inc_not_zero(&bo->cmap.mmap_count)) {
refcount_set(&bo->cmap.mmap_count, 1);
mutex_lock(&bo->base.gpuva.lock);
panthor_gem_update_reclaim_state_locked(bo, NULL);
mutex_unlock(&bo->base.gpuva.lock);
}
dma_resv_unlock(obj->resv);
}
@@ -692,6 +853,10 @@ static vm_fault_t blocking_page_setup(struct vm_fault *vmf, unsigned int order,
} else {
struct page *page = bo->backing.pages[page_offset];
mutex_lock(&bo->base.gpuva.lock);
panthor_gem_update_reclaim_state_locked(bo, NULL);
mutex_unlock(&bo->base.gpuva.lock);
if (mmap_lock_held)
ret = insert_page(vmf, order, page);
else
@@ -734,7 +899,12 @@ static vm_fault_t panthor_gem_any_fault(struct vm_fault *vmf, unsigned int order
/* Wait with the mmap lock released, if we're allowed to. */
drm_gem_object_get(&bo->base);
mmap_read_unlock(vmf->vma->vm_mm);
if (vmf->flags & FAULT_FLAG_VMA_LOCK)
vma_end_read(vmf->vma);
else
mmap_read_unlock(vmf->vma->vm_mm);
ret = blocking_page_setup(vmf, order, bo, page_offset, false);
drm_gem_object_put(&bo->base);
return ret;
@@ -770,7 +940,9 @@ static void panthor_gem_vm_close(struct vm_area_struct *vma)
dma_resv_lock(bo->base.resv, NULL);
if (refcount_dec_and_test(&bo->cmap.mmap_count)) {
/* Nothing to do, pages are reclaimed lazily. */
mutex_lock(&bo->base.gpuva.lock);
panthor_gem_update_reclaim_state_locked(bo, NULL);
mutex_unlock(&bo->base.gpuva.lock);
}
dma_resv_unlock(bo->base.resv);
@@ -810,6 +982,7 @@ panthor_gem_alloc_object(u32 flags)
if (!bo)
return ERR_PTR(-ENOMEM);
bo->reclaim_state = PANTHOR_GEM_UNRECLAIMABLE;
bo->base.funcs = &panthor_gem_funcs;
bo->flags = flags;
mutex_init(&bo->label.lock);
@@ -968,6 +1141,7 @@ panthor_gem_sync(struct drm_gem_object *obj, u32 type,
struct sg_table *sgt;
struct scatterlist *sgl;
unsigned int count;
int ret;
/* Make sure the range is in bounds. */
if (offset + size < offset || offset + size > bo->base.size)
@@ -994,9 +1168,21 @@ panthor_gem_sync(struct drm_gem_object *obj, u32 type,
if (size == 0)
return 0;
sgt = panthor_gem_get_dev_sgt(bo);
if (IS_ERR(sgt))
return PTR_ERR(sgt);
ret = dma_resv_lock_interruptible(bo->base.resv, NULL);
if (ret)
return ret;
/* If there's no pages, there's no point pulling those back, bail out early. */
if (!bo->backing.pages) {
ret = 0;
goto out_unlock;
}
sgt = panthor_gem_dev_map_get_sgt_locked(bo);
if (IS_ERR(sgt)) {
ret = PTR_ERR(sgt);
goto out_unlock;
}
for_each_sgtable_dma_sg(sgt, sgl, count) {
if (size == 0)
@@ -1040,7 +1226,11 @@ panthor_gem_sync(struct drm_gem_object *obj, u32 type,
dma_sync_single_for_cpu(dma_dev, paddr, len, DMA_FROM_DEVICE);
}
return 0;
ret = 0;
out_unlock:
dma_resv_unlock(bo->base.resv);
return ret;
}
/**
@@ -1050,11 +1240,13 @@ panthor_gem_sync(struct drm_gem_object *obj, u32 type,
*/
void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo)
{
struct panthor_device *ptdev;
struct panthor_vm *vm;
if (IS_ERR_OR_NULL(bo))
return;
ptdev = container_of(bo->obj->dev, struct panthor_device, base);
vm = bo->vm;
panthor_kernel_bo_vunmap(bo);
@@ -1062,6 +1254,8 @@ void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo)
to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm));
panthor_vm_unmap_range(vm, bo->va_node.start, bo->va_node.size);
panthor_vm_free_va(vm, &bo->va_node);
if (vm == panthor_fw_vm(ptdev))
panthor_gem_unpin(to_panthor_bo(bo->obj));
drm_gem_object_put(bo->obj);
panthor_vm_put(vm);
kfree(bo);
@@ -1110,6 +1304,12 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
kbo->obj = &bo->base;
if (vm == panthor_fw_vm(ptdev)) {
ret = panthor_gem_pin(bo);
if (ret)
goto err_put_obj;
}
panthor_gem_kernel_bo_set_label(kbo, name);
/* The system and GPU MMU page size might differ, which becomes a
@@ -1121,7 +1321,7 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
size = ALIGN(size, panthor_vm_page_size(vm));
ret = panthor_vm_alloc_va(vm, gpu_va, size, &kbo->va_node);
if (ret)
goto err_put_obj;
goto err_unpin;
ret = panthor_vm_map_bo_range(vm, bo, 0, size, kbo->va_node.start, vm_map_flags);
if (ret)
@@ -1133,6 +1333,10 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
err_free_va:
panthor_vm_free_va(vm, &kbo->va_node);
err_unpin:
if (vm == panthor_fw_vm(ptdev))
panthor_gem_unpin(bo);
err_put_obj:
drm_gem_object_put(&bo->base);
@@ -1141,6 +1345,241 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
return ERR_PTR(ret);
}
static bool can_swap(void)
{
return get_nr_swap_pages() > 0;
}
static bool can_block(struct shrink_control *sc)
{
/* If direct reclaim is allowed, we can always block.
* If kswapd reclaim is allowed, we can block, but only if we're called
* by the kswapd thread.
*/
return (sc->gfp_mask & __GFP_DIRECT_RECLAIM) ||
((sc->gfp_mask & __GFP_KSWAPD_RECLAIM) && current_is_kswapd());
}
static unsigned long
panthor_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
{
struct panthor_device *ptdev = shrinker->private_data;
unsigned long count;
/* We currently don't have a flag to tell when the content of a
* BO can be discarded.
*/
if (!can_swap())
return 0;
/* This is racy, but that's okay because the returned count is just a
* hint. That's also what MSM is doing (no atomic var, it's relying on
* the fact unsigned long access is usually atomic), so if it's good
* enough for them, it's good enough for us too.
*/
count = ptdev->reclaim.unused.count;
count += ptdev->reclaim.mmapped.count;
if (can_block(sc))
count += ptdev->reclaim.gpu_mapped_count;
return count ? count : SHRINK_EMPTY;
}
static bool panthor_gem_try_evict_no_resv_wait(struct drm_gem_object *obj,
struct ww_acquire_ctx *ticket)
{
/*
* Track last locked entry for unwinding locks in error and
* success paths
*/
struct panthor_gem_object *bo = to_panthor_bo(obj);
struct drm_gpuvm_bo *vm_bo, *last_locked = NULL;
enum panthor_gem_reclaim_state old_state;
int ret = 0;
/* To avoid potential lock ordering issue between bo_gpuva and
* mapping->i_mmap_rwsem, unmap the pages from CPU side before
* acquring the bo_gpuva lock. As the bo_resv lock is held, CPU
* page fault handler won't be able to map in the pages whilst
* eviction is in progress.
*/
drm_vma_node_unmap(&bo->base.vma_node, bo->base.dev->anon_inode->i_mapping);
/* We take this lock when walking the list to prevent
* insertion/deletion.
*/
/* We can only trylock in that path, because
* - allocation might happen while some of these locks are held
* - lock ordering is different in other paths
* vm_resv -> bo_resv -> bo_gpuva
* vs
* bo_resv -> bo_gpuva -> vm_resv
*
* If we fail to lock that's fine, we back off and will get
* back to it later.
*/
if (!mutex_trylock(&bo->base.gpuva.lock))
return false;
drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm);
if (resv == obj->resv)
continue;
if (!dma_resv_trylock(resv)) {
ret = -EDEADLK;
goto out_unlock;
}
last_locked = vm_bo;
}
/* Update the state before trying to evict the buffer, if the state was
* updated to something that's harder to reclaim (higher value in the
* enum), skip it (will be processed when the relevant LRU is).
*/
panthor_gem_update_reclaim_state_locked(bo, &old_state);
if (old_state < bo->reclaim_state) {
ret = -EAGAIN;
goto out_unlock;
}
/* Couldn't teardown the GPU mappings? Skip. */
ret = panthor_vm_evict_bo_mappings_locked(bo);
if (ret)
goto out_unlock;
/* If everything went fine, evict the object. */
panthor_gem_evict_locked(bo);
out_unlock:
if (last_locked) {
drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm);
if (resv == obj->resv)
continue;
dma_resv_unlock(resv);
if (last_locked == vm_bo)
break;
}
}
mutex_unlock(&bo->base.gpuva.lock);
return ret == 0;
}
static bool panthor_gem_try_evict(struct drm_gem_object *obj,
struct ww_acquire_ctx *ticket)
{
struct panthor_gem_object *bo = to_panthor_bo(obj);
/* Wait was too long, skip. */
if (dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_BOOKKEEP, false, 10) <= 0)
return false;
return panthor_gem_try_evict_no_resv_wait(&bo->base, ticket);
}
static unsigned long
panthor_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
{
struct panthor_device *ptdev = shrinker->private_data;
unsigned long remaining = 0;
unsigned long freed = 0;
if (!can_swap())
goto out;
freed += drm_gem_lru_scan(&ptdev->reclaim.unused,
sc->nr_to_scan - freed, &remaining,
panthor_gem_try_evict_no_resv_wait, NULL);
if (freed >= sc->nr_to_scan)
goto out;
freed += drm_gem_lru_scan(&ptdev->reclaim.mmapped,
sc->nr_to_scan - freed, &remaining,
panthor_gem_try_evict_no_resv_wait, NULL);
if (freed >= sc->nr_to_scan)
goto out;
if (!can_block(sc))
goto out;
freed += panthor_mmu_reclaim_priv_bos(ptdev, sc->nr_to_scan - freed,
&remaining, panthor_gem_try_evict);
if (freed >= sc->nr_to_scan)
goto out;
freed += drm_gem_lru_scan(&ptdev->reclaim.gpu_mapped_shared,
sc->nr_to_scan - freed, &remaining,
panthor_gem_try_evict, NULL);
out:
#ifdef CONFIG_DEBUG_FS
/* This is racy, but that's okay, because this is just debugfs
* reporting and doesn't need to be accurate.
*/
ptdev->reclaim.nr_pages_reclaimed_on_last_scan = freed;
#endif
/* If there are things to reclaim, try a couple times before giving up. */
if (!freed && remaining > 0 &&
atomic_inc_return(&ptdev->reclaim.retry_count) < 2)
return 0;
atomic_set(&ptdev->reclaim.retry_count, 0);
if (freed)
return freed;
/* There's nothing left to reclaim, or the resources are contended. Give up now. */
return SHRINK_STOP;
}
int panthor_gem_shrinker_init(struct panthor_device *ptdev)
{
struct shrinker *shrinker;
int ret;
ret = drmm_mutex_init(&ptdev->base, &ptdev->reclaim.lock);
if (ret)
return ret;
INIT_LIST_HEAD(&ptdev->reclaim.vms);
drm_gem_lru_init(&ptdev->reclaim.unused, &ptdev->reclaim.lock);
drm_gem_lru_init(&ptdev->reclaim.mmapped, &ptdev->reclaim.lock);
drm_gem_lru_init(&ptdev->reclaim.gpu_mapped_shared, &ptdev->reclaim.lock);
ptdev->reclaim.gpu_mapped_count = 0;
/* Teach lockdep about lock ordering wrt. shrinker: */
fs_reclaim_acquire(GFP_KERNEL);
might_lock(&ptdev->reclaim.lock);
fs_reclaim_release(GFP_KERNEL);
shrinker = shrinker_alloc(0, "drm-panthor-gem");
if (!shrinker)
return -ENOMEM;
shrinker->count_objects = panthor_gem_shrinker_count;
shrinker->scan_objects = panthor_gem_shrinker_scan;
shrinker->private_data = ptdev;
ptdev->reclaim.shrinker = shrinker;
shrinker_register(shrinker);
return 0;
}
void panthor_gem_shrinker_unplug(struct panthor_device *ptdev)
{
if (ptdev->reclaim.shrinker)
shrinker_free(ptdev->reclaim.shrinker);
}
#ifdef CONFIG_DEBUG_FS
struct gem_size_totals {
size_t size;
@@ -1184,6 +1623,7 @@ static void panthor_gem_debugfs_bo_print(struct panthor_gem_object *bo,
struct seq_file *m,
struct gem_size_totals *totals)
{
enum panthor_gem_reclaim_state reclaim_state = bo->reclaim_state;
unsigned int refcount = kref_read(&bo->base.refcount);
char creator_info[32] = {};
size_t resident_size;
@@ -1219,6 +1659,8 @@ static void panthor_gem_debugfs_bo_print(struct panthor_gem_object *bo,
totals->size += bo->base.size;
totals->resident += resident_size;
if (reclaim_state != PANTHOR_GEM_UNRECLAIMABLE)
totals->reclaimable += resident_size;
}
static void panthor_gem_debugfs_print_bos(struct panthor_device *ptdev,
@@ -1259,10 +1701,42 @@ static struct drm_info_list panthor_gem_debugfs_list[] = {
{ "gems", panthor_gem_show_bos, 0, NULL },
};
static int shrink_get(void *data, u64 *val)
{
struct panthor_device *ptdev =
container_of(data, struct panthor_device, base);
*val = ptdev->reclaim.nr_pages_reclaimed_on_last_scan;
return 0;
}
static int shrink_set(void *data, u64 val)
{
struct panthor_device *ptdev =
container_of(data, struct panthor_device, base);
struct shrink_control sc = {
.gfp_mask = GFP_KERNEL,
.nr_to_scan = val,
};
fs_reclaim_acquire(GFP_KERNEL);
if (ptdev->reclaim.shrinker)
panthor_gem_shrinker_scan(ptdev->reclaim.shrinker, &sc);
fs_reclaim_release(GFP_KERNEL);
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(panthor_gem_debugfs_shrink_fops,
shrink_get, shrink_set,
"0x%08llx\n");
void panthor_gem_debugfs_init(struct drm_minor *minor)
{
drm_debugfs_create_files(panthor_gem_debugfs_list,
ARRAY_SIZE(panthor_gem_debugfs_list),
minor->debugfs_root, minor);
debugfs_create_file("shrink", 0600, minor->debugfs_root,
minor->dev, &panthor_gem_debugfs_shrink_fops);
}
#endif

68
drivers/gpu/drm/panthor/panthor_gem.h
View File

@@ -1,6 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 or MIT */
/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
/* Copyright 2023 Collabora ltd. */
/* Copyright 2025 ARM Limited. All rights reserved. */
#ifndef __PANTHOR_GEM_H__
#define __PANTHOR_GEM_H__
@@ -93,6 +94,65 @@ struct panthor_gem_dev_map {
struct sg_table *sgt;
};
/**
* enum panthor_gem_reclaim_state - Reclaim state of a GEM object
*
* This is defined in descending reclaimability order and some part
* of the code depends on that.
*/
enum panthor_gem_reclaim_state {
/**
* @PANTHOR_GEM_UNUSED: GEM is currently unused
*
* This can happen when the GEM was previously vmap-ed, mmap-ed,
* and/or GPU mapped and got unmapped. Because pages are lazily
* returned to the shmem layer, we want to keep a list of such
* BOs, because they should be fairly easy to reclaim (no need
* to wait for GPU to be done, and no need to tear down user
* mappings either).
*/
PANTHOR_GEM_UNUSED,
/**
* @PANTHOR_GEM_MMAPPED: GEM is currently mmap-ed
*
* When a GEM has pages allocated and the mmap_count is > 0, the
* GEM is placed in the mmapped list. This comes right after
* unused because we can relatively easily tear down user mappings.
*/
PANTHOR_GEM_MMAPPED,
/**
* @PANTHOR_GEM_GPU_MAPPED_SINGLE_VM: GEM is GPU mapped to only one VM
*
* When a GEM is mapped to a single VM, reclaim requests have more
* chances to succeed, because we only need to synchronize against
* a single GPU context. This is more annoying than reclaiming
* mmap-ed pages still, because we have to wait for in-flight jobs
* to land, and we might not be able to acquire all necessary locks
* at reclaim time either.
*/
PANTHOR_GEM_GPU_MAPPED_SINGLE_VM,
/**
* @PANTHOR_GEM_GPU_MAPPED_MULTI_VM: GEM is GPU mapped to multiple VMs
*
* Like PANTHOR_GEM_GPU_MAPPED_SINGLE_VM, but the synchronization across
* VMs makes such BOs harder to reclaim.
*/
PANTHOR_GEM_GPU_MAPPED_MULTI_VM,
/**
* @PANTHOR_GEM_UNRECLAIMABLE: GEM can't be reclaimed
*
* Happens when the GEM memory is pinned. It's also the state all GEM
* objects start in, because no memory is allocated until explicitly
* requested by a CPU or GPU map, meaning there's nothing to reclaim
* until such an allocation happens.
*/
PANTHOR_GEM_UNRECLAIMABLE,
};
/**
* struct panthor_gem_object - Driver specific GEM object.
*/
@@ -109,6 +169,9 @@ struct panthor_gem_object {
/** @dmap: Device mapping state */
struct panthor_gem_dev_map dmap;
/** @reclaim_state: Cached reclaim state */
enum panthor_gem_reclaim_state reclaim_state;
/**
* @exclusive_vm_root_gem: Root GEM of the exclusive VM this GEM object
* is attached to.
@@ -190,6 +253,11 @@ struct sg_table *
panthor_gem_get_dev_sgt(struct panthor_gem_object *bo);
int panthor_gem_pin(struct panthor_gem_object *bo);
void panthor_gem_unpin(struct panthor_gem_object *bo);
int panthor_gem_swapin_locked(struct panthor_gem_object *bo);
void panthor_gem_update_reclaim_state_locked(struct panthor_gem_object *bo,
enum panthor_gem_reclaim_state *old_state);
int panthor_gem_shrinker_init(struct panthor_device *ptdev);
void panthor_gem_shrinker_unplug(struct panthor_device *ptdev);
void panthor_gem_bo_set_label(struct drm_gem_object *obj, const char *label);
void panthor_gem_kernel_bo_set_label(struct panthor_kernel_bo *bo, const char *label);

397
drivers/gpu/drm/panthor/panthor_mmu.c
View File

@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 or MIT
/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
/* Copyright 2023 Collabora ltd. */
/* Copyright 2025 ARM Limited. All rights reserved. */
#include <drm/drm_debugfs.h>
#include <drm/drm_drv.h>
@@ -131,6 +132,9 @@ struct panthor_vma {
* Only map related flags are accepted.
*/
u32 flags;
/** @evicted: True if the VMA has been evicted. */
bool evicted;
};
/**
@@ -198,6 +202,9 @@ struct panthor_vm_op_ctx {
* to allocate in ::run_job().
*/
struct sg_table *sgt;
/** @map.bo: the BO being mapped. */
struct panthor_gem_object *bo;
} map;
};
@@ -380,6 +387,18 @@ struct panthor_vm {
/** @locked_region.size: Size of the locked region. */
u64 size;
} locked_region;
/** @reclaim: Fields related to BO reclaim. */
struct {
/** @reclaim.lru: LRU of BOs that are only mapped to this VM. */
struct drm_gem_lru lru;
/**
* @reclaim.lru_node: Node used to insert the VM in
* panthor_device::reclaim::vms.
*/
struct list_head lru_node;
} reclaim;
};
/**
@@ -684,6 +703,16 @@ int panthor_vm_active(struct panthor_vm *vm)
if (refcount_inc_not_zero(&vm->as.active_cnt))
goto out_dev_exit;
/* As soon as active is called, we place the VM at the end of the VM LRU.
* If something fails after that, the only downside is that this VM that
* never became active in the first place will be reclaimed last, but
* that's an acceptable trade-off.
*/
mutex_lock(&ptdev->reclaim.lock);
if (vm->reclaim.lru.count)
list_move_tail(&vm->reclaim.lru_node, &ptdev->reclaim.vms);
mutex_unlock(&ptdev->reclaim.lock);
/* Make sure we don't race with lock/unlock_region() calls
* happening around VM bind operations.
*/
@@ -1079,7 +1108,15 @@ static void panthor_vm_bo_free(struct drm_gpuvm_bo *vm_bo)
{
struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
panthor_gem_unpin(bo);
/* We couldn't call this when we unlinked, because the resv lock can't
* be taken in the dma signalling path, so call it now.
*/
dma_resv_lock(bo->base.resv, NULL);
mutex_lock(&bo->base.gpuva.lock);
panthor_gem_update_reclaim_state_locked(bo, NULL);
mutex_unlock(&bo->base.gpuva.lock);
dma_resv_unlock(bo->base.resv);
kfree(vm_bo);
}
@@ -1088,6 +1125,16 @@ static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,
{
u32 remaining_pt_count = op_ctx->rsvd_page_tables.count -
op_ctx->rsvd_page_tables.ptr;
u32 op_type = op_ctx->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK;
/* If this is a map operation and no BO is attached, we're being called
* from vm_bo_validate() and we can't acquire the VM lock because it's
* already held. In that case, we just skip the deferred vm_bo cleanup,
* which is fine, because the vm_bo validation is not calling
* drm_gpuvm_bo_put_deferred().
*/
bool skip_deferred_cleanup = op_type == DRM_PANTHOR_VM_BIND_OP_TYPE_MAP &&
!op_ctx->map.bo;
if (remaining_pt_count) {
kmem_cache_free_bulk(pt_cache, remaining_pt_count,
@@ -1100,10 +1147,16 @@ static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,
if (op_ctx->map.vm_bo)
drm_gpuvm_bo_put_deferred(op_ctx->map.vm_bo);
if (op_ctx->map.bo) {
panthor_gem_unpin(op_ctx->map.bo);
drm_gem_object_put(&op_ctx->map.bo->base);
}
for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++)
kfree(op_ctx->preallocated_vmas[i]);
drm_gpuvm_bo_deferred_cleanup(&vm->base);
if (!skip_deferred_cleanup)
drm_gpuvm_bo_deferred_cleanup(&vm->base);
}
static void
@@ -1259,18 +1312,17 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
if (ret)
goto err_cleanup;
drm_gem_object_get(&bo->base);
op_ctx->map.bo = bo;
sgt = panthor_gem_get_dev_sgt(bo);
if (IS_ERR(sgt)) {
panthor_gem_unpin(bo);
ret = PTR_ERR(sgt);
goto err_cleanup;
}
op_ctx->map.sgt = sgt;
preallocated_vm_bo = drm_gpuvm_bo_create(&vm->base, &bo->base);
if (!preallocated_vm_bo) {
panthor_gem_unpin(bo);
ret = -ENOMEM;
goto err_cleanup;
}
@@ -1289,6 +1341,13 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
dma_resv_unlock(panthor_vm_resv(vm));
}
/* And finally update the BO state. */
dma_resv_lock(bo->base.resv, NULL);
mutex_lock(&bo->base.gpuva.lock);
panthor_gem_update_reclaim_state_locked(bo, NULL);
mutex_unlock(&bo->base.gpuva.lock);
dma_resv_unlock(bo->base.resv);
return 0;
err_cleanup:
@@ -1876,6 +1935,10 @@ static void panthor_vm_free(struct drm_gpuvm *gpuvm)
struct panthor_vm *vm = container_of(gpuvm, struct panthor_vm, base);
struct panthor_device *ptdev = vm->ptdev;
mutex_lock(&ptdev->reclaim.lock);
list_del_init(&vm->reclaim.lru_node);
mutex_unlock(&ptdev->reclaim.lock);
mutex_lock(&vm->heaps.lock);
if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))
panthor_heap_pool_destroy(vm->heaps.pool);
@@ -2089,7 +2152,7 @@ static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
panthor_vma_init(vma, op_ctx->flags & PANTHOR_VM_MAP_FLAGS);
ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags),
op_ctx->map.sgt, op->map.gem.offset,
op_ctx->map.bo->dmap.sgt, op->map.gem.offset,
op->map.va.range);
if (ret) {
panthor_vm_op_ctx_return_vma(op_ctx, vma);
@@ -2173,21 +2236,27 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
* atomicity. panthor_vm_lock_region() bails out early if the new region
* is already part of the locked region, so no need to do this check here.
*/
panthor_vm_lock_region(vm, unmap_start, unmap_range);
panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
if (!unmap_vma->evicted) {
panthor_vm_lock_region(vm, unmap_start, unmap_range);
panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
}
if (op->remap.prev) {
struct panthor_gem_object *bo = to_panthor_bo(op->remap.prev->gem.obj);
u64 offset = op->remap.prev->gem.offset + unmap_start - op->remap.prev->va.addr;
u64 size = op->remap.prev->va.addr + op->remap.prev->va.range - unmap_start;
ret = panthor_vm_map_pages(vm, unmap_start, flags_to_prot(unmap_vma->flags),
bo->dmap.sgt, offset, size);
if (ret)
return ret;
if (!unmap_vma->evicted) {
ret = panthor_vm_map_pages(vm, unmap_start,
flags_to_prot(unmap_vma->flags),
bo->dmap.sgt, offset, size);
if (ret)
return ret;
}
prev_vma = panthor_vm_op_ctx_get_vma(op_ctx);
panthor_vma_init(prev_vma, unmap_vma->flags);
prev_vma->evicted = unmap_vma->evicted;
}
if (op->remap.next) {
@@ -2195,13 +2264,17 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
u64 addr = op->remap.next->va.addr;
u64 size = unmap_start + unmap_range - op->remap.next->va.addr;
ret = panthor_vm_map_pages(vm, addr, flags_to_prot(unmap_vma->flags),
bo->dmap.sgt, op->remap.next->gem.offset, size);
if (ret)
return ret;
if (!unmap_vma->evicted) {
ret = panthor_vm_map_pages(vm, addr, flags_to_prot(unmap_vma->flags),
bo->dmap.sgt, op->remap.next->gem.offset,
size);
if (ret)
return ret;
}
next_vma = panthor_vm_op_ctx_get_vma(op_ctx);
panthor_vma_init(next_vma, unmap_vma->flags);
next_vma->evicted = unmap_vma->evicted;
}
drm_gpuva_remap(prev_vma ? &prev_vma->base : NULL,
@@ -2231,19 +2304,230 @@ static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op,
struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base);
struct panthor_vm *vm = priv;
panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
unmap_vma->base.va.range);
if (!unmap_vma->evicted) {
panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
unmap_vma->base.va.range);
}
drm_gpuva_unmap(&op->unmap);
panthor_vma_unlink(unmap_vma);
return 0;
}
void panthor_vm_update_bo_reclaim_lru_locked(struct panthor_gem_object *bo)
{
struct panthor_device *ptdev = container_of(bo->base.dev, struct panthor_device, base);
struct panthor_vm *vm = NULL;
struct drm_gpuvm_bo *vm_bo;
dma_resv_assert_held(bo->base.resv);
lockdep_assert_held(&bo->base.gpuva.lock);
drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) {
if (vm_bo->evicted)
continue;
/* We're only supposed to have one non-evicted vm_bo in the list if we get
* there.
*/
drm_WARN_ON(&ptdev->base, vm);
vm = container_of(vm_bo->vm, struct panthor_vm, base);
mutex_lock(&ptdev->reclaim.lock);
drm_gem_lru_move_tail_locked(&vm->reclaim.lru, &bo->base);
if (list_empty(&vm->reclaim.lru_node))
list_move(&vm->reclaim.lru_node, &ptdev->reclaim.vms);
mutex_unlock(&ptdev->reclaim.lock);
}
}
int panthor_vm_evict_bo_mappings_locked(struct panthor_gem_object *bo)
{
struct drm_gpuvm_bo *vm_bo;
int ret = 0;
drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) {
struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
struct drm_gpuva *va;
/* Skip already evicted GPU mappings. */
if (vm_bo->evicted)
continue;
if (!mutex_trylock(&vm->op_lock))
return -EDEADLK;
drm_gpuvm_bo_evict(vm_bo, true);
drm_gpuvm_bo_for_each_va(va, vm_bo) {
struct panthor_vma *vma = container_of(va, struct panthor_vma, base);
if (vma->evicted)
continue;
/* If something fail in the middle of a VM_BO eviction, the VM_BO
* is considered fully evicted, but some of its VMAs might still be
* active. That's okay because the pages won't be released if this
* function returns an error.
*
* On the next job targeting this VM, the partially evicted VM_BO
* will be validated, causing all its evicted VMAs to be repopulated
* before the job runs. So no GPU fault expected.
*/
ret = panthor_vm_lock_region(vm, va->va.addr, va->va.range);
if (ret)
break;
panthor_vm_unmap_pages(vm, va->va.addr, va->va.range);
panthor_vm_unlock_region(vm);
vma->evicted = true;
}
mutex_unlock(&vm->op_lock);
if (ret)
break;
}
return ret;
}
static struct panthor_vma *select_evicted_vma(struct drm_gpuvm_bo *vm_bo,
struct panthor_vm_op_ctx *op_ctx)
{
struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
struct panthor_vma *first_evicted_vma = NULL;
struct drm_gpuva *va;
/* Take op_lock to protect against va insertion/removal. */
mutex_lock(&vm->op_lock);
drm_gpuvm_bo_for_each_va(va, vm_bo) {
struct panthor_vma *vma = container_of(va, struct panthor_vma, base);
if (vma->evicted) {
first_evicted_vma = vma;
panthor_vm_init_op_ctx(op_ctx, va->va.range, va->va.addr, vma->flags);
op_ctx->map.bo_offset = va->gem.offset;
break;
}
}
mutex_unlock(&vm->op_lock);
return first_evicted_vma;
}
static int remap_evicted_vma(struct drm_gpuvm_bo *vm_bo,
struct panthor_vma *evicted_vma,
struct panthor_vm_op_ctx *op_ctx)
{
struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
struct drm_gpuva *va;
bool found = false;
int ret;
ret = panthor_vm_op_ctx_prealloc_pts(op_ctx);
if (ret)
goto out_cleanup;
/* Take op_lock to protect against va insertion/removal. Note that the
* evicted_vma selection was done with the same lock held, but we had
* to release it so we can allocate PTs, because this very same lock
* is taken in a DMA-signalling path.
*/
mutex_lock(&vm->op_lock);
drm_gpuvm_bo_for_each_va(va, vm_bo) {
struct panthor_vma *vma = container_of(va, struct panthor_vma, base);
if (vma != evicted_vma)
continue;
/* Because we had to release the lock between the evicted_vma selection
* and its repopulation, we can't rely solely on pointer equality (the
* VMA might have been freed and a new one allocated at the same address).
* If the evicted bit is still set, we're sure it's our VMA, because
* population/eviction is serialized with the BO resv lock.
*/
if (vma->evicted)
found = true;
break;
}
if (found) {
vm->op_ctx = op_ctx;
ret = panthor_vm_lock_region(vm, evicted_vma->base.va.addr,
evicted_vma->base.va.range);
if (!ret) {
ret = panthor_vm_map_pages(vm, evicted_vma->base.va.addr,
flags_to_prot(evicted_vma->flags),
bo->dmap.sgt,
evicted_vma->base.gem.offset,
evicted_vma->base.va.range);
if (!ret)
evicted_vma->evicted = false;
panthor_vm_unlock_region(vm);
}
vm->op_ctx = NULL;
}
mutex_unlock(&vm->op_lock);
out_cleanup:
panthor_vm_cleanup_op_ctx(op_ctx, vm);
return ret;
}
static int panthor_vm_restore_vmas(struct drm_gpuvm_bo *vm_bo)
{
struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
struct panthor_vm_op_ctx op_ctx;
if (drm_WARN_ON_ONCE(&vm->ptdev->base, !bo->dmap.sgt))
return -EINVAL;
for (struct panthor_vma *vma = select_evicted_vma(vm_bo, &op_ctx);
vma; vma = select_evicted_vma(vm_bo, &op_ctx)) {
int ret;
ret = remap_evicted_vma(vm_bo, vma, &op_ctx);
if (ret)
return ret;
}
return 0;
}
static int panthor_vm_bo_validate(struct drm_gpuvm_bo *vm_bo,
struct drm_exec *exec)
{
struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
int ret;
ret = panthor_gem_swapin_locked(bo);
if (ret)
return ret;
ret = panthor_vm_restore_vmas(vm_bo);
if (ret)
return ret;
drm_gpuvm_bo_evict(vm_bo, false);
mutex_lock(&bo->base.gpuva.lock);
panthor_gem_update_reclaim_state_locked(bo, NULL);
mutex_unlock(&bo->base.gpuva.lock);
return 0;
}
static const struct drm_gpuvm_ops panthor_gpuvm_ops = {
.vm_free = panthor_vm_free,
.vm_bo_free = panthor_vm_bo_free,
.sm_step_map = panthor_gpuva_sm_step_map,
.sm_step_remap = panthor_gpuva_sm_step_remap,
.sm_step_unmap = panthor_gpuva_sm_step_unmap,
.vm_bo_validate = panthor_vm_bo_validate,
};
/**
@@ -2458,6 +2742,8 @@ panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
vm->kernel_auto_va.start = auto_kernel_va_start;
vm->kernel_auto_va.end = vm->kernel_auto_va.start + auto_kernel_va_size - 1;
drm_gem_lru_init(&vm->reclaim.lru, &ptdev->reclaim.lock);
INIT_LIST_HEAD(&vm->reclaim.lru_node);
INIT_LIST_HEAD(&vm->node);
INIT_LIST_HEAD(&vm->as.lru_node);
vm->as.id = -1;
@@ -2805,7 +3091,78 @@ int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm
if (ret)
return ret;
return drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);
ret = drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);
if (ret)
return ret;
return drm_gpuvm_validate(&vm->base, exec);
}
unsigned long
panthor_mmu_reclaim_priv_bos(struct panthor_device *ptdev,
unsigned int nr_to_scan, unsigned long *remaining,
bool (*shrink)(struct drm_gem_object *,
struct ww_acquire_ctx *))
{
unsigned long freed = 0;
LIST_HEAD(remaining_vms);
LIST_HEAD(vms);
mutex_lock(&ptdev->reclaim.lock);
list_splice_init(&ptdev->reclaim.vms, &vms);
while (freed < nr_to_scan) {
struct panthor_vm *vm;
vm = list_first_entry_or_null(&vms, typeof(*vm),
reclaim.lru_node);
if (!vm)
break;
if (!kref_get_unless_zero(&vm->base.kref)) {
list_del_init(&vm->reclaim.lru_node);
continue;
}
mutex_unlock(&ptdev->reclaim.lock);
freed += drm_gem_lru_scan(&vm->reclaim.lru, nr_to_scan - freed,
remaining, shrink, NULL);
mutex_lock(&ptdev->reclaim.lock);
/* If the VM is still in the temporary list, remove it so we
* can proceed with the next VM.
*/
if (vm == list_first_entry_or_null(&vms, typeof(*vm), reclaim.lru_node)) {
list_del_init(&vm->reclaim.lru_node);
/* Keep the VM around if there are still things to
* reclaim, so we can preserve the LRU order when
* re-inserting in ptdev->reclaim.vms at the end.
*/
if (vm->reclaim.lru.count > 0)
list_add_tail(&vm->reclaim.lru_node, &remaining_vms);
}
mutex_unlock(&ptdev->reclaim.lock);
panthor_vm_put(vm);
mutex_lock(&ptdev->reclaim.lock);
}
/* Re-insert VMs with remaining data to reclaim at the beginning of
* the LRU. Note that any activeness change on the VM that happened
* while we were reclaiming would have moved the VM out of our
* temporary [remaining_]vms list, meaning anything we re-insert here
* preserves the LRU order.
*/
list_splice_tail(&vms, &remaining_vms);
list_splice(&remaining_vms, &ptdev->reclaim.vms);
mutex_unlock(&ptdev->reclaim.lock);
return freed;
}
/**

8
drivers/gpu/drm/panthor/panthor_mmu.h
View File

@@ -1,6 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 or MIT */
/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
/* Copyright 2023 Collabora ltd. */
/* Copyright 2025 ARM Limited. All rights reserved. */
#ifndef __PANTHOR_MMU_H__
#define __PANTHOR_MMU_H__
@@ -46,6 +47,13 @@ struct panthor_vm *panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
u64 kernel_auto_va_start,
u64 kernel_auto_va_size);
void panthor_vm_update_bo_reclaim_lru_locked(struct panthor_gem_object *bo);
int panthor_vm_evict_bo_mappings_locked(struct panthor_gem_object *bo);
unsigned long
panthor_mmu_reclaim_priv_bos(struct panthor_device *ptdev,
unsigned int nr_to_scan, unsigned long *remaining,
bool (*shrink)(struct drm_gem_object *,
struct ww_acquire_ctx *));
int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec,
struct panthor_vm *vm,
u32 slot_count);