diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 21dc4a9c8714..50ee550fef73 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -500,29 +500,40 @@ static int page_pool_register_dma_index(struct page_pool *pool, return err; } -static int page_pool_release_dma_index(struct page_pool *pool, - netmem_ref netmem) +static void __page_pool_unmap_netmem_dma(struct page_pool *pool, + netmem_ref netmem) { struct page *old, *page = netmem_to_page(netmem); unsigned long id; + dma_addr_t dma; - if (unlikely(!PP_DMA_INDEX_BITS)) - return 0; + if (!pool->dma_map) + return; - id = netmem_get_dma_index(netmem); - if (!id) - return -1; + /* Cache dma_addr before xa_cmpxchg. The scrub path holds no page ref; + * the unref path calls put_page() regardless of cmpxchg outcome, so + * after the cmpxchg we cannot safely touch netmem fields. + */ + dma = page_pool_get_dma_addr_netmem(netmem); - if (in_softirq()) - old = xa_cmpxchg(&pool->dma_mapped, id, page, NULL, 0); - else - old = xa_cmpxchg_bh(&pool->dma_mapped, id, page, NULL, 0); - if (old != page) - return -1; + if (likely(PP_DMA_INDEX_BITS)) { + id = netmem_get_dma_index(netmem); + if (!id) + return; - netmem_set_dma_index(netmem, 0); + if (in_softirq()) + old = xa_cmpxchg(&pool->dma_mapped, + id, page, NULL, 0); + else + old = xa_cmpxchg_bh(&pool->dma_mapped, + id, page, NULL, 0); + if (old != page) + return; + } - return 0; + dma_unmap_page_attrs(pool->p.dev, dma, + PAGE_SIZE << pool->p.order, pool->p.dma_dir, + DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); } static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem, gfp_t gfp) @@ -728,24 +739,16 @@ void page_pool_clear_pp_info(netmem_ref netmem) static __always_inline void __page_pool_release_netmem_dma(struct page_pool *pool, netmem_ref netmem) { - dma_addr_t dma; - + /* Caller must hold a page ref: __page_pool_unmap_netmem_dma() is + * safe without a ref, but the field clears below require it. + */ if (!pool->dma_map) - /* Always account for inflight pages, even if we didn't - * map them - */ return; - if (page_pool_release_dma_index(pool, netmem)) - return; - - dma = page_pool_get_dma_addr_netmem(netmem); - - /* When page is unmapped, it cannot be returned to our pool */ - dma_unmap_page_attrs(pool->p.dev, dma, - PAGE_SIZE << pool->p.order, pool->p.dma_dir, - DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); + __page_pool_unmap_netmem_dma(pool, netmem); page_pool_set_dma_addr_netmem(netmem, 0); + if (likely(PP_DMA_INDEX_BITS)) + netmem_set_dma_index(netmem, 0); } /* Disconnects a page (from a page_pool). API users can have a need @@ -1171,8 +1174,9 @@ static void page_pool_scrub(struct page_pool *pool) synchronize_net(); } + /* No page ref, dma-unmap only. */ xa_for_each(&pool->dma_mapped, id, ptr) - __page_pool_release_netmem_dma(pool, page_to_netmem((struct page *)ptr)); + __page_pool_unmap_netmem_dma(pool, page_to_netmem((struct page *)ptr)); } /* No more consumers should exist, but producers could still