mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-08-31 14:04:27 -04:00
mm/swap: introduce struct swap_io_ctx
Generalize the context currently provided by double pointers to struct swap_iocb to an on-stack context. This cleans up the code and prepares for adding more fields and supporting batching multiple folios into a single bio for block-based swap as well. This new swap_io_ctx is required for all functions using it, the old way of allowing a NULL iocb for some callers is removed to keep the interface consistent. To reduce code duplication caused by this, a new swap_cache_read_folio_sync helper is added to consolidate the code to call swap_cache_read_folio with a local swap_io_ctx. The unpug helpers are renamed to use the submit wording as they are generalized. Link: https://lore.kernel.org/20260713093350.2154226-3-hch@lst.de Signed-off-by: Christoph Hellwig <hch@lst.de> Reviewed-by: Nhat Pham <nphamcs@gmail.com> Cc: Baolin Wang <baolin.wang@linux.alibaba.com> Cc: Baoquan He <baoquan.he@linux.dev> Cc: Barry Song <baohua@kernel.org> Cc: Chris Li <chrisl@kernel.org> Cc: Kairui Song <kasong@tencent.com> Cc: Kemeng Shi <shikemeng@huaweicloud.com> Cc: Youngjun Park <youngjun.park@lge.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
committed by
Andrew Morton
parent
a8efc69a65
commit
8f29aa226f
16
mm/madvise.c
16
mm/madvise.c
@@ -188,7 +188,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
|
||||
unsigned long end, struct mm_walk *walk)
|
||||
{
|
||||
struct vm_area_struct *vma = walk->private;
|
||||
struct swap_iocb *splug = NULL;
|
||||
struct swap_io_ctx ctx = {};
|
||||
pte_t *ptep = NULL;
|
||||
spinlock_t *ptl;
|
||||
unsigned long addr;
|
||||
@@ -212,15 +212,15 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
|
||||
pte_unmap_unlock(ptep, ptl);
|
||||
ptep = NULL;
|
||||
|
||||
folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
|
||||
vma, addr, &splug);
|
||||
folio = read_swap_cache_async(&ctx, entry, GFP_HIGHUSER_MOVABLE,
|
||||
vma, addr);
|
||||
if (folio)
|
||||
folio_put(folio);
|
||||
}
|
||||
|
||||
if (ptep)
|
||||
pte_unmap_unlock(ptep, ptl);
|
||||
swap_read_unplug(splug);
|
||||
swap_read_submit(&ctx);
|
||||
cond_resched();
|
||||
|
||||
return 0;
|
||||
@@ -238,7 +238,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma,
|
||||
XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start));
|
||||
pgoff_t end_index = linear_page_index(vma, end) - 1;
|
||||
struct folio *folio;
|
||||
struct swap_iocb *splug = NULL;
|
||||
struct swap_io_ctx ctx = {};
|
||||
|
||||
rcu_read_lock();
|
||||
xas_for_each(&xas, folio, end_index) {
|
||||
@@ -257,15 +257,15 @@ static void shmem_swapin_range(struct vm_area_struct *vma,
|
||||
xas_pause(&xas);
|
||||
rcu_read_unlock();
|
||||
|
||||
folio = read_swap_cache_async(entry, mapping_gfp_mask(mapping),
|
||||
vma, addr, &splug);
|
||||
folio = read_swap_cache_async(&ctx, entry,
|
||||
mapping_gfp_mask(mapping), vma, addr);
|
||||
if (folio)
|
||||
folio_put(folio);
|
||||
|
||||
rcu_read_lock();
|
||||
}
|
||||
rcu_read_unlock();
|
||||
swap_read_unplug(splug);
|
||||
swap_read_submit(&ctx);
|
||||
}
|
||||
#endif /* CONFIG_SWAP */
|
||||
|
||||
|
||||
60
mm/page_io.c
60
mm/page_io.c
@@ -248,7 +248,7 @@ static void swap_zeromap_folio_clear(struct folio *folio)
|
||||
* We may have stale swap cache pages in memory: notice
|
||||
* them here and get rid of the unnecessary final write.
|
||||
*/
|
||||
int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug)
|
||||
int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
@@ -295,7 +295,7 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug)
|
||||
}
|
||||
rcu_read_unlock();
|
||||
|
||||
__swap_writepage(folio, swap_plug);
|
||||
__swap_writepage(ctx, folio);
|
||||
return 0;
|
||||
out_unlock:
|
||||
folio_unlock(folio);
|
||||
@@ -390,9 +390,9 @@ static void sio_write_complete(struct kiocb *iocb, long ret)
|
||||
mempool_free(sio, sio_pool);
|
||||
}
|
||||
|
||||
static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)
|
||||
static void swap_writepage_fs(struct swap_io_ctx *ctx, struct folio *folio)
|
||||
{
|
||||
struct swap_iocb *sio = swap_plug ? *swap_plug : NULL;
|
||||
struct swap_iocb *sio = ctx->sio;
|
||||
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
|
||||
struct file *swap_file = sis->swap_file;
|
||||
loff_t pos = swap_dev_pos(folio->swap);
|
||||
@@ -403,7 +403,7 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)
|
||||
if (sio) {
|
||||
if (sio->iocb.ki_filp != swap_file ||
|
||||
sio->iocb.ki_pos + sio->len != pos) {
|
||||
swap_write_unplug(sio);
|
||||
swap_write_submit(ctx);
|
||||
sio = NULL;
|
||||
}
|
||||
}
|
||||
@@ -418,12 +418,11 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)
|
||||
bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);
|
||||
sio->len += folio_size(folio);
|
||||
sio->nr_bvecs += 1;
|
||||
if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !swap_plug) {
|
||||
swap_write_unplug(sio);
|
||||
if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs)) {
|
||||
swap_write_submit(ctx);
|
||||
sio = NULL;
|
||||
}
|
||||
if (swap_plug)
|
||||
*swap_plug = sio;
|
||||
ctx->sio = sio;
|
||||
}
|
||||
|
||||
static void swap_writepage_bdev_sync(struct folio *folio,
|
||||
@@ -463,7 +462,7 @@ static void swap_writepage_bdev_async(struct folio *folio,
|
||||
submit_bio(bio);
|
||||
}
|
||||
|
||||
void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)
|
||||
void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio)
|
||||
{
|
||||
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
|
||||
|
||||
@@ -474,7 +473,7 @@ void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)
|
||||
* is safe.
|
||||
*/
|
||||
if (data_race(sis->flags & SWP_FS_OPS))
|
||||
swap_writepage_fs(folio, swap_plug);
|
||||
swap_writepage_fs(ctx, folio);
|
||||
/*
|
||||
* ->flags can be updated non-atomically,
|
||||
* but that will never affect SWP_SYNCHRONOUS_IO, so the data_race
|
||||
@@ -486,16 +485,20 @@ void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)
|
||||
swap_writepage_bdev_async(folio, sis);
|
||||
}
|
||||
|
||||
void swap_write_unplug(struct swap_iocb *sio)
|
||||
void swap_write_submit(struct swap_io_ctx *ctx)
|
||||
{
|
||||
struct swap_iocb *sio = ctx->sio;
|
||||
struct iov_iter from;
|
||||
struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
|
||||
int ret;
|
||||
|
||||
if (!sio)
|
||||
return;
|
||||
|
||||
iov_iter_bvec(&from, ITER_SOURCE, sio->bvecs, sio->nr_bvecs, sio->len);
|
||||
ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
|
||||
ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw(&sio->iocb, &from);
|
||||
if (ret != -EIOCBQUEUED)
|
||||
sio_write_complete(&sio->iocb, ret);
|
||||
ctx->sio = NULL;
|
||||
}
|
||||
|
||||
static void sio_read_complete(struct kiocb *iocb, long ret)
|
||||
@@ -587,18 +590,16 @@ static bool swap_read_folio_zeromap(struct folio *folio)
|
||||
return true;
|
||||
}
|
||||
|
||||
static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug)
|
||||
static void swap_read_folio_fs(struct swap_io_ctx *ctx, struct folio *folio)
|
||||
{
|
||||
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
|
||||
struct swap_iocb *sio = NULL;
|
||||
struct swap_iocb *sio = ctx->sio;
|
||||
loff_t pos = swap_dev_pos(folio->swap);
|
||||
|
||||
if (plug)
|
||||
sio = *plug;
|
||||
if (sio) {
|
||||
if (sio->iocb.ki_filp != sis->swap_file ||
|
||||
sio->iocb.ki_pos + sio->len != pos) {
|
||||
swap_read_unplug(sio);
|
||||
swap_read_submit(ctx);
|
||||
sio = NULL;
|
||||
}
|
||||
}
|
||||
@@ -613,12 +614,11 @@ static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug)
|
||||
bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);
|
||||
sio->len += folio_size(folio);
|
||||
sio->nr_bvecs += 1;
|
||||
if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !plug) {
|
||||
swap_read_unplug(sio);
|
||||
if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs)) {
|
||||
swap_read_submit(ctx);
|
||||
sio = NULL;
|
||||
}
|
||||
if (plug)
|
||||
*plug = sio;
|
||||
ctx->sio = sio;
|
||||
}
|
||||
|
||||
static void swap_read_folio_bdev_sync(struct folio *folio,
|
||||
@@ -658,7 +658,7 @@ static void swap_read_folio_bdev_async(struct folio *folio,
|
||||
submit_bio(bio);
|
||||
}
|
||||
|
||||
void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
|
||||
void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
|
||||
{
|
||||
struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
|
||||
bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO;
|
||||
@@ -693,7 +693,7 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
|
||||
zswap_folio_swapin(folio);
|
||||
|
||||
if (data_race(sis->flags & SWP_FS_OPS)) {
|
||||
swap_read_folio_fs(folio, plug);
|
||||
swap_read_folio_fs(ctx, folio);
|
||||
} else if (synchronous) {
|
||||
swap_read_folio_bdev_sync(folio, sis);
|
||||
} else {
|
||||
@@ -708,14 +708,18 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
|
||||
delayacct_swapin_end();
|
||||
}
|
||||
|
||||
void __swap_read_unplug(struct swap_iocb *sio)
|
||||
void swap_read_submit(struct swap_io_ctx *ctx)
|
||||
{
|
||||
struct swap_iocb *sio = ctx->sio;
|
||||
struct iov_iter from;
|
||||
struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
|
||||
int ret;
|
||||
|
||||
if (!sio)
|
||||
return;
|
||||
|
||||
iov_iter_bvec(&from, ITER_DEST, sio->bvecs, sio->nr_bvecs, sio->len);
|
||||
ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
|
||||
ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw(&sio->iocb, &from);
|
||||
if (ret != -EIOCBQUEUED)
|
||||
sio_read_complete(&sio->iocb, ret);
|
||||
ctx->sio = NULL;
|
||||
}
|
||||
|
||||
13
mm/shmem.c
13
mm/shmem.c
@@ -1597,13 +1597,13 @@ int shmem_unuse(unsigned int type)
|
||||
|
||||
/**
|
||||
* shmem_writeout - Write the folio to swap
|
||||
* @ctx: swap I/O context
|
||||
* @folio: The folio to write
|
||||
* @plug: swap plug
|
||||
* @folio_list: list to put back folios on split
|
||||
*
|
||||
* Move the folio from the page cache to the swap cache.
|
||||
*/
|
||||
int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
|
||||
int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
|
||||
struct list_head *folio_list)
|
||||
{
|
||||
struct address_space *mapping = folio->mapping;
|
||||
@@ -1715,7 +1715,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
|
||||
shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap));
|
||||
|
||||
BUG_ON(folio_mapped(folio));
|
||||
error = swap_writeout(folio, plug);
|
||||
error = swap_writeout(ctx, folio);
|
||||
if (error != AOP_WRITEPAGE_ACTIVATE) {
|
||||
/* folio has been unlocked */
|
||||
return error;
|
||||
@@ -1754,7 +1754,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
|
||||
|
||||
int shmem_write_folio(struct folio *folio)
|
||||
{
|
||||
return shmem_writeout(folio, NULL, NULL);
|
||||
struct swap_io_ctx ctx = {};
|
||||
int err;
|
||||
|
||||
err = shmem_writeout(&ctx, folio, NULL);
|
||||
swap_write_submit(&ctx);
|
||||
return err;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(shmem_write_folio);
|
||||
|
||||
|
||||
35
mm/swap.h
35
mm/swap.h
@@ -91,6 +91,10 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
|
||||
return READ_ONCE(vm_swappiness);
|
||||
}
|
||||
|
||||
struct swap_io_ctx {
|
||||
struct swap_iocb *sio;
|
||||
};
|
||||
|
||||
#ifdef CONFIG_SWAP
|
||||
#include <linux/swapops.h> /* for swp_offset */
|
||||
#include <linux/blk_types.h> /* for bio_end_io_t */
|
||||
@@ -253,17 +257,11 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si,
|
||||
|
||||
/* linux/mm/page_io.c */
|
||||
int sio_pool_init(void);
|
||||
struct swap_iocb;
|
||||
void swap_read_folio(struct folio *folio, struct swap_iocb **plug);
|
||||
void __swap_read_unplug(struct swap_iocb *plug);
|
||||
static inline void swap_read_unplug(struct swap_iocb *plug)
|
||||
{
|
||||
if (unlikely(plug))
|
||||
__swap_read_unplug(plug);
|
||||
}
|
||||
void swap_write_unplug(struct swap_iocb *sio);
|
||||
int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug);
|
||||
void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug);
|
||||
void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio);
|
||||
void swap_read_submit(struct swap_io_ctx *ctx);
|
||||
void swap_write_submit(struct swap_io_ctx *ctx);
|
||||
int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio);
|
||||
void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio);
|
||||
|
||||
/* linux/mm/swap_state.c */
|
||||
extern struct address_space swap_space __read_mostly;
|
||||
@@ -330,9 +328,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,
|
||||
|
||||
void show_swap_cache_info(void);
|
||||
void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr);
|
||||
struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
|
||||
struct vm_area_struct *vma, unsigned long addr,
|
||||
struct swap_iocb **plug);
|
||||
struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry,
|
||||
gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr);
|
||||
struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
|
||||
struct mempolicy *mpol, pgoff_t ilx);
|
||||
struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,
|
||||
@@ -348,7 +345,6 @@ static inline unsigned int folio_swap_flags(struct folio *folio)
|
||||
}
|
||||
|
||||
#else /* CONFIG_SWAP */
|
||||
struct swap_iocb;
|
||||
static inline struct swap_cluster_info *swap_cluster_lock(
|
||||
struct swap_info_struct *si, pgoff_t offset, bool irq)
|
||||
{
|
||||
@@ -394,11 +390,11 @@ static inline void folio_put_swap(struct folio *folio, struct page *page)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
|
||||
static inline void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void swap_write_unplug(struct swap_iocb *sio)
|
||||
static inline void swap_write_submit(struct swap_io_ctx *ctx)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -440,8 +436,7 @@ static inline void swap_update_readahead(struct folio *folio,
|
||||
{
|
||||
}
|
||||
|
||||
static inline int swap_writeout(struct folio *folio,
|
||||
struct swap_iocb **swap_plug)
|
||||
static inline int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
@@ -487,7 +482,7 @@ static inline unsigned int folio_swap_flags(struct folio *folio)
|
||||
|
||||
#endif /* CONFIG_SWAP */
|
||||
|
||||
int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
|
||||
int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
|
||||
struct list_head *folio_list);
|
||||
|
||||
#endif /* _MM_SWAP_H */
|
||||
|
||||
@@ -638,9 +638,9 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma,
|
||||
}
|
||||
}
|
||||
|
||||
static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,
|
||||
struct mempolicy *mpol, pgoff_t ilx,
|
||||
struct swap_iocb **plug, bool readahead)
|
||||
static struct folio *swap_cache_read_folio(struct swap_io_ctx *ctx,
|
||||
swp_entry_t entry, gfp_t gfp, struct mempolicy *mpol,
|
||||
pgoff_t ilx, bool readahead)
|
||||
{
|
||||
struct folio *folio;
|
||||
|
||||
@@ -654,7 +654,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,
|
||||
if (IS_ERR_OR_NULL(folio))
|
||||
return NULL;
|
||||
|
||||
swap_read_folio(folio, plug);
|
||||
swap_read_folio(ctx, folio);
|
||||
if (readahead) {
|
||||
folio_set_readahead(folio);
|
||||
count_vm_event(SWAP_RA);
|
||||
@@ -682,6 +682,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,
|
||||
struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,
|
||||
struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx)
|
||||
{
|
||||
struct swap_io_ctx ctx = {};
|
||||
struct folio *folio;
|
||||
|
||||
do {
|
||||
@@ -694,7 +695,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,
|
||||
if (IS_ERR(folio))
|
||||
return folio;
|
||||
|
||||
swap_read_folio(folio, NULL);
|
||||
swap_read_folio(&ctx, folio);
|
||||
swap_read_submit(&ctx);
|
||||
return folio;
|
||||
}
|
||||
|
||||
@@ -704,9 +706,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,
|
||||
* A failure return means that either the page allocation failed or that
|
||||
* the swap entry is no longer in use.
|
||||
*/
|
||||
struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
|
||||
struct vm_area_struct *vma, unsigned long addr,
|
||||
struct swap_iocb **plug)
|
||||
struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry,
|
||||
gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr)
|
||||
{
|
||||
struct swap_info_struct *si;
|
||||
struct mempolicy *mpol;
|
||||
@@ -718,13 +719,24 @@ struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
|
||||
return NULL;
|
||||
|
||||
mpol = get_vma_policy(vma, addr, 0, &ilx);
|
||||
folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, plug, false);
|
||||
folio = swap_cache_read_folio(ctx, entry, gfp_mask, mpol, ilx, false);
|
||||
mpol_cond_put(mpol);
|
||||
|
||||
put_swap_device(si);
|
||||
return folio;
|
||||
}
|
||||
|
||||
static struct folio *swap_cache_read_folio_sync(swp_entry_t entry, gfp_t gfp,
|
||||
struct mempolicy *mpol, pgoff_t ilx)
|
||||
{
|
||||
struct swap_io_ctx ctx = {};
|
||||
struct folio *folio;
|
||||
|
||||
folio = swap_cache_read_folio(&ctx, entry, gfp, mpol, ilx, false);
|
||||
swap_read_submit(&ctx);
|
||||
return folio;
|
||||
}
|
||||
|
||||
static unsigned int __swapin_nr_pages(unsigned long prev_offset,
|
||||
unsigned long offset,
|
||||
int hits,
|
||||
@@ -813,8 +825,8 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
|
||||
unsigned long start_offset, end_offset;
|
||||
unsigned long mask;
|
||||
struct swap_info_struct *si = __swap_entry_to_info(entry);
|
||||
struct swap_io_ctx ctx = {};
|
||||
struct blk_plug plug;
|
||||
struct swap_iocb *splug = NULL;
|
||||
swp_entry_t ra_entry;
|
||||
|
||||
mask = swapin_nr_pages(offset) - 1;
|
||||
@@ -833,17 +845,16 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
|
||||
for (offset = start_offset; offset <= end_offset ; offset++) {
|
||||
/* Ok, do the async read-ahead now */
|
||||
ra_entry = swp_entry(swp_type(entry), offset);
|
||||
folio = swap_cache_read_folio(ra_entry, gfp_mask, mpol, ilx,
|
||||
&splug, offset != entry_offset);
|
||||
folio = swap_cache_read_folio(&ctx, ra_entry, gfp_mask, mpol,
|
||||
ilx, offset != entry_offset);
|
||||
if (!folio)
|
||||
continue;
|
||||
folio_put(folio);
|
||||
}
|
||||
blk_finish_plug(&plug);
|
||||
swap_read_unplug(splug);
|
||||
swap_read_submit(&ctx);
|
||||
skip:
|
||||
/* The page was likely read above, so no need for plugging here */
|
||||
return swap_cache_read_folio(entry, gfp_mask, mpol, ilx, NULL, false);
|
||||
return swap_cache_read_folio_sync(entry, gfp_mask, mpol, ilx);
|
||||
}
|
||||
|
||||
static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,
|
||||
@@ -903,8 +914,8 @@ static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,
|
||||
static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
|
||||
struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf)
|
||||
{
|
||||
struct swap_io_ctx ctx = {};
|
||||
struct blk_plug plug;
|
||||
struct swap_iocb *splug = NULL;
|
||||
struct folio *folio;
|
||||
pte_t *pte = NULL, pentry;
|
||||
int win;
|
||||
@@ -943,8 +954,8 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
|
||||
if (!si)
|
||||
continue;
|
||||
}
|
||||
folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx,
|
||||
&splug, addr != vmf->address);
|
||||
folio = swap_cache_read_folio(&ctx, entry, gfp_mask, mpol, ilx,
|
||||
addr != vmf->address);
|
||||
if (si)
|
||||
put_swap_device(si);
|
||||
if (!folio)
|
||||
@@ -954,12 +965,10 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
|
||||
if (pte)
|
||||
pte_unmap(pte);
|
||||
blk_finish_plug(&plug);
|
||||
swap_read_unplug(splug);
|
||||
swap_read_submit(&ctx);
|
||||
skip:
|
||||
/* The folio was likely read above, so no need for plugging here */
|
||||
folio = swap_cache_read_folio(targ_entry, gfp_mask, mpol, targ_ilx,
|
||||
NULL, false);
|
||||
return folio;
|
||||
return swap_cache_read_folio_sync(targ_entry, gfp_mask, mpol, targ_ilx);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
15
mm/vmscan.c
15
mm/vmscan.c
@@ -617,8 +617,8 @@ typedef enum {
|
||||
/*
|
||||
* pageout is called by shrink_folio_list() for each dirty folio.
|
||||
*/
|
||||
static pageout_t pageout(struct folio *folio, struct address_space *mapping,
|
||||
struct swap_iocb **plug, struct list_head *folio_list)
|
||||
static pageout_t pageout(struct swap_io_ctx *ctx, struct address_space *mapping,
|
||||
struct folio *folio, struct list_head *folio_list)
|
||||
{
|
||||
int res;
|
||||
|
||||
@@ -654,9 +654,9 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,
|
||||
* the split out folios get added back to folio_list.
|
||||
*/
|
||||
if (shmem_mapping(mapping))
|
||||
res = shmem_writeout(folio, plug, folio_list);
|
||||
res = shmem_writeout(ctx, folio, folio_list);
|
||||
else
|
||||
res = swap_writeout(folio, plug);
|
||||
res = swap_writeout(ctx, folio);
|
||||
|
||||
if (res < 0)
|
||||
handle_write_error(mapping, folio, res);
|
||||
@@ -1066,7 +1066,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
|
||||
unsigned int nr_reclaimed = 0, nr_demoted = 0;
|
||||
unsigned int pgactivate = 0;
|
||||
bool do_demote_pass;
|
||||
struct swap_iocb *plug = NULL;
|
||||
struct swap_io_ctx ctx = {};
|
||||
|
||||
folio_batch_init(&free_folios);
|
||||
memset(stat, 0, sizeof(*stat));
|
||||
@@ -1394,7 +1394,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
|
||||
* starts and then write it out here.
|
||||
*/
|
||||
try_to_unmap_flush_dirty();
|
||||
switch (pageout(folio, mapping, &plug, folio_list)) {
|
||||
switch (pageout(&ctx, mapping, folio, folio_list)) {
|
||||
case PAGE_KEEP:
|
||||
goto keep_locked;
|
||||
case PAGE_ACTIVATE:
|
||||
@@ -1582,8 +1582,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
|
||||
list_splice(&ret_folios, folio_list);
|
||||
count_vm_events(PGACTIVATE, pgactivate);
|
||||
|
||||
if (plug)
|
||||
swap_write_unplug(plug);
|
||||
swap_write_submit(&ctx);
|
||||
return nr_reclaimed;
|
||||
}
|
||||
|
||||
|
||||
@@ -992,6 +992,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
|
||||
struct folio *folio;
|
||||
struct mempolicy *mpol;
|
||||
struct swap_info_struct *si;
|
||||
struct swap_io_ctx ctx = {};
|
||||
int ret = 0;
|
||||
|
||||
/* try to allocate swap cache folio */
|
||||
@@ -1049,7 +1050,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
|
||||
folio_set_reclaim(folio);
|
||||
|
||||
/* start writeback */
|
||||
__swap_writepage(folio, NULL);
|
||||
__swap_writepage(&ctx, folio);
|
||||
swap_write_submit(&ctx);
|
||||
|
||||
out:
|
||||
if (ret) {
|
||||
|
||||
Reference in New Issue
Block a user