mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 01:11:51 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'vfs-7.3-rc1.iomap' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs

Browse Source 
Pull iomap updates from Christian Brauner:
 "The bulk of this is the conversion of iomap to a single ->iomap_next()
  callback and thus finishing the move to an iterator model.

  Every iomap operation drove its iteration through a struct iomap_ops
  holding ->iomap_begin() and ->iomap_end(). iomap_iter() only ever sees
  those as pointers. That means every step of every iteration is an
  indirect call.

  This collapses both into one ->iomap_next() callback that finishes the
  previous mapping and produces the next one. This lets callers inline
  the iteration loop and pass its ->iomap_next() as a compile time
  constant. That means the compiler can turn it into a direct and hence
  inlineable call.

  This also allows future callers to express custom logic to drive the
  iteration forward better. xfs, btrfs, ext4, ext2, erofs, f2fs, gfs2,
  hpfs, fuse, exfat, zonefs, ntfs, ntfs3 and the block device mapping
  are all converted. No functional changes are intended.

  This also adds a simple direct I/O path for small reads. On Gen5 NVMe
  the __iomap_dio_rw() dominates 4K random reads. The same single-core
  io_uring poll mode workload reaches ~3.2M IOPS against the raw block
  device but only ~1.92M through ext4 or XFS.

  __iomap_dio_rw(), iomap_iter(), iomap_dio_bio_iter() and kfree() were
  at the top of the profile. The new path is very lightweight if no
  special behavior is requested. The bio comes from a dedicated bioset
  and laid out so the whole request is a single cacheline aligned
  allocation. Completion runs inline.

  That takes ext4 from 1.92M to 2.19M IOPS in the original workload. fio
  shows around:
   - 4% at libaio queue depths of 64 and up
   - around 5% for io_uring
   - up to 10% for io_uring poll mode at depth 256
  on both ext4 and xfs.

  A few other patches:

   - iomap_folio_mark_uptodate() lets a filesystem that writes into the
     page cache outside the iomap read and write paths keep iomap's
     internal uptodate bitmap in sync, which fuse needs for
     server-pushed notify stores before it can enable large folios;

   - two fixes for iomap_bio_read_folio_range_sync(): a potential crash
     when device integrity behavior is changed and a missing
     bio_uninit().

   - a folio batch release fix on iomap callback failures

   - FGP_NOFS is dropped from iomap_get_folio()

   - documentation fix"

* tag 'vfs-7.3-rc1.iomap' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs: (29 commits)
  iomap: iomap_bio_read_folio_range_sync is missing a call to bio_uninit
  iomap: don't free integrity payload that doesn't exist
  docs: fix grammatical error in iomap docs
  exfat: convert iomap ops to ->iomap_next()
  fuse: convert iomap ops to ->iomap_next()
  hpfs: convert iomap ops to ->iomap_next()
  gfs2: convert iomap ops to ->iomap_next()
  f2fs: convert iomap ops to ->iomap_next()
  block: convert iomap ops to ->iomap_next()
  ext2: convert iomap ops to ->iomap_next()
  zonefs: convert iomap ops to ->iomap_next()
  erofs: convert iomap ops to ->iomap_next()
  ext4: convert iomap ops to ->iomap_next()
  ntfs: convert iomap ops to ->iomap_next()
  ntfs3: convert iomap ops to ->iomap_next()
  btrfs: convert iomap ops to ->iomap_next()
  xfs: convert iomap ops to ->iomap_next()
  iomap: add ->iomap_next()
  iomap: use GFP_NOWAIT when application for iomap_dio_simple allocations
  iomap: decouple simple direct I/O reads from iomap_dio_rw
  ...
This commit is contained in:
Linus Torvalds
2026-08-17 10:03:55 -07:00
parent 3d1f952677 9948bc9aa1
commit 9ea8d6197d
29 changed files with 570 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Documentation/filesystems/iomap/design.rst
View File

@@ -43,7 +43,7 @@ as:
* lseek ``SEEK_DATA`` and ``SEEK_HOLE``
* swapfile activation
This origins of this library is the file I/O path that XFS once used; it
The origins of this library is the file I/O path that XFS once used; it
has now been extended to cover several other operations.
Who Should Read This?

4
block/fops.c
View File

@@ -453,8 +453,10 @@ static int blkdev_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT(blkdev_iomap_next, blkdev_iomap_begin);
static const struct iomap_ops blkdev_iomap_ops = {
.iomap_begin = blkdev_iomap_begin,
.iomap_next = blkdev_iomap_next,
};
#ifdef CONFIG_BUFFER_HEAD

6
fs/btrfs/direct-io.c
View File

@@ -798,9 +798,11 @@ static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio,
btrfs_submit_bbio(bbio, 0);
}
static DEFINE_IOMAP_ITER_NEXT_END(btrfs_dio_iomap_next, btrfs_dio_iomap_begin,
btrfs_dio_iomap_end);
static const struct iomap_ops btrfs_dio_iomap_ops = {
.iomap_begin = btrfs_dio_iomap_begin,
.iomap_end = btrfs_dio_iomap_end,
.iomap_next = btrfs_dio_iomap_next,
};
static const struct iomap_dio_ops btrfs_dio_ops = {

6
fs/erofs/data.c
View File

@@ -380,9 +380,11 @@ static int erofs_iomap_end(struct inode *inode, loff_t pos, loff_t length,
return written;
}
static DEFINE_IOMAP_ITER_NEXT_END(erofs_iomap_next, erofs_iomap_begin,
erofs_iomap_end);
static const struct iomap_ops erofs_iomap_ops = {
.iomap_begin = erofs_iomap_begin,
.iomap_end = erofs_iomap_end,
.iomap_next = erofs_iomap_next,
};
int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,

5
fs/erofs/zmap.c
View File

@@ -822,6 +822,9 @@ static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT(z_erofs_iomap_next_report,
z_erofs_iomap_begin_report);
const struct iomap_ops z_erofs_iomap_report_ops = {
.iomap_begin = z_erofs_iomap_begin_report,
.iomap_next = z_erofs_iomap_next_report,
};

10
fs/exfat/iomap.c
View File

@@ -151,8 +151,10 @@ static int exfat_write_iomap_begin(struct inode *inode, loff_t offset, loff_t le
return __exfat_iomap_begin(inode, offset, length, flags, iomap, true);
}
static DEFINE_IOMAP_ITER_NEXT(exfat_iomap_next, exfat_iomap_begin);
const struct iomap_ops exfat_iomap_ops = {
.iomap_begin = exfat_iomap_begin,
.iomap_next = exfat_iomap_next,
};
/*
@@ -186,9 +188,11 @@ static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length,
return written;
}
static DEFINE_IOMAP_ITER_NEXT_END(exfat_write_iomap_next,
exfat_write_iomap_begin, exfat_write_iomap_end);
const struct iomap_ops exfat_write_iomap_ops = {
.iomap_begin = exfat_write_iomap_begin,
.iomap_end = exfat_write_iomap_end,
.iomap_next = exfat_write_iomap_next,
};
/*

6
fs/ext2/inode.c
View File

@@ -860,9 +860,11 @@ ext2_iomap_end(struct inode *inode, loff_t offset, loff_t length,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT_END(ext2_iomap_next, ext2_iomap_begin,
ext2_iomap_end);
const struct iomap_ops ext2_iomap_ops = {
.iomap_begin = ext2_iomap_begin,
.iomap_end = ext2_iomap_end,
.iomap_next = ext2_iomap_next,
};
int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,

3
fs/ext4/ext4.h
View File

@@ -4007,6 +4007,9 @@ static inline void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end)
extern const struct iomap_ops ext4_iomap_ops;
extern const struct iomap_ops ext4_iomap_report_ops;
int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
unsigned flags, struct iomap *iomap, struct iomap *srcmap);
static inline int ext4_buffer_uptodate(struct buffer_head *bh)
{
/*

4
fs/ext4/extents.c
View File

@@ -5171,8 +5171,10 @@ static int ext4_iomap_xattr_begin(struct inode *inode, loff_t offset,
return error;
}
static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_xattr_next, ext4_iomap_xattr_begin);
static const struct iomap_ops ext4_iomap_xattr_ops = {
.iomap_begin = ext4_iomap_xattr_begin,
.iomap_next = ext4_iomap_xattr_next,
};
static int ext4_fiemap_check_ranges(struct inode *inode, u64 start, u64 *len)

4
fs/ext4/file.c
View File

@@ -91,7 +91,9 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)
return generic_file_read_iter(iocb, to);
}
ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0);
ret = iomap_dio_read_simple(iocb, to, ext4_iomap_begin);
if (ret == -ENOTBLK)
ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0);
inode_unlock_shared(inode);
file_accessed(iocb->ki_filp);

10
fs/ext4/inode.c
View File

@@ -3771,7 +3771,7 @@ static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
}
static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
unsigned flags, struct iomap *iomap, struct iomap *srcmap)
{
int ret;
@@ -3850,8 +3850,10 @@ static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin);
const struct iomap_ops ext4_iomap_ops = {
.iomap_begin = ext4_iomap_begin,
.iomap_next = ext4_iomap_next,
};
static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,
@@ -3905,8 +3907,10 @@ static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report);
const struct iomap_ops ext4_iomap_report_ops = {
.iomap_begin = ext4_iomap_begin_report,
.iomap_next = ext4_iomap_next_report,
};
/*

4
fs/f2fs/data.c
View File

@@ -4653,6 +4653,8 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT(f2fs_iomap_next, f2fs_iomap_begin);
const struct iomap_ops f2fs_iomap_ops = {
.iomap_begin = f2fs_iomap_begin,
.iomap_next = f2fs_iomap_next,
};

6
fs/fuse/dax.c
View File

@@ -653,9 +653,11 @@ static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT_END(fuse_iomap_next, fuse_iomap_begin,
fuse_iomap_end);
static const struct iomap_ops fuse_iomap_ops = {
.iomap_begin = fuse_iomap_begin,
.iomap_end = fuse_iomap_end,
.iomap_next = fuse_iomap_next,
};
static void fuse_wait_dax_page(struct inode *inode)

24
fs/fuse/file.c
View File

@@ -890,8 +890,10 @@ static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT(fuse_iomap_next, fuse_iomap_begin);
static const struct iomap_ops fuse_iomap_ops = {
.iomap_begin = fuse_iomap_begin,
.iomap_next = fuse_iomap_next,
};
struct fuse_fill_read_data {
@@ -1219,8 +1221,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
struct file *file = iocb->ki_filp;
struct fuse_file *ff = file->private_data;
struct fuse_mount *fm = ff->fm;
unsigned int offset, i;
bool short_write;
unsigned int i;
int err;
for (i = 0; i < ap->num_folios; i++)
@@ -1235,24 +1236,9 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
if (!err && ia->write.out.size > count)
err = -EIO;
short_write = ia->write.out.size < count;
offset = ap->descs[0].offset;
count = ia->write.out.size;
for (i = 0; i < ap->num_folios; i++) {
struct folio *folio = ap->folios[i];
if (err) {
folio_clear_uptodate(folio);
} else {
if (count >= folio_size(folio) - offset)
count -= folio_size(folio) - offset;
else {
if (short_write)
folio_clear_uptodate(folio);
count = 0;
}
offset = 0;
}
if (ia->write.folio_locked && (i == ap->num_folios - 1))
folio_unlock(folio);
folio_put(folio);
@@ -1327,7 +1313,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
/* If we copied full folio, mark it uptodate */
if (tmp == folio_size(folio))
folio_mark_uptodate(folio);
iomap_folio_mark_uptodate(folio);
if (folio_test_uptodate(folio)) {
folio_unlock(folio);

4
fs/fuse/notify.c
View File

@@ -2,6 +2,8 @@
#include "dev.h"
#include "fuse_i.h"
#include <linux/iomap.h>
#include <linux/pagemap.h>
static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
@@ -192,7 +194,7 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
if (!folio_test_uptodate(folio) && !err && folio_offset == 0 &&
(nr_bytes == folio_size(folio) || file_size == end)) {
folio_zero_segment(folio, nr_bytes, folio_size(folio));
folio_mark_uptodate(folio);
iomap_folio_mark_uptodate(folio);
}
folio_unlock(folio);
folio_put(folio);

3
fs/fuse/virtio_fs.c
View File

@@ -1024,8 +1024,7 @@ static void virtio_fs_cleanup_vqs(struct virtio_device *vdev)
}
/* Map a window offset to a page frame number. The window offset will have
* been produced by .iomap_begin(), which maps a file offset to a window
* offset.
* been produced by .iomap_next(), which maps a file offset to a window offset.
*/
static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
long nr_pages, enum dax_access_mode mode,

6
fs/gfs2/bmap.c
View File

@@ -1200,9 +1200,11 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT_END(gfs2_iomap_next, gfs2_iomap_begin,
gfs2_iomap_end);
const struct iomap_ops gfs2_iomap_ops = {
.iomap_begin = gfs2_iomap_begin,
.iomap_end = gfs2_iomap_end,
.iomap_next = gfs2_iomap_next,
};
/**

4
fs/hpfs/file.c
View File

@@ -156,8 +156,10 @@ static int hpfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT(hpfs_iomap_next, hpfs_iomap_begin);
static const struct iomap_ops hpfs_iomap_ops = {
.iomap_begin = hpfs_iomap_begin,
.iomap_next = hpfs_iomap_next,
};
static int hpfs_read_folio(struct file *file, struct folio *folio)

3
fs/iomap/bio.c
View File

@@ -179,10 +179,11 @@ int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter,
if (srcmap->flags & IOMAP_F_INTEGRITY)
fs_bio_integrity_alloc(&bio);
error = submit_bio_wait(&bio);
if (srcmap->flags & IOMAP_F_INTEGRITY) {
if (bio_integrity(&bio)) {
if (!error)
error = fs_bio_integrity_verify(&bio, sector, len);
fs_bio_integrity_free(&bio);
}
bio_uninit(&bio);
return error;
}

8
fs/iomap/buffered-io.c
View File

@@ -107,6 +107,12 @@ static void iomap_set_range_uptodate(struct folio *folio, size_t off,
folio_mark_uptodate(folio);
}
void iomap_folio_mark_uptodate(struct folio *folio)
{
iomap_set_range_uptodate(folio, 0, folio_size(folio));
}
EXPORT_SYMBOL_GPL(iomap_folio_mark_uptodate);
/*
* Find the next dirty block in the folio. end_blk is inclusive.
* If no dirty block is found, this will return end_blk + 1.
@@ -792,7 +798,7 @@ EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate);
*/
struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len)
{
fgf_t fgp = FGP_WRITEBEGIN | FGP_NOFS;
fgf_t fgp = FGP_WRITEBEGIN;
if (iter->flags & IOMAP_NOWAIT)
fgp |= FGP_NOWAIT;

190
fs/iomap/direct-io.c
View File

@@ -10,6 +10,7 @@
#include <linux/iomap.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/fserror.h>
#include <linux/init.h>
#include "internal.h"
#include "trace.h"
@@ -88,9 +89,9 @@ static inline enum fserror_type iomap_dio_err_type(const struct iomap_dio *dio)
return FSERR_DIRECTIO_READ;
}
static inline bool should_report_dio_fserror(const struct iomap_dio *dio)
static inline bool should_report_dio_fserror(int error)
{
switch (dio->error) {
switch (error) {
case 0:
case -EAGAIN:
case -ENOTBLK:
@@ -110,7 +111,7 @@ ssize_t iomap_dio_complete(struct iomap_dio *dio)
if (dops && dops->end_io)
ret = dops->end_io(iocb, dio->size, ret, dio->flags);
if (should_report_dio_fserror(dio))
if (should_report_dio_fserror(dio->error))
fserror_report_io(file_inode(iocb->ki_filp),
iomap_dio_err_type(dio), offset, dio->size,
dio->error, GFP_NOFS);
@@ -403,6 +404,14 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
return ret;
}
static inline unsigned int iomap_dio_alignment(struct inode *inode,
struct block_device *bdev, unsigned int dio_flags)
{
if (dio_flags & IOMAP_DIO_FSBLOCK_ALIGNED)
return i_blocksize(inode);
return bdev_logical_block_size(bdev);
}
static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
{
const struct iomap *iomap = &iter->iomap;
@@ -421,10 +430,7 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
* File systems that write out of place and always allocate new blocks
* need each bio to be block aligned as that's the unit of allocation.
*/
if (dio->flags & IOMAP_DIO_FSBLOCK_ALIGNED)
alignment = fs_block_size;
else
alignment = bdev_logical_block_size(iomap->bdev);
alignment = iomap_dio_alignment(inode, iomap->bdev, dio->flags);
if ((pos | length) & (alignment - 1))
return -EINVAL;
@@ -907,3 +913,173 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
return iomap_dio_complete(dio);
}
EXPORT_SYMBOL_GPL(iomap_dio_rw);
struct iomap_dio_simple {
struct kiocb *iocb;
size_t size;
unsigned int dio_flags;
struct work_struct work;
/*
* Align @bio to a cacheline boundary so that, combined with the
* front_pad passed to bioset_init(), the bio sits at the start of
* a cacheline in memory returned by the (HWCACHE-aligned) bio
* slab. This keeps the hot fields block layer touches on submit
* and completion (bi_iter, bi_status, ...) within a single line.
*/
struct bio bio ____cacheline_aligned_in_smp;
};
static struct bio_set iomap_dio_simple_pool;
static ssize_t iomap_dio_simple_complete(struct iomap_dio_simple *sr)
{
struct bio *bio = &sr->bio;
struct kiocb *iocb = sr->iocb;
struct inode *inode = file_inode(iocb->ki_filp);
ssize_t ret;
if (unlikely(bio->bi_status)) {
ret = blk_status_to_errno(bio->bi_status);
if (should_report_dio_fserror(ret))
fserror_report_io(inode, FSERR_DIRECTIO_READ,
iocb->ki_pos, sr->size, ret,
GFP_NOFS);
} else {
ret = sr->size;
iocb->ki_pos += ret;
}
if (sr->dio_flags & IOMAP_DIO_USER_BACKED) {
bio_check_pages_dirty(bio);
} else {
bio_release_pages(bio, false);
bio_put(bio);
}
inode_dio_end(inode);
trace_iomap_dio_complete(iocb, ret < 0 ? ret : 0, ret);
return ret;
}
static void iomap_dio_simple_complete_work(struct work_struct *work)
{
struct iomap_dio_simple *sr =
container_of(work, struct iomap_dio_simple, work);
struct kiocb *iocb = sr->iocb;
WRITE_ONCE(iocb->private, NULL);
iocb->ki_complete(iocb, iomap_dio_simple_complete(sr));
}
static void iomap_dio_simple_end_io(struct bio *bio)
{
struct iomap_dio_simple *sr =
container_of(bio, struct iomap_dio_simple, bio);
struct kiocb *iocb = sr->iocb;
if (unlikely(sr->bio.bi_status)) {
struct inode *inode = file_inode(iocb->ki_filp);
INIT_WORK(&sr->work, iomap_dio_simple_complete_work);
queue_work(inode->i_sb->s_dio_done_wq, &sr->work);
return;
}
WRITE_ONCE(iocb->private, NULL);
iocb->ki_complete(iocb, iomap_dio_simple_complete(sr));
}
ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter,
struct iomap_iter *iomi)
{
gfp_t gfp = (iomi->flags & IOMAP_NOWAIT) ? GFP_NOWAIT : GFP_KERNEL;
struct iomap_dio_simple *sr;
unsigned int alignment;
struct bio *bio;
ssize_t ret;
if (iomi->iomap.type != IOMAP_MAPPED ||
iomi->iomap.offset + iomi->iomap.length < iomi->pos + iomi->len ||
(iomi->iomap.flags & IOMAP_F_INTEGRITY)) {
ret = -ENOTBLK;
goto out_dio_end;
}
alignment = iomap_dio_alignment(iomi->inode, iomi->iomap.bdev, 0);
if ((iomi->pos | iomi->len) & (alignment - 1)) {
ret = -EINVAL;
goto out_dio_end;
}
if (unlikely(!iomi->inode->i_sb->s_dio_done_wq &&
!is_sync_kiocb(iocb))) {
ret = sb_init_dio_done_wq(iomi->inode->i_sb);
if (ret < 0)
goto out_dio_end;
}
trace_iomap_dio_rw_begin(iocb, iter, 0, 0);
bio = bio_alloc_bioset(iomi->iomap.bdev,
bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS),
REQ_OP_READ, gfp, &iomap_dio_simple_pool);
if (!bio) {
ret = -EAGAIN;
goto out_dio_end;
}
sr = container_of(bio, struct iomap_dio_simple, bio);
sr->iocb = iocb;
sr->dio_flags = 0;
bio->bi_iter.bi_sector = iomap_sector(&iomi->iomap, iomi->pos);
bio->bi_ioprio = iocb->ki_ioprio;
ret = bio_iov_iter_get_pages(bio, iter, alignment - 1);
if (unlikely(ret))
goto out_bio_put;
if (bio->bi_iter.bi_size != iomi->len) {
iov_iter_revert(iter, bio->bi_iter.bi_size);
ret = -ENOTBLK;
goto out_bio_release_pages;
}
sr->size = bio->bi_iter.bi_size;
if (user_backed_iter(iter)) {
bio_set_pages_dirty(bio);
sr->dio_flags |= IOMAP_DIO_USER_BACKED;
}
if (iocb->ki_flags & IOCB_NOWAIT)
bio->bi_opf |= REQ_NOWAIT;
if (is_sync_kiocb(iocb)) {
submit_bio_wait(bio);
return iomap_dio_simple_complete(sr);
}
if ((iocb->ki_flags & IOCB_HIPRI)) {
bio->bi_opf |= REQ_POLLED;
WRITE_ONCE(iocb->private, bio);
}
bio->bi_end_io = iomap_dio_simple_end_io;
submit_bio(bio);
trace_iomap_dio_rw_queued(iomi->inode, iocb->ki_pos, iomi->len);
return -EIOCBQUEUED;
out_bio_release_pages:
bio_release_pages(bio, false);
out_bio_put:
bio_put(bio);
out_dio_end:
inode_dio_end(iomi->inode);
return ret;
}
EXPORT_SYMBOL_GPL(__iomap_dio_read_simple);
static int __init iomap_dio_init(void)
{
return bioset_init(&iomap_dio_simple_pool, 4,
offsetof(struct iomap_dio_simple, bio),
BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE);
}
fs_initcall(iomap_dio_init);

130
fs/iomap/iter.c
View File

@@ -6,12 +6,19 @@
#include <linux/iomap.h>
#include "trace.h"
static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter)
/*
* Release the iter folio batch. Note that the iomap flag is meant to control
* the I/O path for the mapping and may not be set in error situations.
*/
static inline void iomap_iter_clean_fbatch(const struct iomap_iter *iter,
struct iomap *iomap)
{
if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) {
if (!iter->fbatch)
return;
iomap->flags &= ~IOMAP_F_FOLIO_BATCH;
if (folio_batch_count(iter->fbatch)) {
folio_batch_release(iter->fbatch);
folio_batch_reinit(iter->fbatch);
iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH;
}
}
@@ -40,9 +47,60 @@ static inline void iomap_iter_done(struct iomap_iter *iter)
}
/**
* iomap_iter - iterate over a ranges in a file
* @iter: iteration structue
* @ops: iomap ops provided by the file system
* iomap_iter_continue - decide whether iteration should continue
* @iter: iteration structure
* @iomap: the mapping that was just processed
* @srcmap: the source mapping that was just processed
*
* Helper normally called via iomap_iter_next(). Called after the previous
* mapping has been finished to determine whether there is more of the file
* range left to process.
*
* Returns 1 if there is more work to do, in which case @iomap and @srcmap are
* cleared so the caller can produce the next mapping; zero if the range is
* fully consumed; or a negative errno on error.
*/
int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap,
struct iomap *srcmap, int ret)
{
const bool stale = iomap->flags & IOMAP_F_STALE;
const ssize_t advanced = iter->pos - iter->iter_start_pos;
if (ret < 0 && !advanced)
return ret;
/*
* Use iter->len to determine whether to continue onto the next mapping.
* Explicitly terminate on error status or if the current iter has not
* advanced at all (i.e. no work was done for some reason) unless the
* mapping has been marked stale and needs to be reprocessed.
*/
if (WARN_ON_ONCE(iter->status > 0))
/* detect old return semantics where this would advance */
ret = -EIO;
else if (iter->status < 0)
ret = iter->status;
else if (iter->len == 0 || (!advanced && !stale))
ret = 0;
else
ret = 1;
iomap_iter_clean_fbatch(iter, iomap);
if (ret <= 0)
return ret;
memset(iomap, 0, sizeof(*iomap));
memset(srcmap, 0, sizeof(*srcmap));
return ret;
}
EXPORT_SYMBOL_GPL(iomap_iter_continue);
/**
* iomap_iter - iterate over ranges in a file
* @iter: iteration structure
* @ops: iomap ops provided by the filesystem
*
* Iterate over filesystem-provided space mappings for the provided file range.
*
@@ -56,61 +114,21 @@ static inline void iomap_iter_done(struct iomap_iter *iter)
*/
int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops)
{
bool stale = iter->iomap.flags & IOMAP_F_STALE;
ssize_t advanced;
u64 olen;
int ret;
trace_iomap_iter(iter, ops, _RET_IP_);
if (!iter->iomap.length)
goto begin;
/*
* Calculate how far the iter was advanced and the original length bytes
* for ->iomap_end().
*/
advanced = iter->pos - iter->iter_start_pos;
olen = iter->len + advanced;
if (ops->iomap_end) {
ret = ops->iomap_end(iter->inode, iter->iter_start_pos,
iomap_length_trim(iter, iter->iter_start_pos,
olen),
advanced, iter->flags, &iter->iomap);
if (ret < 0 && !advanced)
return ret;
}
/* detect old return semantics where this would advance */
if (WARN_ON_ONCE(iter->status > 0))
iter->status = -EIO;
/*
* Use iter->len to determine whether to continue onto the next mapping.
* Explicitly terminate on error status or if the current iter has not
* advanced at all (i.e. no work was done for some reason) unless the
* mapping has been marked stale and needs to be reprocessed.
*/
if (iter->status < 0)
ret = iter->status;
else if (iter->len == 0 || (!advanced && !stale))
ret = 0;
if (ops->iomap_next)
ret = ops->iomap_next(iter, &iter->iomap, &iter->srcmap);
else
ret = 1;
iomap_iter_clean_fbatch(iter);
ret = iomap_iter_next(iter, &iter->iomap, &iter->srcmap,
ops->iomap_begin, ops->iomap_end);
iter->status = 0;
if (ret <= 0)
return ret;
if (ret > 0)
iomap_iter_done(iter);
else if (ret < 0)
iomap_iter_clean_fbatch(iter, &iter->iomap);
memset(&iter->iomap, 0, sizeof(iter->iomap));
memset(&iter->srcmap, 0, sizeof(iter->srcmap));
begin:
ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags,
&iter->iomap, &iter->srcmap);
if (ret < 0)
return ret;
iomap_iter_done(iter);
return 1;
return ret;
}

32
fs/ntfs/iomap.c
View File

@@ -277,8 +277,10 @@ static int ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t leng
srcmap, true);
}
static DEFINE_IOMAP_ITER_NEXT(ntfs_read_iomap_next, ntfs_read_iomap_begin);
const struct iomap_ops ntfs_read_iomap_ops = {
.iomap_begin = ntfs_read_iomap_begin,
.iomap_next = ntfs_read_iomap_next,
};
/*
@@ -329,13 +331,17 @@ static int ntfs_zero_read_iomap_end(struct inode *inode, loff_t pos, loff_t leng
return written;
}
static DEFINE_IOMAP_ITER_NEXT_END(ntfs_zero_read_iomap_next,
ntfs_seek_iomap_begin, ntfs_zero_read_iomap_end);
static const struct iomap_ops ntfs_zero_read_iomap_ops = {
.iomap_begin = ntfs_seek_iomap_begin,
.iomap_end = ntfs_zero_read_iomap_end,
.iomap_next = ntfs_zero_read_iomap_next,
};
static DEFINE_IOMAP_ITER_NEXT(ntfs_seek_iomap_next, ntfs_seek_iomap_begin);
const struct iomap_ops ntfs_seek_iomap_ops = {
.iomap_begin = ntfs_seek_iomap_begin,
.iomap_next = ntfs_seek_iomap_next,
};
int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length)
@@ -764,9 +770,11 @@ static int ntfs_write_iomap_end(struct inode *inode, loff_t pos, loff_t length,
return written;
}
static DEFINE_IOMAP_ITER_NEXT_END(ntfs_write_iomap_next,
ntfs_write_iomap_begin, ntfs_write_iomap_end);
const struct iomap_ops ntfs_write_iomap_ops = {
.iomap_begin = ntfs_write_iomap_begin,
.iomap_end = ntfs_write_iomap_end,
.iomap_next = ntfs_write_iomap_next,
};
static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset,
@@ -777,9 +785,11 @@ static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset,
NTFS_IOMAP_FLAGS_MKWRITE);
}
static DEFINE_IOMAP_ITER_NEXT_END(ntfs_page_mkwrite_iomap_next,
ntfs_page_mkwrite_iomap_begin, ntfs_write_iomap_end);
const struct iomap_ops ntfs_page_mkwrite_iomap_ops = {
.iomap_begin = ntfs_page_mkwrite_iomap_begin,
.iomap_end = ntfs_write_iomap_end,
.iomap_next = ntfs_page_mkwrite_iomap_next,
};
static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset,
@@ -790,9 +800,11 @@ static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset,
NTFS_IOMAP_FLAGS_DIO);
}
static DEFINE_IOMAP_ITER_NEXT_END(ntfs_dio_iomap_next,
ntfs_dio_iomap_begin, ntfs_write_iomap_end);
const struct iomap_ops ntfs_dio_iomap_ops = {
.iomap_begin = ntfs_dio_iomap_begin,
.iomap_end = ntfs_write_iomap_end,
.iomap_next = ntfs_dio_iomap_next,
};
static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc,

6
fs/ntfs3/inode.c
View File

@@ -2098,9 +2098,11 @@ const struct address_space_operations ntfs_aops_cmpr = {
.invalidate_folio = iomap_invalidate_folio,
};
static DEFINE_IOMAP_ITER_NEXT_END(ntfs_iomap_next, ntfs_iomap_begin,
ntfs_iomap_end);
const struct iomap_ops ntfs_iomap_ops = {
.iomap_begin = ntfs_iomap_begin,
.iomap_end = ntfs_iomap_end,
.iomap_next = ntfs_iomap_next,
};
const struct iomap_write_ops ntfs_iomap_folio_ops = {

18
fs/xfs/xfs_file.c
View File

@@ -251,8 +251,6 @@ xfs_file_dio_read(
struct iov_iter *to)
{
struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp));
unsigned int dio_flags = 0;
const struct iomap_dio_ops *dio_ops = NULL;
ssize_t ret;
trace_xfs_file_direct_read(iocb, to);
@@ -266,11 +264,15 @@ xfs_file_dio_read(
if (ret)
return ret;
if (mapping_stable_writes(iocb->ki_filp->f_mapping)) {
dio_ops = &xfs_dio_read_bounce_ops;
dio_flags |= IOMAP_DIO_BOUNCE;
ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops,
&xfs_dio_read_bounce_ops, IOMAP_DIO_BOUNCE,
NULL, 0);
} else {
ret = iomap_dio_read_simple(iocb, to, xfs_read_iomap_begin);
if (ret == -ENOTBLK)
ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL,
0, NULL, 0);
}
ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, dio_ops, dio_flags,
NULL, 0);
xfs_iunlock(ip, XFS_IOLOCK_SHARED);
return ret;
@@ -857,9 +859,9 @@ xfs_file_dio_write_atomic(
NULL, 0);
/*
* The retry mechanism is based on the ->iomap_begin method returning
* The retry mechanism is based on the ->iomap_next method returning
* -ENOPROTOOPT, which would be when the REQ_ATOMIC-based write is not
* possible. The REQ_ATOMIC-based method typically not be possible if
* possible. The REQ_ATOMIC-based method is typically not possible if
* the write spans multiple extents or the disk blocks are misaligned.
*/
if (ret == -ENOPROTOOPT && dops == &xfs_direct_write_iomap_ops) {

41
fs/xfs/xfs_iomap.c
View File

@@ -1037,8 +1037,11 @@ xfs_direct_write_iomap_begin(
return error;
}
static DEFINE_IOMAP_ITER_NEXT(xfs_direct_write_iomap_next,
xfs_direct_write_iomap_begin);
const struct iomap_ops xfs_direct_write_iomap_ops = {
.iomap_begin = xfs_direct_write_iomap_begin,
.iomap_next = xfs_direct_write_iomap_next,
};
#ifdef CONFIG_XFS_RT
@@ -1089,8 +1092,11 @@ xfs_zoned_direct_write_iomap_begin(
return 0;
}
static DEFINE_IOMAP_ITER_NEXT(xfs_zoned_direct_write_iomap_next,
xfs_zoned_direct_write_iomap_begin);
const struct iomap_ops xfs_zoned_direct_write_iomap_ops = {
.iomap_begin = xfs_zoned_direct_write_iomap_begin,
.iomap_next = xfs_zoned_direct_write_iomap_next,
};
#endif /* CONFIG_XFS_RT */
@@ -1274,8 +1280,11 @@ xfs_atomic_write_cow_iomap_begin(
return error;
}
static DEFINE_IOMAP_ITER_NEXT(xfs_atomic_write_cow_iomap_next,
xfs_atomic_write_cow_iomap_begin);
const struct iomap_ops xfs_atomic_write_cow_iomap_ops = {
.iomap_begin = xfs_atomic_write_cow_iomap_begin,
.iomap_next = xfs_atomic_write_cow_iomap_next,
};
static int
@@ -1298,9 +1307,11 @@ xfs_dax_write_iomap_end(
return xfs_reflink_end_cow(ip, pos, written);
}
static DEFINE_IOMAP_ITER_NEXT_END(xfs_dax_write_iomap_next,
xfs_direct_write_iomap_begin, xfs_dax_write_iomap_end);
const struct iomap_ops xfs_dax_write_iomap_ops = {
.iomap_begin = xfs_direct_write_iomap_begin,
.iomap_end = xfs_dax_write_iomap_end,
.iomap_next = xfs_dax_write_iomap_next,
};
/*
@@ -2168,12 +2179,14 @@ xfs_buffered_write_iomap_end(
return 0;
}
static DEFINE_IOMAP_ITER_NEXT_END(xfs_buffered_write_iomap_next,
xfs_buffered_write_iomap_begin, xfs_buffered_write_iomap_end);
const struct iomap_ops xfs_buffered_write_iomap_ops = {
.iomap_begin = xfs_buffered_write_iomap_begin,
.iomap_end = xfs_buffered_write_iomap_end,
.iomap_next = xfs_buffered_write_iomap_next,
};
static int
int
xfs_read_iomap_begin(
struct inode *inode,
loff_t offset,
@@ -2214,8 +2227,10 @@ xfs_read_iomap_begin(
shared ? IOMAP_F_SHARED : 0, seq);
}
static DEFINE_IOMAP_ITER_NEXT(xfs_read_iomap_next, xfs_read_iomap_begin);
const struct iomap_ops xfs_read_iomap_ops = {
.iomap_begin = xfs_read_iomap_begin,
.iomap_next = xfs_read_iomap_next,
};
static int
@@ -2302,8 +2317,10 @@ xfs_seek_iomap_begin(
return error;
}
static DEFINE_IOMAP_ITER_NEXT(xfs_seek_iomap_next, xfs_seek_iomap_begin);
const struct iomap_ops xfs_seek_iomap_ops = {
.iomap_begin = xfs_seek_iomap_begin,
.iomap_next = xfs_seek_iomap_next,
};
static int
@@ -2349,8 +2366,10 @@ xfs_xattr_iomap_begin(
return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_XATTR, seq);
}
static DEFINE_IOMAP_ITER_NEXT(xfs_xattr_iomap_next, xfs_xattr_iomap_begin);
const struct iomap_ops xfs_xattr_iomap_ops = {
.iomap_begin = xfs_xattr_iomap_begin,
.iomap_next = xfs_xattr_iomap_next,
};
int

4
fs/xfs/xfs_iomap.h
View File

@@ -49,6 +49,10 @@ xfs_aligned_fsb_count(
return count_fsb;
}
int xfs_read_iomap_begin(struct inode *inode, loff_t offset,
loff_t length, unsigned flags, struct iomap *iomap,
struct iomap *srcmap);
extern const struct iomap_ops xfs_buffered_write_iomap_ops;
extern const struct iomap_ops xfs_direct_write_iomap_ops;
extern const struct iomap_ops xfs_zoned_direct_write_iomap_ops;

9
fs/zonefs/file.c
View File

@@ -57,8 +57,10 @@ static int zonefs_read_iomap_begin(struct inode *inode, loff_t offset,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT(zonefs_read_iomap_next, zonefs_read_iomap_begin);
static const struct iomap_ops zonefs_read_iomap_ops = {
.iomap_begin = zonefs_read_iomap_begin,
.iomap_next = zonefs_read_iomap_next,
};
static int zonefs_write_iomap_begin(struct inode *inode, loff_t offset,
@@ -106,8 +108,11 @@ static int zonefs_write_iomap_begin(struct inode *inode, loff_t offset,
return 0;
}
static DEFINE_IOMAP_ITER_NEXT(zonefs_write_iomap_next,
zonefs_write_iomap_begin);
static const struct iomap_ops zonefs_write_iomap_ops = {
.iomap_begin = zonefs_write_iomap_begin,
.iomap_next = zonefs_write_iomap_next,
};
static int zonefs_read_folio(struct file *unused, struct folio *folio)

178
include/linux/iomap.h
View File

@@ -10,6 +10,7 @@
#include <linux/mm_types.h>
#include <linux/blkdev.h>
#include <linux/folio_batch.h>
#include <linux/pagemap.h>
struct address_space;
struct fiemap_extent_info;
@@ -212,24 +213,36 @@ struct iomap_write_ops {
#define IOMAP_ATOMIC (1 << 9) /* torn-write protection */
#define IOMAP_DONTCACHE (1 << 10)
struct iomap_ops {
/*
* Return the existing mapping at pos, or reserve space starting at
* pos for up to length, as long as we can do it as a single mapping.
* The actual length is returned in iomap->length.
*/
int (*iomap_begin)(struct inode *inode, loff_t pos, loff_t length,
unsigned flags, struct iomap *iomap,
struct iomap *srcmap);
/*
* Return the existing mapping at pos, or reserve space starting at pos for up
* to length, as long as we can do it as a single mapping.
* The actual length is returned in iomap->length.
*/
typedef int (*iomap_iter_begin_fn)(struct inode *inode, loff_t pos,
loff_t length, unsigned flags, struct iomap *iomap,
struct iomap *srcmap);
/*
* Commit and/or unreserve space previous allocated using iomap_begin.
* Written indicates the length of the successful write operation which
* needs to be commited, while the rest needs to be unreserved.
* Written might be zero if no data was written.
*/
int (*iomap_end)(struct inode *inode, loff_t pos, loff_t length,
ssize_t written, unsigned flags, struct iomap *iomap);
/*
* Commit and/or unreserve space previously allocated by iomap_iter_begin_fn.
* Written indicates the length of the successful write operation which needs
* to be committed, while the rest needs to be unreserved.
* Written might be zero if no data was written.
*/
typedef int (*iomap_iter_end_fn)(struct inode *inode, loff_t pos, loff_t length,
ssize_t written, unsigned flags, struct iomap *iomap);
/*
* Produce the next mapping (finishing the previous one if needed).
* Return 1 to continue iterating, 0 if the range is fully consumed, or a
* negative error on failure.
*/
typedef int (*iomap_iter_next_fn)(const struct iomap_iter *iter,
struct iomap *iomap, struct iomap *srcmap);
struct iomap_ops {
iomap_iter_begin_fn iomap_begin;
iomap_iter_end_fn iomap_end;
iomap_iter_next_fn iomap_next;
};
/**
@@ -317,6 +330,71 @@ static inline const struct iomap *iomap_iter_srcmap(const struct iomap_iter *i)
return &i->iomap;
}
int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap,
struct iomap *srcmap, int ret);
/**
* iomap_iter_next - finish the previous mapping and produce the next one
* @iter: iteration structure
* @iomap: mapping to finish and then repopulate
* @srcmap: source mapping to finish and then repopulate
* @begin: callback that produces a mapping for the current position
* @end: optional callback that finishes the previous mapping, or NULL
*
* Inline helper that implements the common body of an ->iomap_next()
* callback: it finishes the previous mapping via @end (if present), decides
* via iomap_iter_continue() whether to keep going, and obtains the next
* mapping via @begin.
*
* This helper is marked __always_inline so that when a caller passes
* compile-time-constant @begin and @end callbacks, the compiler can call them
* directly, avoiding the indirect-call overhead.
*
* Returns 1 to continue iterating, 0 once the range is fully consumed, or a
* negative errno on error.
*/
static __always_inline int iomap_iter_next(const struct iomap_iter *iter,
struct iomap *iomap, struct iomap *srcmap,
iomap_iter_begin_fn begin, iomap_iter_end_fn end)
{
int ret = 0;
if (iomap->length) {
if (end) {
/*
* Calculate how far the iter was advanced and the
* original length bytes for end().
*/
ssize_t advanced = iter->pos - iter->iter_start_pos;
loff_t len;
len = iomap_length_trim(iter, iter->iter_start_pos,
iter->len + advanced);
ret = end(iter->inode, iter->iter_start_pos, len,
advanced, iter->flags, iomap);
}
ret = iomap_iter_continue(iter, iomap, srcmap, ret);
if (ret <= 0)
return ret;
}
ret = begin(iter->inode, iter->pos, iter->len, iter->flags, iomap,
srcmap);
return ret < 0 ? ret : 1;
}
#define DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, end_fn) \
int name(const struct iomap_iter *iter, struct iomap *iomap, \
struct iomap *srcmap) \
{ \
return iomap_iter_next(iter, iomap, srcmap, begin_fn, end_fn); \
}
#define DEFINE_IOMAP_ITER_NEXT(name, begin_fn) \
DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, NULL)
/*
* Return the file offset for the first unchanged block after a short write.
*
@@ -365,6 +443,7 @@ struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len);
bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags);
void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len);
bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio);
void iomap_folio_mark_uptodate(struct folio *folio);
int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len,
const struct iomap_ops *ops,
const struct iomap_write_ops *write_ops);
@@ -606,6 +685,71 @@ struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
ssize_t iomap_dio_complete(struct iomap_dio *dio);
void iomap_dio_bio_end_io(struct bio *bio);
/*
* Fast path for small, block-aligned direct I/Os that map to a single
* contiguous on-disk extent.
*
* @iter must describe a non-empty READ no larger than the inode block size:
* writes, zero-length I/O, and larger requests need the generic iomap direct
* I/O path.
*
* Does not support iomap_dio_ops, dio_flags, done_before or private data.
* The range must also stay within i_size and encrypted inodes must use the
* generic iomap direct I/O path.
*
* -ENOTBLK indicates the generic path must be used by the caller instead.
* Any other errno is a real result and is propagated as-is, in particular
* -EAGAIN for IOCB_NOWAIT must reach the caller.
*
* The caller can only provide an iomap begin handler, and the iterator
* is never advanced.
*/
ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter,
struct iomap_iter *iomi);
static __always_inline ssize_t iomap_dio_read_simple(struct kiocb *iocb,
struct iov_iter *iter, iomap_iter_begin_fn begin)
{
struct iomap_iter iomi = {
.inode = file_inode(iocb->ki_filp),
.pos = iocb->ki_pos,
.len = iov_iter_count(iter),
.flags = IOMAP_DIRECT,
};
ssize_t ret;
if (!iomi.len)
return 0;
/*
* Simple dio is an optimization for small IO. Filter out large IO
* early as it's the most common case to fail for typical direct IO
* workloads.
*/
if (iomi.len > iomi.inode->i_sb->s_blocksize)
return -ENOTBLK;
if (iocb->ki_pos + iomi.len > i_size_read(iomi.inode))
return -ENOTBLK;
if (IS_ENCRYPTED(iomi.inode))
return -ENOTBLK;
ret = kiocb_write_and_wait(iocb, iomi.len);
if (ret)
return ret;
if (iocb->ki_flags & IOCB_NOWAIT)
iomi.flags |= IOMAP_NOWAIT;
inode_dio_begin(iomi.inode);
ret = begin(iomi.inode, iomi.pos, iomi.len, iomi.flags, &iomi.iomap,
&iomi.srcmap);
if (ret) {
inode_dio_end(iomi.inode);
return ret;
}
return __iomap_dio_read_simple(iocb, iter, &iomi);
}
#ifdef CONFIG_SWAP
struct file;
struct swap_info_struct;