From 6b1a162ba1a2dfd1ceac24029e4d0c2443173ed2 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:14 +0100 Subject: [PATCH 01/34] buffer: Remove forward declaration of submit_bh_wbc() Rearrange functions to avoid this forward declaration. Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-2-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 150 +++++++++++++++++++++++++--------------------------- 1 file changed, 73 insertions(+), 77 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index b0b3792b1496..d4e9a10d2a26 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -54,9 +54,6 @@ #include "internal.h" -static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, - enum rw_hint hint, struct writeback_control *wbc); - #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) inline void touch_buffer(struct buffer_head *bh) @@ -1157,6 +1154,79 @@ void __bforget(struct buffer_head *bh) } EXPORT_SYMBOL(__bforget); +static void end_bio_bh_io_sync(struct bio *bio) +{ + struct buffer_head *bh = bio->bi_private; + + if (unlikely(bio_flagged(bio, BIO_QUIET))) + set_bit(BH_Quiet, &bh->b_state); + + bh->b_end_io(bh, !bio->bi_status); + bio_put(bio); +} + +static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh, + gfp_t gfp_mask) +{ + const struct address_space *mapping = folio_mapping(bh->b_folio); + + /* + * The ext4 journal (jbd2) can submit a buffer_head it directly created + * for a non-pagecache page. fscrypt doesn't care about these. + */ + if (!mapping) + return; + fscrypt_set_bio_crypt_ctx(bio, mapping->host, + folio_pos(bh->b_folio) + bh_offset(bh), gfp_mask); +} + +static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, + enum rw_hint write_hint, struct writeback_control *wbc) +{ + const enum req_op op = opf & REQ_OP_MASK; + struct bio *bio; + + BUG_ON(!buffer_locked(bh)); + BUG_ON(!buffer_mapped(bh)); + BUG_ON(!bh->b_end_io); + BUG_ON(buffer_delay(bh)); + BUG_ON(buffer_unwritten(bh)); + + /* + * Only clear out a write error when rewriting + */ + if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE)) + clear_buffer_write_io_error(bh); + + if (buffer_meta(bh)) + opf |= REQ_META; + if (buffer_prio(bh)) + opf |= REQ_PRIO; + + bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); + + if (IS_ENABLED(CONFIG_FS_ENCRYPTION)) + buffer_set_crypto_ctx(bio, bh, GFP_NOIO); + + bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); + bio->bi_write_hint = write_hint; + + bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh)); + + bio->bi_end_io = end_bio_bh_io_sync; + bio->bi_private = bh; + + /* Take care of bh's that straddle the end of the device */ + guard_bio_eod(bio); + + if (wbc) { + wbc_init_bio(wbc, bio); + wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size); + } + + blk_crypto_submit_bio(bio); +} + static struct buffer_head *__bread_slow(struct buffer_head *bh) { lock_buffer(bh); @@ -2663,80 +2733,6 @@ sector_t generic_block_bmap(struct address_space *mapping, sector_t block, } EXPORT_SYMBOL(generic_block_bmap); -static void end_bio_bh_io_sync(struct bio *bio) -{ - struct buffer_head *bh = bio->bi_private; - - if (unlikely(bio_flagged(bio, BIO_QUIET))) - set_bit(BH_Quiet, &bh->b_state); - - bh->b_end_io(bh, !bio->bi_status); - bio_put(bio); -} - -static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh, - gfp_t gfp_mask) -{ - const struct address_space *mapping = folio_mapping(bh->b_folio); - - /* - * The ext4 journal (jbd2) can submit a buffer_head it directly created - * for a non-pagecache page. fscrypt doesn't care about these. - */ - if (!mapping) - return; - fscrypt_set_bio_crypt_ctx(bio, mapping->host, - folio_pos(bh->b_folio) + bh_offset(bh), gfp_mask); -} - -static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, - enum rw_hint write_hint, - struct writeback_control *wbc) -{ - const enum req_op op = opf & REQ_OP_MASK; - struct bio *bio; - - BUG_ON(!buffer_locked(bh)); - BUG_ON(!buffer_mapped(bh)); - BUG_ON(!bh->b_end_io); - BUG_ON(buffer_delay(bh)); - BUG_ON(buffer_unwritten(bh)); - - /* - * Only clear out a write error when rewriting - */ - if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE)) - clear_buffer_write_io_error(bh); - - if (buffer_meta(bh)) - opf |= REQ_META; - if (buffer_prio(bh)) - opf |= REQ_PRIO; - - bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); - - if (IS_ENABLED(CONFIG_FS_ENCRYPTION)) - buffer_set_crypto_ctx(bio, bh, GFP_NOIO); - - bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); - bio->bi_write_hint = write_hint; - - bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh)); - - bio->bi_end_io = end_bio_bh_io_sync; - bio->bi_private = bh; - - /* Take care of bh's that straddle the end of the device */ - guard_bio_eod(bio); - - if (wbc) { - wbc_init_bio(wbc, bio); - wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size); - } - - blk_crypto_submit_bio(bio); -} - void submit_bh(blk_opf_t opf, struct buffer_head *bh) { submit_bh_wbc(opf, bh, WRITE_LIFE_NOT_SET, NULL); From 4b7381326424809577eb43bf635ea1d43a095d89 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:15 +0100 Subject: [PATCH 02/34] buffer: Add bh_submit() bh_submit() takes a bio_end_io allowing users to avoid the indirect function call through bh->b_end_io, and eventually allowing us to remove bh->b_end_io. Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-3-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 31 +++++++++++++++++++++++++++---- include/linux/buffer_head.h | 1 + 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index d4e9a10d2a26..ea926963529e 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1180,15 +1180,15 @@ static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh, folio_pos(bh->b_folio) + bh_offset(bh), gfp_mask); } -static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, - enum rw_hint write_hint, struct writeback_control *wbc) +static void __bh_submit(struct buffer_head *bh, blk_opf_t opf, + enum rw_hint write_hint, struct writeback_control *wbc, + bio_end_io_t end_bio) { const enum req_op op = opf & REQ_OP_MASK; struct bio *bio; BUG_ON(!buffer_locked(bh)); BUG_ON(!buffer_mapped(bh)); - BUG_ON(!bh->b_end_io); BUG_ON(buffer_delay(bh)); BUG_ON(buffer_unwritten(bh)); @@ -1213,7 +1213,7 @@ static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh)); - bio->bi_end_io = end_bio_bh_io_sync; + bio->bi_end_io = end_bio; bio->bi_private = bh; /* Take care of bh's that straddle the end of the device */ @@ -1227,6 +1227,29 @@ static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, blk_crypto_submit_bio(bio); } +static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, + enum rw_hint write_hint, + struct writeback_control *wbc) +{ + BUG_ON(!bh->b_end_io); + __bh_submit(bh, opf, write_hint, wbc, end_bio_bh_io_sync); +} + +/** + * bh_submit - Start I/O against a buffer head + * @bh: The buffer head to perform I/O on. + * @opf: Operation and flags for bio. + * @end_io: The routine to call when I/O has completed. + * + * If you need to do I/O on an individual bh (instead of allowing the + * page cache to do I/O on the folio that it is in), call this function. + */ +void bh_submit(struct buffer_head *bh, blk_opf_t opf, bio_end_io_t end_io) +{ + __bh_submit(bh, opf, WRITE_LIFE_NOT_SET, NULL, end_io); +} +EXPORT_SYMBOL(bh_submit); + static struct buffer_head *__bread_slow(struct buffer_head *bh) { lock_buffer(bh); diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index e4939e33b4b5..d59980e4adda 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -240,6 +240,7 @@ int sync_dirty_buffer(struct buffer_head *bh); int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); void submit_bh(blk_opf_t, struct buffer_head *); +void bh_submit(struct buffer_head *, blk_opf_t, bio_end_io_t); void write_boundary_block(struct block_device *bdev, sector_t bblock, unsigned blocksize); int bh_uptodate_or_lock(struct buffer_head *bh); From df21e33ff9c4ab20eb2a98273c2d9fed300aba26 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:16 +0100 Subject: [PATCH 03/34] buffer: Remove mark_buffer_async_write_endio() All callers of mark_buffer_async_write_endio() pass end_buffer_async_write, so we can inline mark_buffer_async_write_endio() into mark_buffer_async_write() and just call that instead. Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-4-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index ea926963529e..4dcce64ef006 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -443,16 +443,10 @@ static void mark_buffer_async_read(struct buffer_head *bh) set_buffer_async_read(bh); } -static void mark_buffer_async_write_endio(struct buffer_head *bh, - bh_end_io_t *handler) -{ - bh->b_end_io = handler; - set_buffer_async_write(bh); -} - void mark_buffer_async_write(struct buffer_head *bh) { - mark_buffer_async_write_endio(bh, end_buffer_async_write); + bh->b_end_io = end_buffer_async_write; + set_buffer_async_write(bh); } EXPORT_SYMBOL(mark_buffer_async_write); @@ -1903,8 +1897,7 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, continue; } if (test_clear_buffer_dirty(bh)) { - mark_buffer_async_write_endio(bh, - end_buffer_async_write); + mark_buffer_async_write(bh); } else { unlock_buffer(bh); } @@ -1958,8 +1951,7 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, if (buffer_mapped(bh) && buffer_dirty(bh) && !buffer_delay(bh)) { lock_buffer(bh); - mark_buffer_async_write_endio(bh, - end_buffer_async_write); + mark_buffer_async_write(bh); } else { /* * The buffer may have been set dirty during From 86ecf5704ed3c731284b9c60e46dcc5d115ebe7c Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:17 +0100 Subject: [PATCH 04/34] buffer: Add bh_end_read(), bh_end_write() and bh_end_async_write() These are the bio_end_io_t versions of end_buffer_read_sync(), end_buffer_write_sync() and end_buffer_async_write(). They do not contain a put_bh() call as it is no longer necessary. Also add the helper function bio_endio_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-5-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 89 ++++++++++++++++++++++++++++++++++--- include/linux/buffer_head.h | 6 +++ 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index 4dcce64ef006..e22a94afa385 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -129,6 +129,34 @@ static void buffer_io_error(struct buffer_head *bh, char *msg) bh->b_bdev, (unsigned long long)bh->b_blocknr, msg); } +/** + * bio_endio_bh - Discard the bio used to submit a buffer. + * @bio: The bio. + * @bhp: Where to return the buffer_head. + * + * Call this in your bio_end_io handler to retrieve the buffer_head + * submitted in bh_submit(). If you did not call bh_submit(), do not + * call this function; it will return garbage. + * + * This function consumes the bio refcount which will probably free the + * bio. + * + * Return: True if the I/O succeeded. + */ +bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp) +{ + bool success = bio->bi_status == BLK_STS_OK; + struct buffer_head *bh = bio->bi_private; + + if (unlikely(bio_flagged(bio, BIO_QUIET))) + set_bit(BH_Quiet, &bh->b_state); + bio_put(bio); + + *bhp = bh; + return success; +} +EXPORT_SYMBOL(bio_endio_bh); + /* * End-of-IO handler helper function which does not touch the bh after * unlocking it. @@ -159,7 +187,22 @@ void end_buffer_read_sync(struct buffer_head *bh, int uptodate) } EXPORT_SYMBOL(end_buffer_read_sync); -void end_buffer_write_sync(struct buffer_head *bh, int uptodate) +/** + * bh_end_read - I/O end handler for reads + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're reading into the buffer, + * unless you need your own special I/O end handler. + */ +void bh_end_read(struct bio *bio) +{ + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + __end_buffer_read_notouch(bh, uptodate); +} +EXPORT_SYMBOL(bh_end_read); + +static void __end_buffer_write_sync(struct buffer_head *bh, int uptodate) { if (uptodate) { set_buffer_uptodate(bh); @@ -169,10 +212,30 @@ void end_buffer_write_sync(struct buffer_head *bh, int uptodate) clear_buffer_uptodate(bh); } unlock_buffer(bh); +} + +void end_buffer_write_sync(struct buffer_head *bh, int uptodate) +{ + __end_buffer_write_sync(bh, uptodate); put_bh(bh); } EXPORT_SYMBOL(end_buffer_write_sync); +/** + * bh_end_write - I/O end handler for writes + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're writing from the buffer, + * unless you need your own special I/O end handler. + */ +void bh_end_write(struct bio *bio) +{ + struct buffer_head *bh; + bool success = bio_endio_bh(bio, &bh); + __end_buffer_write_sync(bh, success); +} +EXPORT_SYMBOL(bh_end_write); + static struct buffer_head * __find_get_block_slow(struct block_device *bdev, sector_t block, bool atomic) { @@ -416,6 +479,21 @@ static void end_buffer_async_write(struct buffer_head *bh, int uptodate) spin_unlock_irqrestore(&first->b_uptodate_lock, flags); } +/** + * bh_end_async_write - I/O end handler for async folio writes + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're doing the equivalent of + * block_write_full_folio(). + */ +void bh_end_async_write(struct bio *bio) +{ + struct buffer_head *bh; + bool success = bio_endio_bh(bio, &bh); + end_buffer_async_write(bh, success); +} +EXPORT_SYMBOL(bh_end_async_write); + /* * If a page's buffers are under async readin (end_buffer_async_read * completion) then there is a possibility that another thread of @@ -1150,13 +1228,10 @@ EXPORT_SYMBOL(__bforget); static void end_bio_bh_io_sync(struct bio *bio) { - struct buffer_head *bh = bio->bi_private; + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); - if (unlikely(bio_flagged(bio, BIO_QUIET))) - set_bit(BH_Quiet, &bh->b_state); - - bh->b_end_io(bh, !bio->bi_status); - bio_put(bio); + bh->b_end_io(bh, uptodate); } static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh, diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index d59980e4adda..b0a31a90fa79 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -204,6 +204,12 @@ struct buffer_head *create_empty_buffers(struct folio *folio, unsigned long blocksize, unsigned long b_state); void end_buffer_read_sync(struct buffer_head *bh, int uptodate); void end_buffer_write_sync(struct buffer_head *bh, int uptodate); +bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp); + +/* Completion routines suitable for passing to bh_submit() */ +void bh_end_read(struct bio *bio); +void bh_end_write(struct bio *bio); +void bh_end_async_write(struct bio *bio); /* Things to do with metadata buffers list */ void mmb_mark_buffer_dirty(struct buffer_head *bh, struct mapping_metadata_bhs *mmb); From b50e811b8fc9b1f28cfba1b74d5c1ad54e521107 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:18 +0100 Subject: [PATCH 05/34] buffer: Convert write_dirty_buffer to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-6-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index e22a94afa385..6329f2b554da 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2836,9 +2836,7 @@ void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) unlock_buffer(bh); return; } - bh->b_end_io = end_buffer_write_sync; - get_bh(bh); - submit_bh(REQ_OP_WRITE | op_flags, bh); + bh_submit(bh, REQ_OP_WRITE | op_flags, bh_end_write); } EXPORT_SYMBOL(write_dirty_buffer); From 8b7846fe5b27c11d883a05afe3481f39252a9e65 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:19 +0100 Subject: [PATCH 06/34] buffer: Convert __bread_slow to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-7-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index 6329f2b554da..f8428f802b68 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1326,9 +1326,7 @@ static struct buffer_head *__bread_slow(struct buffer_head *bh) unlock_buffer(bh); return bh; } else { - get_bh(bh); - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, bh_end_read); wait_on_buffer(bh); if (buffer_uptodate(bh)) return bh; From ac13eb47988fa871abcc3f34df946b5c039b3481 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:20 +0100 Subject: [PATCH 07/34] buffer: Convert __sync_dirty_buffer to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-8-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index f8428f802b68..deabf7479325 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2857,9 +2857,7 @@ int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) return -EIO; } - get_bh(bh); - bh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE | op_flags, bh); + bh_submit(bh, REQ_OP_WRITE | op_flags, bh_end_write); wait_on_buffer(bh); if (!buffer_uptodate(bh)) return -EIO; From a70db2d6f0bb9a3cf5b49a4bb30d34da955f051c Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:21 +0100 Subject: [PATCH 08/34] buffer: Convert __bh_read to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-9-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index deabf7479325..97c8b9966060 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -3089,9 +3089,7 @@ int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait) BUG_ON(!buffer_locked(bh)); - get_bh(bh); - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ | op_flags, bh); + bh_submit(bh, REQ_OP_READ | op_flags, bh_end_read); if (wait) { wait_on_buffer(bh); if (!buffer_uptodate(bh)) From 44d325ebba175e73711a4bd90f30e1bc192242d9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:22 +0100 Subject: [PATCH 09/34] buffer: Convert __bh_read_batch to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Jan Kara Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-10-willy@infradead.org Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index 97c8b9966060..d96320c18acb 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -3131,9 +3131,7 @@ void __bh_read_batch(int nr, struct buffer_head *bhs[], continue; } - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(REQ_OP_READ | op_flags, bh); + bh_submit(bh, REQ_OP_READ | op_flags, bh_end_read); } } EXPORT_SYMBOL(__bh_read_batch); From 5059fb945ca895af80506c7a90d46b7308d6f6c5 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:23 +0100 Subject: [PATCH 10/34] buffer: Convert block_read_full_folio to bh_submit() Avoid an extra indirect function call by using bh_submit() instead of submit_bh(). Since mark_buffer_async_read() would collapse to a single function call, inline it into block_read_full_folio() along with its extensive comment. Convert end_buffer_async_read_io() to bh_end_async_read(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-11-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 65 ++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index d96320c18acb..b668a73d48ec 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -402,11 +402,13 @@ static void decrypt_bh(struct work_struct *work) } /* - * I/O completion handler for block_read_full_folio() - pages + * I/O completion handler for block_read_full_folio() - folios * which come unlocked at the end of I/O. */ -static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate) +static void bh_end_async_read(struct bio *bio) { + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); struct inode *inode = bh->b_folio->mapping->host; bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode); struct fsverity_info *vi = NULL; @@ -431,7 +433,7 @@ static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate) } return; } - uptodate = 0; + uptodate = false; } end_buffer_async_read(bh, uptodate); } @@ -494,33 +496,6 @@ void bh_end_async_write(struct bio *bio) } EXPORT_SYMBOL(bh_end_async_write); -/* - * If a page's buffers are under async readin (end_buffer_async_read - * completion) then there is a possibility that another thread of - * control could lock one of the buffers after it has completed - * but while some of the other buffers have not completed. This - * locked buffer would confuse end_buffer_async_read() into not unlocking - * the page. So the absence of BH_Async_Read tells end_buffer_async_read() - * that this buffer is not under async I/O. - * - * The page comes unlocked when it has no locked buffer_async buffers - * left. - * - * PageLocked prevents anyone starting new async I/O reads any of - * the buffers. - * - * PageWriteback is used to prevent simultaneous writeout of the same - * page. - * - * PageLocked prevents anyone from starting writeback of a page which is - * under read I/O (PageWriteback is only ever set against a locked page). - */ -static void mark_buffer_async_read(struct buffer_head *bh) -{ - bh->b_end_io = end_buffer_async_read_io; - set_buffer_async_read(bh); -} - void mark_buffer_async_write(struct buffer_head *bh) { bh->b_end_io = end_buffer_async_write; @@ -2497,9 +2472,33 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block) continue; } - mark_buffer_async_read(bh); + /* + * If a folio's buffers are under async readin + * (end_buffer_async_read completion) then there is a + * possibility that another thread of control could lock + * one of the buffers after it has completed but while + * some of the other buffers have not completed. This + * locked buffer would confuse end_buffer_async_read() + * into not unlocking the folio. So the absence of + * BH_Async_Read tells end_buffer_async_read() that this + * buffer is not under async I/O. + * + * The folio comes unlocked when it has no locked + * buffer_async buffers left. + * + * The folio lock prevents anyone starting new async + * I/O reads into any of the buffers. + * + * The writeback flag is used to prevent simultaneous + * writeout of the same folio. + * + * The folio lock prevents anyone from starting writeback + * of a folio which is under read I/O (the writeback + * flag is only ever set on a locked folio). + */ + set_buffer_async_read(bh); if (prev) - submit_bh(REQ_OP_READ, prev); + bh_submit(prev, REQ_OP_READ, bh_end_async_read); prev = bh; } while (iblock++, (bh = bh->b_this_page) != head); @@ -2513,7 +2512,7 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block) * in this folio. */ if (prev) - submit_bh(REQ_OP_READ, prev); + bh_submit(prev, REQ_OP_READ, bh_end_async_read); else folio_end_read(folio, !page_error); From cc81507980f2d482ed5a0e9e3483306158f06665 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:24 +0100 Subject: [PATCH 11/34] buffer: Convert __block_write_full_folio to __bh_submit() Avoid an extra indirect function call by using __bh_submit() instead of submit_bh_wbc(). Since there is only one caller of submit_bh_wbc() left, inline it into submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-12-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index b668a73d48ec..50bcef50a4a4 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1271,14 +1271,6 @@ static void __bh_submit(struct buffer_head *bh, blk_opf_t opf, blk_crypto_submit_bio(bio); } -static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, - enum rw_hint write_hint, - struct writeback_control *wbc) -{ - BUG_ON(!bh->b_end_io); - __bh_submit(bh, opf, write_hint, wbc, end_bio_bh_io_sync); -} - /** * bh_submit - Start I/O against a buffer head * @bh: The buffer head to perform I/O on. @@ -1945,7 +1937,7 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, continue; } if (test_clear_buffer_dirty(bh)) { - mark_buffer_async_write(bh); + set_buffer_async_write(bh); } else { unlock_buffer(bh); } @@ -1961,8 +1953,9 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, do { struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { - submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, - inode->i_write_hint, wbc); + __bh_submit(bh, REQ_OP_WRITE | write_flags, + inode->i_write_hint, wbc, + bh_end_async_write); nr_underway++; } bh = next; @@ -1999,7 +1992,7 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, if (buffer_mapped(bh) && buffer_dirty(bh) && !buffer_delay(bh)) { lock_buffer(bh); - mark_buffer_async_write(bh); + set_buffer_async_write(bh); } else { /* * The buffer may have been set dirty during @@ -2015,8 +2008,9 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { clear_buffer_dirty(bh); - submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, - inode->i_write_hint, wbc); + __bh_submit(bh, REQ_OP_WRITE | write_flags, + inode->i_write_hint, wbc, + bh_end_async_write); nr_underway++; } bh = next; @@ -2822,7 +2816,8 @@ EXPORT_SYMBOL(generic_block_bmap); void submit_bh(blk_opf_t opf, struct buffer_head *bh) { - submit_bh_wbc(opf, bh, WRITE_LIFE_NOT_SET, NULL); + BUG_ON(!bh->b_end_io); + __bh_submit(bh, opf, WRITE_LIFE_NOT_SET, NULL, end_bio_bh_io_sync); } EXPORT_SYMBOL(submit_bh); From 576c783cc0d6395feaa56d4a89f2b7b4933a5a3d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:25 +0100 Subject: [PATCH 12/34] ext4; Convert __ext4_read_bh() to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by converting ext4_end_bitmap_read() from bh_end_io_t to bio_end_io_t and calling bh_submit(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-13-willy@infradead.org Reviewed-by: Jan Kara Cc: linux-ext4@vger.kernel.org Signed-off-by: Christian Brauner (Amutable) --- fs/ext4/ext4.h | 10 +++++----- fs/ext4/ialloc.c | 6 ++++-- fs/ext4/super.c | 12 ++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 94283a991e5c..6af11f0ff1c5 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -2959,7 +2959,7 @@ extern unsigned long ext4_count_dirs(struct super_block *); extern void ext4_mark_bitmap_end(int start_bit, int end_bit, char *bitmap); extern int ext4_init_inode_table(struct super_block *sb, ext4_group_t group, int barrier); -extern void ext4_end_bitmap_read(struct buffer_head *bh, int uptodate); +void ext4_end_bitmap_read(struct bio *bio); /* fast_commit.c */ int ext4_fc_info_show(struct seq_file *seq, void *v); @@ -3184,10 +3184,10 @@ extern struct buffer_head *ext4_sb_bread_unmovable(struct super_block *sb, sector_t block); extern struct buffer_head *ext4_sb_bread_nofail(struct super_block *sb, sector_t block); -extern void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail); -extern int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail); +void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail); +int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail); extern int ext4_read_bh_lock(struct buffer_head *bh, blk_opf_t op_flags, bool wait); extern void ext4_sb_breadahead_unmovable(struct super_block *sb, sector_t block); extern int ext4_seq_options_show(struct seq_file *seq, void *offset); diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index 3fd8f0099852..9aacd629a683 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -66,14 +66,16 @@ void ext4_mark_bitmap_end(int start_bit, int end_bit, char *bitmap) memset(bitmap + (i >> 3), 0xff, (end_bit - i) >> 3); } -void ext4_end_bitmap_read(struct buffer_head *bh, int uptodate) +void ext4_end_bitmap_read(struct bio *bio) { + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + if (uptodate) { set_buffer_uptodate(bh); set_bitmap_uptodate(bh); } unlock_buffer(bh); - put_bh(bh); } static int ext4_validate_inode_bitmap(struct super_block *sb, diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 6a77db4d3124..bc7faedcb8e4 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -161,7 +161,7 @@ MODULE_ALIAS("ext3"); static inline void __ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail) + bio_end_io_t end_io, bool simu_fail) { if (simu_fail) { clear_buffer_uptodate(bh); @@ -176,13 +176,13 @@ static inline void __ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, */ clear_buffer_verified(bh); - bh->b_end_io = end_io ? end_io : end_buffer_read_sync; - get_bh(bh); - submit_bh(REQ_OP_READ | op_flags, bh); + if (!end_io) + end_io = bh_end_read; + bh_submit(bh, REQ_OP_READ | op_flags, end_io); } void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail) + bio_end_io_t end_io, bool simu_fail) { BUG_ON(!buffer_locked(bh)); @@ -194,7 +194,7 @@ void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, } int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, - bh_end_io_t *end_io, bool simu_fail) + bio_end_io_t end_io, bool simu_fail) { BUG_ON(!buffer_locked(bh)); From cf3c71eed0b7c089f96905fdfe3cec994737f24f Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:26 +0100 Subject: [PATCH 13/34] ext4: Convert ext4_fc_submit_bh() to bh_submit() Avoid an extra indirect function call by converting ext4_end_buffer_io_sync() from bh_end_io_t to bio_end_io_t and calling bh_submit(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-14-willy@infradead.org Reviewed-by: Jan Kara Cc: linux-ext4@vger.kernel.org Signed-off-by: Christian Brauner (Amutable) --- fs/ext4/fast_commit.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index b3c22636251d..5773b85e43cb 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -184,8 +184,11 @@ #include static struct kmem_cache *ext4_fc_dentry_cachep; -static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate) +static void ext4_end_buffer_io_sync(struct bio *bio) { + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + BUFFER_TRACE(bh, ""); if (uptodate) { ext4_debug("%s: Block %lld up-to-date", @@ -659,8 +662,7 @@ static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail) lock_buffer(bh); set_buffer_dirty(bh); set_buffer_uptodate(bh); - bh->b_end_io = ext4_end_buffer_io_sync; - submit_bh(REQ_OP_WRITE | write_flags, bh); + bh_submit(bh, REQ_OP_WRITE | write_flags, ext4_end_buffer_io_sync); EXT4_SB(sb)->s_fc_bh = NULL; } From 786a9941c727d5cebdc40557bb28117c9d0c21ad Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:27 +0100 Subject: [PATCH 14/34] ext4: Convert write_mmp_block_thawed() to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-15-willy@infradead.org Reviewed-by: Jan Kara Cc: linux-ext4@vger.kernel.org Signed-off-by: Christian Brauner (Amutable) --- fs/ext4/mmp.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/ext4/mmp.c b/fs/ext4/mmp.c index 6f57c181ff77..7ce361484b38 100644 --- a/fs/ext4/mmp.c +++ b/fs/ext4/mmp.c @@ -46,9 +46,8 @@ static int write_mmp_block_thawed(struct super_block *sb, ext4_mmp_csum_set(sb, mmp); lock_buffer(bh); - bh->b_end_io = end_buffer_write_sync; - get_bh(bh); - submit_bh(REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO, bh); + bh_submit(bh, REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO, + bh_end_write); wait_on_buffer(bh); if (unlikely(!buffer_uptodate(bh))) return -EIO; From aa17449d60c7f088d4ffb77044d88c4692fc76bb Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:28 +0100 Subject: [PATCH 15/34] ext4: Convert ext4_commit_super() to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-16-willy@infradead.org Acked-by: Theodore Ts'o Reviewed-by: Jan Kara Cc: linux-ext4@vger.kernel.org Signed-off-by: Christian Brauner (Amutable) --- fs/ext4/super.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fs/ext4/super.c b/fs/ext4/super.c index bc7faedcb8e4..7283108d7609 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -6316,12 +6316,10 @@ static int ext4_commit_super(struct super_block *sb) clear_buffer_write_io_error(sbh); set_buffer_uptodate(sbh); } - get_bh(sbh); /* Clear potential dirty bit if it was journalled update */ clear_buffer_dirty(sbh); - sbh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE | REQ_SYNC | - (test_opt(sb, BARRIER) ? REQ_FUA : 0), sbh); + bh_submit(sbh, REQ_OP_WRITE | REQ_SYNC | + (test_opt(sb, BARRIER) ? REQ_FUA : 0), bh_end_write); wait_on_buffer(sbh); if (buffer_write_io_error(sbh)) { ext4_msg(sb, KERN_ERR, "I/O error while writing " From e4b09ba06e6f884dbbfc6f43ecc079ef6864c4e2 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:29 +0100 Subject: [PATCH 16/34] jbd2: Convert journal commit to bh_submit() Avoid an extra indirect function call by using bh_submit() instead of submit_bh() in journal_submit_commit_record() and jbd2_journal_commit_transaction(). These both use journal_end_buffer_io_sync(), so it's more straightforward to do them both at once. Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-17-willy@infradead.org Acked-by: Theodore Ts'o Reviewed-by: Jan Kara Cc: linux-ext4@vger.kernel.org Signed-off-by: Christian Brauner (Amutable) --- fs/jbd2/commit.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c index 8cf61e7185c4..4e91593d27e5 100644 --- a/fs/jbd2/commit.c +++ b/fs/jbd2/commit.c @@ -29,8 +29,10 @@ /* * IO end handler for temporary buffer_heads handling writes to the journal. */ -static void journal_end_buffer_io_sync(struct buffer_head *bh, int uptodate) +static void journal_end_buffer_io_sync(struct bio *bio) { + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); struct buffer_head *orig_bh = bh->b_private; BUFFER_TRACE(bh, ""); @@ -147,13 +149,12 @@ static int journal_submit_commit_record(journal_t *journal, lock_buffer(bh); clear_buffer_dirty(bh); set_buffer_uptodate(bh); - bh->b_end_io = journal_end_buffer_io_sync; if (journal->j_flags & JBD2_BARRIER && !jbd2_has_feature_async_commit(journal)) write_flags |= REQ_PREFLUSH | REQ_FUA; - submit_bh(write_flags, bh); + bh_submit(bh, write_flags, journal_end_buffer_io_sync); *cbh = bh; return 0; } @@ -751,9 +752,9 @@ void jbd2_journal_commit_transaction(journal_t *journal) lock_buffer(bh); clear_buffer_dirty(bh); set_buffer_uptodate(bh); - bh->b_end_io = journal_end_buffer_io_sync; - submit_bh(REQ_OP_WRITE | JBD2_JOURNAL_REQ_FLAGS, - bh); + bh_submit(bh, + REQ_OP_WRITE | JBD2_JOURNAL_REQ_FLAGS, + journal_end_buffer_io_sync); } cond_resched(); From 8a30f324e2cd93605038ee20ff25d0dec58839b6 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:30 +0100 Subject: [PATCH 17/34] jbd2: Convert jbd2_write_superblock() to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-18-willy@infradead.org Acked-by: Theodore Ts'o Reviewed-by: Jan Kara Cc: linux-ext4@vger.kernel.org Signed-off-by: Christian Brauner (Amutable) --- fs/jbd2/journal.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index 4f397fcdb13c..2040af8c84cb 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -1820,9 +1820,7 @@ static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags) } if (jbd2_journal_has_csum_v2or3(journal)) sb->s_checksum = jbd2_superblock_csum(sb); - get_bh(bh); - bh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE | write_flags, bh); + bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write); wait_on_buffer(bh); if (buffer_write_io_error(bh)) { clear_buffer_write_io_error(bh); From 36ec1a90fe117ff2255a497eb3b73e6587b2ed70 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:31 +0100 Subject: [PATCH 18/34] ocfs2: Convert ocfs2_write_block to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-19-willy@infradead.org Reviewed-by: Joseph Qi Reviewed-by: Jan Kara Cc: ocfs2-devel@lists.linux.dev Signed-off-by: Christian Brauner (Amutable) --- fs/ocfs2/buffer_head_io.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c index 701d27d908d4..714daa4bbda2 100644 --- a/fs/ocfs2/buffer_head_io.c +++ b/fs/ocfs2/buffer_head_io.c @@ -62,9 +62,7 @@ int ocfs2_write_block(struct ocfs2_super *osb, struct buffer_head *bh, /* remove from dirty list before I/O. */ clear_buffer_dirty(bh); - get_bh(bh); /* for end_buffer_write_sync() */ - bh->b_end_io = end_buffer_write_sync; - submit_bh(REQ_OP_WRITE, bh); + bh_submit(bh, REQ_OP_WRITE, bh_end_write); wait_on_buffer(bh); From 4b08d56b09d75efb0788dbaea619cf0f5d509576 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:32 +0100 Subject: [PATCH 19/34] ocfs2: Convert ocfs2_read_block to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-20-willy@infradead.org Reviewed-by: Joseph Qi Reviewed-by: Jan Kara Cc: ocfs2-devel@lists.linux.dev Signed-off-by: Christian Brauner (Amutable) --- fs/ocfs2/buffer_head_io.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c index 714daa4bbda2..9edaeef5c4b9 100644 --- a/fs/ocfs2/buffer_head_io.c +++ b/fs/ocfs2/buffer_head_io.c @@ -143,9 +143,7 @@ int ocfs2_read_blocks_sync(struct ocfs2_super *osb, u64 block, #endif } - get_bh(bh); /* for end_buffer_read_sync() */ - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, bh_end_read); } read_failure: From 4416bb661b1cb40d78837f630755d2b8440cf4f3 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:33 +0100 Subject: [PATCH 20/34] ocfs2: Convert ocfs2_read_blocks to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-21-willy@infradead.org Reviewed-by: Joseph Qi Reviewed-by: Jan Kara Cc: ocfs2-devel@lists.linux.dev Signed-off-by: Christian Brauner (Amutable) --- fs/ocfs2/buffer_head_io.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c index 9edaeef5c4b9..185ff4dcba75 100644 --- a/fs/ocfs2/buffer_head_io.c +++ b/fs/ocfs2/buffer_head_io.c @@ -319,11 +319,9 @@ int ocfs2_read_blocks(struct ocfs2_caching_info *ci, u64 block, int nr, continue; } - get_bh(bh); /* for end_buffer_read_sync() */ if (validate) set_buffer_needs_validate(bh); - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, bh_end_read); continue; } } From 184dec3146911b923a7eff11545f85797c2b9f81 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:34 +0100 Subject: [PATCH 21/34] ocfs2: Convert ocfs2_write_super_or_backup to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-22-willy@infradead.org Reviewed-by: Joseph Qi Reviewed-by: Jan Kara Cc: ocfs2-devel@lists.linux.dev Signed-off-by: Christian Brauner (Amutable) --- fs/ocfs2/buffer_head_io.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c index 185ff4dcba75..8e67eba70674 100644 --- a/fs/ocfs2/buffer_head_io.c +++ b/fs/ocfs2/buffer_head_io.c @@ -440,10 +440,8 @@ int ocfs2_write_super_or_backup(struct ocfs2_super *osb, /* remove from dirty list before I/O. */ clear_buffer_dirty(bh); - get_bh(bh); /* for end_buffer_write_sync() */ - bh->b_end_io = end_buffer_write_sync; ocfs2_compute_meta_ecc(osb->sb, bh->b_data, &di->i_check); - submit_bh(REQ_OP_WRITE, bh); + bh_submit(bh, REQ_OP_WRITE, bh_end_write); wait_on_buffer(bh); From f8ff032ce5f0c08348b55468330a67c93b58848b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:35 +0100 Subject: [PATCH 22/34] gfs2: Convert gfs2_metapath_ra to bh_submit() Avoid an extra indirect function call by using bh_submit() instead of submit_bh(). Also simplify the control flow now that the buffer refcount is not put by bh_end_read(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-23-willy@infradead.org Reviewed-by: Jan Kara Cc: gfs2@lists.linux.dev Signed-off-by: Christian Brauner (Amutable) --- fs/gfs2/bmap.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index b3d7fcd95f03..d158c4b7413d 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -304,14 +304,15 @@ static void gfs2_metapath_ra(struct gfs2_glock *gl, __be64 *start, __be64 *end) rabh = gfs2_getbuf(gl, be64_to_cpu(*t), CREATE); if (trylock_buffer(rabh)) { if (!buffer_uptodate(rabh)) { - rabh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ | REQ_RAHEAD | REQ_META | - REQ_PRIO, rabh); - continue; + bh_submit(rabh, + REQ_OP_READ | REQ_RAHEAD | REQ_META | + REQ_PRIO, + bh_end_read); + } else { + unlock_buffer(rabh); } - unlock_buffer(rabh); } - brelse(rabh); + put_bh(rabh); } } From 6390838da4901e4a332c11b1d6ddc20f167005f9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:36 +0100 Subject: [PATCH 23/34] gfs2: Convert gfs2_dir_readahead to bh_submit() Avoid an extra indirect function call by using bh_submit() instead of submit_bh(). Also simplify the control flow now that the buffer refcount is not put by bh_end_read(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-24-willy@infradead.org Reviewed-by: Jan Kara Cc: gfs2@lists.linux.dev Signed-off-by: Christian Brauner (Amutable) --- fs/gfs2/dir.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/fs/gfs2/dir.c b/fs/gfs2/dir.c index 022dbb31e0d9..0237b36b9eb1 100644 --- a/fs/gfs2/dir.c +++ b/fs/gfs2/dir.c @@ -1505,15 +1505,13 @@ static void gfs2_dir_readahead(struct inode *inode, unsigned hsize, u32 index, if (trylock_buffer(bh)) { if (buffer_uptodate(bh)) { unlock_buffer(bh); - brelse(bh); - continue; + } else { + bh_submit(bh, REQ_OP_READ | REQ_RAHEAD | + REQ_META | REQ_PRIO, + bh_end_read); } - bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ | REQ_RAHEAD | REQ_META | - REQ_PRIO, bh); - continue; } - brelse(bh); + put_bh(bh); } } From 037f3833ba9f893d2a84c9a9fd74d78201bb2b35 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:37 +0100 Subject: [PATCH 24/34] gfs2: Remove use of b_end_io in gfs2_meta_read_endio() All buffer heads submitted by gfs2_submit_bhs() use end_buffer_read_sync() so we can call it directly. Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-25-willy@infradead.org Reviewed-by: Jan Kara Cc: gfs2@lists.linux.dev Signed-off-by: Christian Brauner (Amutable) --- fs/gfs2/meta_io.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c index d407dd476e72..df5f05054f6e 100644 --- a/fs/gfs2/meta_io.c +++ b/fs/gfs2/meta_io.c @@ -212,7 +212,7 @@ static void gfs2_meta_read_endio(struct bio *bio) do { struct buffer_head *next = bh->b_this_page; len -= bh->b_size; - bh->b_end_io(bh, !bio->bi_status); + end_buffer_read_sync(bh, bio->bi_status == BLK_STS_OK); bh = next; } while (bh && len); } @@ -275,7 +275,6 @@ int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags, unlock_buffer(bh); flags &= ~DIO_WAIT; } else { - bh->b_end_io = end_buffer_read_sync; get_bh(bh); bhs[num++] = bh; } @@ -288,7 +287,6 @@ int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags, unlock_buffer(bh); brelse(bh); } else { - bh->b_end_io = end_buffer_read_sync; bhs[num++] = bh; } } From d755b0a9949b46f207ed079378c40e0181f398b1 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:38 +0100 Subject: [PATCH 25/34] gfs2: Convert gfs2_aspace_write_folio to bh_submit() Avoid an extra indirect function call by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-26-willy@infradead.org Reviewed-by: Jan Kara Cc: gfs2@lists.linux.dev Signed-off-by: Christian Brauner (Amutable) --- fs/gfs2/meta_io.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c index df5f05054f6e..4895bb010cfb 100644 --- a/fs/gfs2/meta_io.c +++ b/fs/gfs2/meta_io.c @@ -59,7 +59,7 @@ static void gfs2_aspace_write_folio(struct folio *folio, continue; } if (test_clear_buffer_dirty(bh)) { - mark_buffer_async_write(bh); + set_buffer_async_write(bh); } else { unlock_buffer(bh); } @@ -75,7 +75,8 @@ static void gfs2_aspace_write_folio(struct folio *folio, do { struct buffer_head *next = bh->b_this_page; if (buffer_async_write(bh)) { - submit_bh(REQ_OP_WRITE | write_flags, bh); + bh_submit(bh, REQ_OP_WRITE | write_flags, + bh_end_async_write); nr_underway++; } bh = next; @@ -221,7 +222,7 @@ static void gfs2_meta_read_endio(struct bio *bio) /* * Submit several consecutive buffer head I/O requests as a single bio I/O - * request. (See submit_bh_wbc.) + * request. (See bh_submit.) */ static void gfs2_submit_bhs(blk_opf_t opf, struct buffer_head *bhs[], int num) { From e6eff926482d0f6ebe038a15348158682f5711a0 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:39 +0100 Subject: [PATCH 26/34] buffer: Remove mark_buffer_async_write() There are no more callers of this function, so delete it. end_buffer_async_write() then has only one caller left, so inline it into bh_end_async_write(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-27-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 38 ++++++++++++------------------------- include/linux/buffer_head.h | 1 - 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index 50bcef50a4a4..d35f584a06f9 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -438,12 +438,19 @@ static void bh_end_async_read(struct bio *bio) end_buffer_async_read(bh, uptodate); } -/* - * Completion handler for block_write_full_folio() - folios which are unlocked - * during I/O, and which have the writeback flag cleared upon I/O completion. +/** + * bh_end_async_write - I/O end handler for async folio writes + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're doing the equivalent of + * block_write_full_folio(). That is, the folio is unlocked, and will + * have its writeback flag cleared once all async write buffers have + * completed. */ -static void end_buffer_async_write(struct buffer_head *bh, int uptodate) +void bh_end_async_write(struct bio *bio) { + struct buffer_head *bh; + bool success = bio_endio_bh(bio, &bh); unsigned long flags; struct buffer_head *first; struct buffer_head *tmp; @@ -452,7 +459,7 @@ static void end_buffer_async_write(struct buffer_head *bh, int uptodate) BUG_ON(!buffer_async_write(bh)); folio = bh->b_folio; - if (uptodate) { + if (success) { set_buffer_uptodate(bh); } else { buffer_io_error(bh, ", lost async page write"); @@ -480,29 +487,8 @@ static void end_buffer_async_write(struct buffer_head *bh, int uptodate) still_busy: spin_unlock_irqrestore(&first->b_uptodate_lock, flags); } - -/** - * bh_end_async_write - I/O end handler for async folio writes - * @bio: The bio being completed. - * - * Pass this function to bh_submit() if you're doing the equivalent of - * block_write_full_folio(). - */ -void bh_end_async_write(struct bio *bio) -{ - struct buffer_head *bh; - bool success = bio_endio_bh(bio, &bh); - end_buffer_async_write(bh, success); -} EXPORT_SYMBOL(bh_end_async_write); -void mark_buffer_async_write(struct buffer_head *bh) -{ - bh->b_end_io = end_buffer_async_write; - set_buffer_async_write(bh); -} -EXPORT_SYMBOL(mark_buffer_async_write); - /* * fs/buffer.c contains helper functions for buffer-backed address space's diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index b0a31a90fa79..f0a9f3f05ac1 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -224,7 +224,6 @@ static inline void clean_bdev_bh_alias(struct buffer_head *bh) clean_bdev_aliases(bh->b_bdev, bh->b_blocknr, 1); } -void mark_buffer_async_write(struct buffer_head *bh); void __wait_on_buffer(struct buffer_head *); wait_queue_head_t *bh_waitq_head(struct buffer_head *bh); struct buffer_head *__find_get_block(struct block_device *bdev, sector_t block, From 673c1546cdd73728bc353bdadbbeea6e2dbef42a Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:40 +0100 Subject: [PATCH 27/34] nilfs2: Convert nilfs_btnode_submit_block to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-28-willy@infradead.org Acked-by: Ryusuke Konishi Reviewed-by: Viacheslav Dubeyko Reviewed-by: Jan Kara Cc: linux-nilfs@vger.kernel.org Signed-off-by: Christian Brauner (Amutable) --- fs/nilfs2/btnode.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c index 2e553d698d0f..680e4009a30d 100644 --- a/fs/nilfs2/btnode.c +++ b/fs/nilfs2/btnode.c @@ -134,9 +134,7 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr, } set_buffer_mapped(bh); bh->b_blocknr = pblocknr; /* set block address for read */ - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(opf, bh); + bh_submit(bh, opf, bh_end_read); bh->b_blocknr = blocknr; /* set back to the given block address */ *submit_ptr = pblocknr; err = 0; From 143a7cea4280562a0cdc3ca43e6e1ced4f8ec61d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:41 +0100 Subject: [PATCH 28/34] nilfs2: Convert nilfs_gccache_submit_read_data to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-29-willy@infradead.org Acked-by: Ryusuke Konishi Reviewed-by: Viacheslav Dubeyko Reviewed-by: Jan Kara Cc: linux-nilfs@vger.kernel.org Signed-off-by: Christian Brauner (Amutable) --- fs/nilfs2/gcinode.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c index 62d4c1b787e9..85379ac23ae2 100644 --- a/fs/nilfs2/gcinode.c +++ b/fs/nilfs2/gcinode.c @@ -83,9 +83,7 @@ int nilfs_gccache_submit_read_data(struct inode *inode, sector_t blkoff, if (!buffer_mapped(bh)) set_buffer_mapped(bh); bh->b_blocknr = pbn; - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, bh_end_read); if (vbn) bh->b_blocknr = vbn; out: From b9fa6916b1141e3648779e99039b02d4c57225f6 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:42 +0100 Subject: [PATCH 29/34] nilfs2: Convert nilfs_mdt_submit_block to bh_submit() Avoid an extra indirect function call and changing the buffer refcount by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-30-willy@infradead.org Acked-by: Ryusuke Konishi Reviewed-by: Viacheslav Dubeyko Reviewed-by: Jan Kara Cc: linux-nilfs@vger.kernel.org Signed-off-by: Christian Brauner (Amutable) --- fs/nilfs2/mdt.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c index 09adb40c65e5..2a435349fd21 100644 --- a/fs/nilfs2/mdt.c +++ b/fs/nilfs2/mdt.c @@ -148,9 +148,7 @@ nilfs_mdt_submit_block(struct inode *inode, unsigned long blkoff, blk_opf_t opf, } map_bh(bh, inode->i_sb, (sector_t)blknum); - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(opf, bh); + bh_submit(bh, opf, bh_end_read); ret = 0; trace_nilfs2_mdt_submit_block(inode, inode->i_ino, blkoff, From 2dc07082e3f5d51ae3bf293f9c39c192a1227242 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:43 +0100 Subject: [PATCH 30/34] md-bitmap: Convert read_file_page and write_file_page to bh_submit() Avoid an extra indirect function call by using bh_submit() instead of submit_bh(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-31-willy@infradead.org Reviewed-by: Jan Kara Cc: linux-raid@vger.kernel.org Signed-off-by: Christian Brauner (Amutable) --- drivers/md/md-bitmap.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 83378c033c72..ffc9891be22b 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -501,6 +501,18 @@ static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index, static void md_bitmap_file_kick(struct bitmap *bitmap); #ifdef CONFIG_MD_BITMAP_FILE +static void end_bitmap_write(struct bio *bio) +{ + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + struct bitmap *bitmap = bh->b_private; + + if (!uptodate) + set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); + if (atomic_dec_and_test(&bitmap->pending_writes)) + wake_up(&bitmap->write_wait); +} + static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) { struct buffer_head *bh = page_buffers(page); @@ -509,7 +521,7 @@ static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) atomic_inc(&bitmap->pending_writes); set_buffer_locked(bh); set_buffer_mapped(bh); - submit_bh(REQ_OP_WRITE | REQ_SYNC, bh); + bh_submit(bh, REQ_OP_WRITE | REQ_SYNC, end_bitmap_write); bh = bh->b_this_page; } @@ -518,16 +530,6 @@ static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) atomic_read(&bitmap->pending_writes) == 0); } -static void end_bitmap_write(struct buffer_head *bh, int uptodate) -{ - struct bitmap *bitmap = bh->b_private; - - if (!uptodate) - set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); - if (atomic_dec_and_test(&bitmap->pending_writes)) - wake_up(&bitmap->write_wait); -} - static void free_buffers(struct page *page) { struct buffer_head *bh; @@ -591,12 +593,11 @@ static int read_file_page(struct file *file, unsigned long index, else count -= blocksize; - bh->b_end_io = end_bitmap_write; bh->b_private = bitmap; atomic_inc(&bitmap->pending_writes); set_buffer_locked(bh); set_buffer_mapped(bh); - submit_bh(REQ_OP_READ, bh); + bh_submit(bh, REQ_OP_READ, end_bitmap_write); } blk_cur++; bh = bh->b_this_page; From 2911b935825523a71a71eecfe908902d181c041e Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:44 +0100 Subject: [PATCH 31/34] buffer: Remove submit_bh() No users are left; remove this API. Also remove/fix comments mentioning it, and end_bio_bh_io_sync() as it's now unused. Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-32-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- Documentation/trace/ftrace.rst | 4 ++-- fs/buffer.c | 27 ++++++--------------------- include/linux/buffer_head.h | 3 +-- mm/vmscan.c | 2 +- 4 files changed, 10 insertions(+), 26 deletions(-) diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst index b9efb148a5c2..2ed1b96e440b 100644 --- a/Documentation/trace/ftrace.rst +++ b/Documentation/trace/ftrace.rst @@ -1624,7 +1624,7 @@ function-trace, we get a much larger output:: => blk_queue_bio => submit_bio_noacct => submit_bio - => submit_bh + => bh_submit => __ext3_get_inode_loc => ext3_iget => ext3_lookup @@ -1909,7 +1909,7 @@ tracers. => blk_queue_bio => submit_bio_noacct => submit_bio - => submit_bh + => bh_submit => ext3_bread => ext3_dir_bread => htree_dirblock_to_tree diff --git a/fs/buffer.c b/fs/buffer.c index d35f584a06f9..3df0ea1a6342 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1187,14 +1187,6 @@ void __bforget(struct buffer_head *bh) } EXPORT_SYMBOL(__bforget); -static void end_bio_bh_io_sync(struct bio *bio) -{ - struct buffer_head *bh; - bool uptodate = bio_endio_bh(bio, &bh); - - bh->b_end_io(bh, uptodate); -} - static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh, gfp_t gfp_mask) { @@ -1829,15 +1821,15 @@ static struct buffer_head *folio_create_buffers(struct folio *folio, /* * While block_write_full_folio is writing back the dirty buffers under - * the page lock, whoever dirtied the buffers may decide to clean them + * the folio lock, whoever dirtied the buffers may decide to clean them * again at any time. We handle that by only looking at the buffer * state inside lock_buffer(). * * If block_write_full_folio() is called for regular writeback - * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a - * locked buffer. This only can happen if someone has written the buffer - * directly, with submit_bh(). At the address_space level PageWriteback - * prevents this contention from occurring. + * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a folio which + * has a locked buffer. This only can happen if someone has written + * the buffer directly, with bh_submit(). At the address_space level + * the folio writeback flag prevents this contention from occurring. * * If block_write_full_folio() is called with wbc->sync_mode == * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this @@ -1954,7 +1946,7 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio, /* * The folio was marked dirty, but the buffers were * clean. Someone wrote them back by hand with - * write_dirty_buffer/submit_bh. A rare case. + * write_dirty_buffer/bh_submit. A rare case. */ folio_end_writeback(folio); @@ -2800,13 +2792,6 @@ sector_t generic_block_bmap(struct address_space *mapping, sector_t block, } EXPORT_SYMBOL(generic_block_bmap); -void submit_bh(blk_opf_t opf, struct buffer_head *bh) -{ - BUG_ON(!bh->b_end_io); - __bh_submit(bh, opf, WRITE_LIFE_NOT_SET, NULL, end_bio_bh_io_sync); -} -EXPORT_SYMBOL(submit_bh); - void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) { lock_buffer(bh); diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index f0a9f3f05ac1..7629130d42c4 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -55,7 +55,7 @@ typedef void (bh_end_io_t)(struct buffer_head *bh, int uptodate); * is the bio, and buffer_heads are used for extracting block * mappings (via a get_block_t call), for tracking state within * a folio (via a folio_mapping) and for wrapping bio submission - * for backward compatibility reasons (e.g. submit_bh). + * for backward compatibility reasons (e.g. bh_submit). */ struct buffer_head { unsigned long b_state; /* buffer state bitmap (see above) */ @@ -244,7 +244,6 @@ void __lock_buffer(struct buffer_head *bh); int sync_dirty_buffer(struct buffer_head *bh); int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); -void submit_bh(blk_opf_t, struct buffer_head *); void bh_submit(struct buffer_head *, blk_opf_t, bio_end_io_t); void write_boundary_block(struct block_device *bdev, sector_t bblock, unsigned blocksize); diff --git a/mm/vmscan.c b/mm/vmscan.c index bd1b1aa12581..67231d3189ef 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1449,7 +1449,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, * is possible for a folio to have the dirty flag set, * but it is actually clean (all its buffers are clean). * This happens if the buffers were written out directly, - * with submit_bh(). ext3 will do this, as well as + * with bh_submit(). ext3 will do this, as well as * the blockdev mapping. filemap_release_folio() will * discover that cleanness and will drop the buffers * and mark the folio clean - it can be freed. From ac75b922bb67cc8edb52006c9346dc0ca91d04c8 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:45 +0100 Subject: [PATCH 32/34] buffer: Remove b_end_io This shrinks buffer_head by 8 bytes, letting us pack more buffer heads per slab. With a Debian config, it shrinks from 104 bytes to 96 bytes which is 42 objects per 4KiB page rather than 39, a 7% reduction in the amount of memory used. Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-33-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- Documentation/filesystems/locking.rst | 14 -------------- drivers/md/raid5.h | 6 +++--- fs/buffer.c | 1 - include/linux/buffer_head.h | 4 +--- 4 files changed, 4 insertions(+), 21 deletions(-) diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index 8421ea21bd35..a27aca42fd85 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -416,20 +416,6 @@ lm_open_conflict yes no no lm_breaker_timedout yes no no ====================== ============= ================= ========= -buffer_head -=========== - -prototypes:: - - void (*b_end_io)(struct buffer_head *bh, int uptodate); - -locking rules: - -called from interrupts. In other words, extreme care is needed here. -bh is locked, but that's all warranties we have here. Currently only RAID1, -highmem, fs/buffer.c, and fs/ntfs/aops.c are providing these. Block devices -call this method upon the IO completion. - block_device_operations ======================= prototypes:: diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h index 1c7b710fc9c1..1dfa60a41d91 100644 --- a/drivers/md/raid5.h +++ b/drivers/md/raid5.h @@ -38,7 +38,7 @@ * Clean -> Dirty - on compute_parity to satisfy write/sync (RECONSTRUCT or RMW) * * The Want->Empty, Want->Clean, Dirty->Clean, transitions - * all happen in b_end_io at interrupt time. + * all happen in end_io at interrupt time. * Each sets the Uptodate bit before releasing the Lock bit. * This leaves one multi-stage transition: * Want->Dirty->Clean @@ -64,7 +64,7 @@ * together, but we are not guaranteed of that so we allow for more. * * If a buffer is on the read list when the associated cache buffer is - * Uptodate, the data is copied into the read buffer and it's b_end_io + * Uptodate, the data is copied into the read buffer and it's end_io * routine is called. This may happen in the end_request routine only * if the buffer has just successfully been read. end_request should * remove the buffers from the list and then set the Uptodate bit on @@ -76,7 +76,7 @@ * into the cache buffer, which is then marked dirty, and moved onto a * third list, the written list (bh_written). Once both the parity * block and the cached buffer are successfully written, any buffer on - * a written list can be returned with b_end_io. + * a written list can be returned with end_io. * * The write list and read list both act as fifos. The read list, * write list and written list are protected by the device_lock. diff --git a/fs/buffer.c b/fs/buffer.c index 3df0ea1a6342..ccda92920175 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -946,7 +946,6 @@ static sector_t folio_init_buffers(struct folio *folio, do { if (!buffer_mapped(bh)) { - bh->b_end_io = NULL; bh->b_private = NULL; bh->b_bdev = bdev; bh->b_blocknr = block; diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 7629130d42c4..1ee56c9f2327 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -46,7 +46,6 @@ enum bh_state_bits { struct page; struct buffer_head; struct address_space; -typedef void (bh_end_io_t)(struct buffer_head *bh, int uptodate); /* * Historically, a buffer_head was used to map a single block @@ -70,8 +69,7 @@ struct buffer_head { char *b_data; /* pointer to data within the page */ struct block_device *b_bdev; - bh_end_io_t *b_end_io; /* I/O completion */ - void *b_private; /* reserved for b_end_io */ + void *b_private; /* reserved for bio_end_io */ struct list_head b_assoc_buffers; /* associated with another mapping */ struct mapping_metadata_bhs *b_mmb; /* head of the list of metadata bhs * this buffer is associated with */ From 15dfe154126743baaf3be669cdec893f3f174118 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:46 +0100 Subject: [PATCH 33/34] buffer: Change calling convention for end_buffer_read_sync() Unify end_buffer_read_sync() and __end_buffer_read_notouch() by requiring the caller put the refcount on the buffer. The only caller is in the gfs2_meta_read() path, and there we can put the refcount after locking the buffer. Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-34-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 28 +++++++++------------------- fs/gfs2/meta_io.c | 2 +- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index ccda92920175..db604c0e54d4 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -157,15 +157,15 @@ bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp) } EXPORT_SYMBOL(bio_endio_bh); -/* - * End-of-IO handler helper function which does not touch the bh after - * unlocking it. - * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but - * a race there is benign: unlock_buffer() only use the bh's address for - * hashing after unlocking the buffer, so it doesn't actually touch the bh - * itself. +/** + * end_buffer_read_sync - Handle buffer reads finishing + * @bh: The buffer. + * @uptodate: True if the read was successful. + * + * If a buffer is read through a mechanism that isn't bh_submit(), you + * can call this function to finish the read. */ -static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) +void end_buffer_read_sync(struct buffer_head *bh, int uptodate) { if (uptodate) { set_buffer_uptodate(bh); @@ -175,16 +175,6 @@ static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) } unlock_buffer(bh); } - -/* - * Default synchronous end-of-IO handler.. Just mark it up-to-date and - * unlock the buffer. - */ -void end_buffer_read_sync(struct buffer_head *bh, int uptodate) -{ - put_bh(bh); - __end_buffer_read_notouch(bh, uptodate); -} EXPORT_SYMBOL(end_buffer_read_sync); /** @@ -198,7 +188,7 @@ void bh_end_read(struct bio *bio) { struct buffer_head *bh; bool uptodate = bio_endio_bh(bio, &bh); - __end_buffer_read_notouch(bh, uptodate); + end_buffer_read_sync(bh, uptodate); } EXPORT_SYMBOL(bh_end_read); diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c index 4895bb010cfb..a87cfbf0df38 100644 --- a/fs/gfs2/meta_io.c +++ b/fs/gfs2/meta_io.c @@ -286,10 +286,10 @@ int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags, lock_buffer(bh); if (buffer_uptodate(bh)) { unlock_buffer(bh); - brelse(bh); } else { bhs[num++] = bh; } + brelse(bh); } gfs2_submit_bhs(REQ_OP_READ | REQ_META | REQ_PRIO, bhs, num); From b20f15420f786a029be09b1d9b81695581d1a122 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:31:47 +0100 Subject: [PATCH 34/34] buffer: Remove end_buffer_write_sync() It has no callers left, so delete it. Inline __end_buffer_write_sync() into bh_end_write(). Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260528173150.1093780-35-willy@infradead.org Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- fs/buffer.c | 29 +++++++++-------------------- include/linux/buffer_head.h | 1 - 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index db604c0e54d4..7ed8dd77d221 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -192,25 +192,6 @@ void bh_end_read(struct bio *bio) } EXPORT_SYMBOL(bh_end_read); -static void __end_buffer_write_sync(struct buffer_head *bh, int uptodate) -{ - if (uptodate) { - set_buffer_uptodate(bh); - } else { - buffer_io_error(bh, ", lost sync page write"); - mark_buffer_write_io_error(bh); - clear_buffer_uptodate(bh); - } - unlock_buffer(bh); -} - -void end_buffer_write_sync(struct buffer_head *bh, int uptodate) -{ - __end_buffer_write_sync(bh, uptodate); - put_bh(bh); -} -EXPORT_SYMBOL(end_buffer_write_sync); - /** * bh_end_write - I/O end handler for writes * @bio: The bio being completed. @@ -222,7 +203,15 @@ void bh_end_write(struct bio *bio) { struct buffer_head *bh; bool success = bio_endio_bh(bio, &bh); - __end_buffer_write_sync(bh, success); + + if (success) { + set_buffer_uptodate(bh); + } else { + buffer_io_error(bh, ", lost sync page write"); + mark_buffer_write_io_error(bh); + clear_buffer_uptodate(bh); + } + unlock_buffer(bh); } EXPORT_SYMBOL(bh_end_write); diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 1ee56c9f2327..8b23bc9a244c 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -201,7 +201,6 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size); struct buffer_head *create_empty_buffers(struct folio *folio, unsigned long blocksize, unsigned long b_state); void end_buffer_read_sync(struct buffer_head *bh, int uptodate); -void end_buffer_write_sync(struct buffer_head *bh, int uptodate); bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp); /* Completion routines suitable for passing to bh_submit() */