From cba38ec4cbd3a7b8b942a8d52531a05be8a9ff0d Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Thu, 21 May 2026 17:50:14 +0800 Subject: [PATCH 1/6] writeback: fix race between cgroup_writeback_umount() and inode_switch_wbs() When a container exits, the following BUG_ON() is occasionally triggered: ================================================================== VFS: Busy inodes after unmount of sdb (ext4) ------------[ cut here ]------------ kernel BUG at fs/super.c:695! CPU: 3 PID: 6 Comm: containerd-shim Tainted: G OE K 6.6 #1 pstate: 63400009 (nZCv daif +PAN -UAO +TCO +DIT -SSBS BTYPE=--) pc : generic_shutdown_super+0xf0/0x100 lr : generic_shutdown_super+0xf0/0x100 Call trace: generic_shutdown_super+0xf0/0x100 kill_block_super+0x20/0x48 ext4_kill_sb+0x28/0x60 deactivate_locked_super+0x54/0x130 deactivate_super+0x84/0xa0 cleanup_mnt+0xa4/0x140 __cleanup_mnt+0x18/0x28 task_work_run+0x78/0xe0 do_notify_resume+0x204/0x240 ================================================================== The root cause is a race between cgroup_writeback_umount() and inode_switch_wbs()/cleanup_offline_cgwb(). There is a window between inode_prepare_wbs_switch() returning true and the subsequent wb_queue_isw() call. Following is the process that triggers the issue: CPU A (umount) | CPU B (writeback) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ inode_switch_wbs/cleanup_offline_cgwb atomic_inc(&isw_nr_in_flight) inode_prepare_wbs_switch -> passes SB_ACTIVE check __iget(inode) generic_shutdown_super sb->s_flags &= ~SB_ACTIVE cgroup_writeback_umount(sb) smp_mb() atomic_read(&isw_nr_in_flight) rcu_barrier() -> no pending RCU callbacks flush_workqueue(isw_wq) -> nothing queued, returns evict_inodes(sb) -> Inode skipped as isw still holds a ref. sop->put_super(sb) /* destroys percpu counters */ -> VFS: Busy inodes after unmount! wb_queue_isw() queue_work(isw_wq, ...) /* later in work function */ inode_switch_wbs_work_fn process_inode_switch_wbs iput() -> evict percpu_counter_dec() // UAF! Fix this by extending the RCU read-side critical section in inode_switch_wbs() and cleanup_offline_cgwb() to cover from inode_prepare_wbs_switch() through wb_queue_isw(). Since there is no sleep in this window, rcu_read_lock() can be used. Then add a synchronize_rcu() in cgroup_writeback_umount() before the existing rcu_barrier(), so that all in-flight switchers that have passed the SB_ACTIVE check have completed queue_work() before flush_workqueue() is called. The existing rcu_barrier() is intentionally retained so this fix can be backported unchanged to stable kernels (5.10.y, 6.6.y, ...) that still queue switches via queue_rcu_work(). It is a no-op on current mainline (since commit e1b849cfa6b6 ("writeback: Avoid contention on wb->list_lock when switching inodes")) and is removed in a follow-up patch. Fixes: a1a0e23e4903 ("writeback: flush inode cgroup wb switches instead of pinning super_block") Cc: stable@vger.kernel.org Suggested-by: Jan Kara Link: https://lore.kernel.org/all/mxnjq2l6guusfchvauxr3v7c4bwjasybxlleqbbh4efloeqspz@iqylk76ohufz Reviewed-by: Jan Kara Signed-off-by: Baokun Li Link: https://patch.msgid.link/20260521095016.2791354-2-libaokun@linux.alibaba.com Acked-by: Tejun Heo Signed-off-by: Christian Brauner (Amutable) --- fs/fs-writeback.c | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index a65694cbfe68..6766de9f9d75 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -660,12 +660,19 @@ static void inode_switch_wbs(struct inode *inode, int new_wb_id) atomic_inc(&isw_nr_in_flight); - /* find and pin the new wb */ + /* + * Paired with synchronize_rcu() in cgroup_writeback_umount(): + * holding rcu_read_lock across inode_prepare_wbs_switch() + * (covering the SB_ACTIVE check and the inode grab) and + * wb_queue_isw() ensures synchronize_rcu() cannot return until + * the work is queued, so the subsequent flush_workqueue() will + * wait for the switch. + */ rcu_read_lock(); + /* find and pin the new wb */ memcg_css = css_from_id(new_wb_id, &memory_cgrp_subsys); if (memcg_css && !css_tryget(memcg_css)) memcg_css = NULL; - rcu_read_unlock(); if (!memcg_css) goto out_free; @@ -681,9 +688,11 @@ static void inode_switch_wbs(struct inode *inode, int new_wb_id) trace_inode_switch_wbs_queue(inode->i_wb, new_wb, 1); wb_queue_isw(new_wb, isw); + rcu_read_unlock(); return; out_free: + rcu_read_unlock(); atomic_dec(&isw_nr_in_flight); if (new_wb) wb_put(new_wb); @@ -741,6 +750,14 @@ bool cleanup_offline_cgwb(struct bdi_writeback *wb) new_wb = &wb->bdi->wb; /* wb_get() is noop for bdi's wb */ nr = 0; + /* + * Paired with synchronize_rcu() in cgroup_writeback_umount(). + * Holding rcu_read_lock across the SB_ACTIVE check, the inode grab + * and wb_queue_isw() ensures synchronize_rcu() cannot return until + * the work is queued, so the subsequent flush_workqueue() will wait + * for the switch. + */ + rcu_read_lock(); spin_lock(&wb->list_lock); /* * In addition to the inodes that have completed writeback, also switch @@ -758,6 +775,7 @@ bool cleanup_offline_cgwb(struct bdi_writeback *wb) /* no attached inodes? bail out */ if (nr == 0) { + rcu_read_unlock(); atomic_dec(&isw_nr_in_flight); wb_put(new_wb); kfree(isw); @@ -766,6 +784,7 @@ bool cleanup_offline_cgwb(struct bdi_writeback *wb) trace_inode_switch_wbs_queue(wb, new_wb, nr); wb_queue_isw(new_wb, isw); + rcu_read_unlock(); return restart; } @@ -1221,6 +1240,14 @@ void cgroup_writeback_umount(struct super_block *sb) smp_mb(); if (atomic_read(&isw_nr_in_flight)) { + /* + * Paired with rcu_read_lock() in inode_switch_wbs() and + * cleanup_offline_cgwb(). synchronize_rcu() waits for any + * in-flight switcher that already passed the SB_ACTIVE check + * to finish queueing its work, so flush_workqueue() below + * will then drain it. + */ + synchronize_rcu(); /* * Use rcu_barrier() to wait for all pending callbacks to * ensure that all in-flight wb switches are in the workqueue. From e90a6d668e26e00a72df2d09c173b563468f09c9 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Thu, 21 May 2026 17:50:15 +0800 Subject: [PATCH 2/6] writeback: drop now-unnecessary rcu_barrier() in cgroup_writeback_umount() Commit e1b849cfa6b6 ("writeback: Avoid contention on wb->list_lock when switching inodes") replaced the queue_rcu_work() based scheduling of inode wb switches with a plain queue_work(). Since then no switcher goes through call_rcu(), so rcu_barrier() in cgroup_writeback_umount() has no callbacks of its own to wait for. It still drains unrelated call_rcu() callbacks from other subsystems on busy systems, which incidentally slows umount down; drop it. Fixes: e1b849cfa6b6 ("writeback: Avoid contention on wb->list_lock when switching inodes") Reviewed-by: Jan Kara Signed-off-by: Baokun Li Link: https://patch.msgid.link/20260521095016.2791354-3-libaokun@linux.alibaba.com Acked-by: Tejun Heo Signed-off-by: Christian Brauner (Amutable) --- fs/fs-writeback.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 6766de9f9d75..325a30cc35bf 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -1248,11 +1248,6 @@ void cgroup_writeback_umount(struct super_block *sb) * will then drain it. */ synchronize_rcu(); - /* - * Use rcu_barrier() to wait for all pending callbacks to - * ensure that all in-flight wb switches are in the workqueue. - */ - rcu_barrier(); flush_workqueue(isw_wq); } } From 31c1d19ead2c26a63859a2757d8b786765ba9cdd Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Thu, 21 May 2026 17:50:16 +0800 Subject: [PATCH 3/6] writeback: use a per-sb counter to drain inode wb switches at umount Tracking in-flight inode wb switches with a single global counter (isw_nr_in_flight) plus a synchronize_rcu() based wait in cgroup_writeback_umount() forces every umount to take a global hit whenever any other superblock on the system has wb switches in flight, even if the superblock being unmounted has none of its own. Replace the global synchronize_rcu()/flush_workqueue() pair with a per-sb counter, s_isw_nr_in_flight, plus three small helpers: - cgroup_writeback_pin(sb) - increment counter - cgroup_writeback_unpin(sb) - decrement and wake drainer if last - cgroup_writeback_drain(sb) - wait for counter to reach zero The wiring is: - inode_prepare_wbs_switch() pins before checking SB_ACTIVE and grabbing the inode; failure paths unpin before returning. A lockless SB_ACTIVE check at the top of the function lets us skip the atomic_inc/smp_mb dance once SB_ACTIVE has been cleared (it is monotonic and never set back). - process_inode_switch_wbs() unpins after the matching iput(). - cgroup_writeback_umount() drains the per-sb counter via wait_var_event(). The smp_mb() pair between inode_prepare_wbs_switch() and cgroup_writeback_umount() keeps the SB_ACTIVE / counter ordering: either the umounter sees a non-zero counter and waits, or the switcher sees SB_ACTIVE cleared and aborts before grabbing the inode. The global isw_nr_in_flight is left in place, since it is still used to throttle in-flight switches via WB_FRN_MAX_IN_FLIGHT. The rcu_read_lock() extension in inode_switch_wbs() and cleanup_offline_cgwb() that the race fix added is no longer needed and is reverted; the synchronize_rcu() that the race fix added to cgroup_writeback_umount() is dropped as well. The following numbers were measured on a 16 vCPU QEMU guest with 4 background superblocks each churning "create memcg -> write 1 MiB -> rmdir memcg" to keep the global isw_nr_in_flight non-zero. Latencies are wall-clock around umount(8); only the target sb's umount is measured. Target sb runs its own cgwb churn: p50 p95 p99 max global synchronize_rcu() 67.6 ms 88.3 ms 88.3 ms 96.8 ms per-sb counter (this) 7.9 ms 10.0 ms 10.0 ms 10.1 ms Idle target umount latency under cross-sb cgwb-switch pressure: p50 p95 p99 max global synchronize_rcu() 62.7 ms 95.4 ms 108.1 ms 108.6 ms per-sb counter (this) 5.3 ms 6.9 ms 7.4 ms 7.4 ms no-pressure baseline 4.9 ms 5.9 ms 6.3 ms 6.7 ms 8 concurrent umounts of idle sbs under the same pressure: p50 p95 max global synchronize_rcu() 61.3 ms 99.5 ms 113.7 ms per-sb counter (this) 8.1 ms 9.1 ms 9.5 ms In-kernel cgroup_writeback_umount() time across the same run (bpftrace, ~340 calls covering all scenarios): global synchronize_rcu() 12371 ms total (~36 ms / call) per-sb counter (this) 1.37 ms total ( ~4 us / call) Suggested-by: Christian Brauner Link: https://lore.kernel.org/r/177910456953.488929.2169908940676707307.b4-review@b4 Reviewed-by: Jan Kara Signed-off-by: Baokun Li Link: https://patch.msgid.link/20260521095016.2791354-4-libaokun@linux.alibaba.com Acked-by: Tejun Heo Signed-off-by: Christian Brauner (Amutable) --- fs/fs-writeback.c | 97 ++++++++++++++++------------------ include/linux/fs/super_types.h | 8 +++ 2 files changed, 55 insertions(+), 50 deletions(-) diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 325a30cc35bf..900ad7818bd4 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -497,6 +497,23 @@ static bool inode_do_switch_wbs(struct inode *inode, return switched; } +static inline void cgroup_writeback_pin(struct super_block *sb) +{ + atomic_inc(&sb->s_isw_nr_in_flight); +} + +static inline void cgroup_writeback_unpin(struct super_block *sb) +{ + if (atomic_dec_and_test(&sb->s_isw_nr_in_flight)) + wake_up_var(&sb->s_isw_nr_in_flight); +} + +static inline void cgroup_writeback_drain(struct super_block *sb) +{ + wait_var_event(&sb->s_isw_nr_in_flight, + !atomic_read(&sb->s_isw_nr_in_flight)); +} + static void process_inode_switch_wbs(struct bdi_writeback *new_wb, struct inode_switch_wbs_context *isw) { @@ -554,8 +571,12 @@ static void process_inode_switch_wbs(struct bdi_writeback *new_wb, wb_put_many(old_wb, nr_switched); } - for (inodep = isw->inodes; *inodep; inodep++) + for (inodep = isw->inodes; *inodep; inodep++) { + struct super_block *sb = (*inodep)->i_sb; + iput(*inodep); + cgroup_writeback_unpin(sb); + } wb_put(new_wb); kfree(isw); atomic_dec(&isw_nr_in_flight); @@ -598,16 +619,19 @@ void inode_switch_wbs_work_fn(struct work_struct *work) static bool inode_prepare_wbs_switch(struct inode *inode, struct bdi_writeback *new_wb) { + /* Avoid the atomic_inc/smp_mb dance once SB_ACTIVE is gone. */ + if (!(inode->i_sb->s_flags & SB_ACTIVE)) + return false; + /* - * Paired with smp_mb() in cgroup_writeback_umount(). - * isw_nr_in_flight must be increased before checking SB_ACTIVE and - * grabbing an inode, otherwise isw_nr_in_flight can be observed as 0 - * in cgroup_writeback_umount() and the isw_wq will be not flushed. + * Pairs with smp_mb() in cgroup_writeback_umount(): the umounter either + * sees a non-zero counter and waits, or we see SB_ACTIVE clear below. */ + cgroup_writeback_pin(inode->i_sb); smp_mb(); if (IS_DAX(inode)) - return false; + goto out_unpin; /* while holding I_WB_SWITCH, no one else can update the association */ spin_lock(&inode->i_lock); @@ -615,13 +639,17 @@ static bool inode_prepare_wbs_switch(struct inode *inode, inode_state_read(inode) & (I_WB_SWITCH | I_FREEING | I_WILL_FREE) || inode_to_wb(inode) == new_wb) { spin_unlock(&inode->i_lock); - return false; + goto out_unpin; } inode_state_set(inode, I_WB_SWITCH); __iget(inode); spin_unlock(&inode->i_lock); return true; + +out_unpin: + cgroup_writeback_unpin(inode->i_sb); + return false; } static void wb_queue_isw(struct bdi_writeback *wb, @@ -660,19 +688,12 @@ static void inode_switch_wbs(struct inode *inode, int new_wb_id) atomic_inc(&isw_nr_in_flight); - /* - * Paired with synchronize_rcu() in cgroup_writeback_umount(): - * holding rcu_read_lock across inode_prepare_wbs_switch() - * (covering the SB_ACTIVE check and the inode grab) and - * wb_queue_isw() ensures synchronize_rcu() cannot return until - * the work is queued, so the subsequent flush_workqueue() will - * wait for the switch. - */ - rcu_read_lock(); /* find and pin the new wb */ + rcu_read_lock(); memcg_css = css_from_id(new_wb_id, &memory_cgrp_subsys); if (memcg_css && !css_tryget(memcg_css)) memcg_css = NULL; + rcu_read_unlock(); if (!memcg_css) goto out_free; @@ -688,11 +709,9 @@ static void inode_switch_wbs(struct inode *inode, int new_wb_id) trace_inode_switch_wbs_queue(inode->i_wb, new_wb, 1); wb_queue_isw(new_wb, isw); - rcu_read_unlock(); return; out_free: - rcu_read_unlock(); atomic_dec(&isw_nr_in_flight); if (new_wb) wb_put(new_wb); @@ -750,14 +769,6 @@ bool cleanup_offline_cgwb(struct bdi_writeback *wb) new_wb = &wb->bdi->wb; /* wb_get() is noop for bdi's wb */ nr = 0; - /* - * Paired with synchronize_rcu() in cgroup_writeback_umount(). - * Holding rcu_read_lock across the SB_ACTIVE check, the inode grab - * and wb_queue_isw() ensures synchronize_rcu() cannot return until - * the work is queued, so the subsequent flush_workqueue() will wait - * for the switch. - */ - rcu_read_lock(); spin_lock(&wb->list_lock); /* * In addition to the inodes that have completed writeback, also switch @@ -775,7 +786,6 @@ bool cleanup_offline_cgwb(struct bdi_writeback *wb) /* no attached inodes? bail out */ if (nr == 0) { - rcu_read_unlock(); atomic_dec(&isw_nr_in_flight); wb_put(new_wb); kfree(isw); @@ -784,7 +794,6 @@ bool cleanup_offline_cgwb(struct bdi_writeback *wb) trace_inode_switch_wbs_queue(wb, new_wb, nr); wb_queue_isw(new_wb, isw); - rcu_read_unlock(); return restart; } @@ -1217,39 +1226,27 @@ int cgroup_writeback_by_id(u64 bdi_id, int memcg_id, } /** - * cgroup_writeback_umount - flush inode wb switches for umount + * cgroup_writeback_umount - wait for in-flight inode wb switches on @sb * @sb: target super_block * - * This function is called when a super_block is about to be destroyed and - * flushes in-flight inode wb switches. An inode wb switch goes through - * RCU and then workqueue, so the two need to be flushed in order to ensure - * that all previously scheduled switches are finished. As wb switches are - * rare occurrences and synchronize_rcu() can take a while, perform - * flushing iff wb switches are in flight. + * Wait until every inode wb switch that already passed the SB_ACTIVE + * check on this superblock has been completed by the worker. Since + * SB_ACTIVE is cleared before this is called, no new switches can start + * for @sb, so s_isw_nr_in_flight will monotonically drop to zero. */ void cgroup_writeback_umount(struct super_block *sb) { - if (!(sb->s_bdi->capabilities & BDI_CAP_WRITEBACK)) return; /* - * SB_ACTIVE should be reliably cleared before checking - * isw_nr_in_flight, see generic_shutdown_super(). + * Pairs with smp_mb() in inode_prepare_wbs_switch(): we either observe + * a non-zero counter and wait, or the switcher sees SB_ACTIVE clear + * (cleared by generic_shutdown_super()) and bails before grabbing the + * inode. */ smp_mb(); - - if (atomic_read(&isw_nr_in_flight)) { - /* - * Paired with rcu_read_lock() in inode_switch_wbs() and - * cleanup_offline_cgwb(). synchronize_rcu() waits for any - * in-flight switcher that already passed the SB_ACTIVE check - * to finish queueing its work, so flush_workqueue() below - * will then drain it. - */ - synchronize_rcu(); - flush_workqueue(isw_wq); - } + cgroup_writeback_drain(sb); } static int __init cgroup_writeback_init(void) diff --git a/include/linux/fs/super_types.h b/include/linux/fs/super_types.h index 383050e7fdf5..1ab4e2265129 100644 --- a/include/linux/fs/super_types.h +++ b/include/linux/fs/super_types.h @@ -274,6 +274,14 @@ struct super_block { /* number of fserrors that are being sent to fsnotify/filesystems */ refcount_t s_pending_errors; + +#ifdef CONFIG_CGROUP_WRITEBACK + /* + * Number of in-flight inode wb switches for this sb. Drained by + * cgroup_writeback_umount() before tear-down. + */ + atomic_t s_isw_nr_in_flight; +#endif } __randomize_layout; /* From 0ad186cf167fdbeb6b29a8005c71973781036bd3 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 11 May 2026 07:58:27 -0400 Subject: [PATCH 4/6] mm: preserve PG_dropbehind flag during folio split __split_folio_to_order() copies page flags from the original folio to newly created sub-folios using an explicit allowlist, but PG_dropbehind is not included. When a large folio with PG_dropbehind set is split, only the head sub-folio retains the flag; all tail sub-folios silently lose it and will not be reclaimed eagerly after writeback completes. Add PG_dropbehind to the flag copy mask so that the drop-behind hint is preserved across folio splits. Fixes: a323281cdfec ("mm: add PG_dropbehind folio flag") Assisted-by: Claude:claude-opus-4-6 Signed-off-by: Jeff Layton Link: https://patch.msgid.link/20260511-dontcache-v7-1-2848ddce8090@kernel.org Acked-by: David Hildenbrand (Arm) Reviewed-by: Jan Kara Signed-off-by: Christian Brauner (Amutable) --- mm/huge_memory.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 970e077019b7..e01917b14d1a 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3642,6 +3642,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order, (1L << PG_arch_3) | #endif (1L << PG_dirty) | + (1L << PG_dropbehind) | LRU_GEN_MASK | LRU_REFS_MASK)); if (handle_hwpoison && From 88d6f128d06d492b6d178c8e8c53db8c82305ae1 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 11 May 2026 07:58:28 -0400 Subject: [PATCH 5/6] mm: track DONTCACHE dirty pages per bdi_writeback Add a per-wb WB_DONTCACHE_DIRTY counter that tracks the number of dirty pages with the dropbehind flag set (i.e., pages dirtied via RWF_DONTCACHE writes). Increment the counter alongside WB_RECLAIMABLE in folio_account_dirtied() when the folio has the dropbehind flag set, and decrement it in folio_clear_dirty_for_io() and folio_account_cleaned(). Also decrement it when a non-DONTCACHE lookup atomically clears the dropbehind flag on a dirty folio in __filemap_get_folio_mpol(), using folio_test_clear_dropbehind() to prevent concurrent lookups from double-decrementing the counter, and guarding the decrement with mapping_can_writeback() to match the increment path. Transfer the counter alongside WB_RECLAIMABLE in inode_do_switch_wbs() so that the stat is properly migrated when an inode switches cgroup writeback domains. The counter will be used by the writeback flusher to determine how many pages to write back when expediting writeback for IOCB_DONTCACHE writes, without flushing the entire BDI's dirty pages. Suggested-by: Jan Kara Assisted-by: Claude:claude-opus-4-6 Signed-off-by: Jeff Layton Link: https://patch.msgid.link/20260511-dontcache-v7-2-2848ddce8090@kernel.org Reviewed-by: Jan Kara Reviewed-by: Ritesh Harjani (IBM) Signed-off-by: Christian Brauner (Amutable) --- fs/fs-writeback.c | 4 ++++ include/linux/backing-dev-defs.h | 1 + mm/filemap.c | 15 +++++++++++++-- mm/page-writeback.c | 6 ++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 900ad7818bd4..b303516f2753 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -432,6 +432,10 @@ static bool inode_do_switch_wbs(struct inode *inode, long nr = folio_nr_pages(folio); wb_stat_mod(old_wb, WB_RECLAIMABLE, -nr); wb_stat_mod(new_wb, WB_RECLAIMABLE, nr); + if (folio_test_dropbehind(folio)) { + wb_stat_mod(old_wb, WB_DONTCACHE_DIRTY, -nr); + wb_stat_mod(new_wb, WB_DONTCACHE_DIRTY, nr); + } } } diff --git a/include/linux/backing-dev-defs.h b/include/linux/backing-dev-defs.h index a06b93446d10..cb660dd37286 100644 --- a/include/linux/backing-dev-defs.h +++ b/include/linux/backing-dev-defs.h @@ -33,6 +33,7 @@ enum wb_stat_item { WB_WRITEBACK, WB_DIRTIED, WB_WRITTEN, + WB_DONTCACHE_DIRTY, NR_WB_STAT_ITEMS }; diff --git a/mm/filemap.c b/mm/filemap.c index 4e636647100c..179f2886f8c0 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2052,8 +2052,19 @@ struct folio *__filemap_get_folio_mpol(struct address_space *mapping, if (!folio) return ERR_PTR(-ENOENT); /* not an uncached lookup, clear uncached if set */ - if (folio_test_dropbehind(folio) && !(fgp_flags & FGP_DONTCACHE)) - folio_clear_dropbehind(folio); + if (!(fgp_flags & FGP_DONTCACHE) && folio_test_clear_dropbehind(folio)) { + if (folio_test_dirty(folio) && + mapping_can_writeback(mapping)) { + struct inode *inode = mapping->host; + struct bdi_writeback *wb; + struct wb_lock_cookie cookie = {}; + long nr = folio_nr_pages(folio); + + wb = unlocked_inode_to_wb_begin(inode, &cookie); + wb_stat_mod(wb, WB_DONTCACHE_DIRTY, -nr); + unlocked_inode_to_wb_end(inode, &cookie); + } + } return folio; } EXPORT_SYMBOL(__filemap_get_folio_mpol); diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 88cd53d4ba09..8e520717d1f6 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2630,6 +2630,8 @@ static void folio_account_dirtied(struct folio *folio, wb = inode_to_wb(inode); lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, nr); + if (folio_test_dropbehind(folio)) + wb_stat_mod(wb, WB_DONTCACHE_DIRTY, nr); __zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, nr); __node_stat_mod_folio(folio, NR_DIRTIED, nr); wb_stat_mod(wb, WB_RECLAIMABLE, nr); @@ -2651,6 +2653,8 @@ void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb) long nr = folio_nr_pages(folio); lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, -nr); + if (folio_test_dropbehind(folio)) + wb_stat_mod(wb, WB_DONTCACHE_DIRTY, -nr); zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr); wb_stat_mod(wb, WB_RECLAIMABLE, -nr); task_io_account_cancelled_write(nr * PAGE_SIZE); @@ -2920,6 +2924,8 @@ bool folio_clear_dirty_for_io(struct folio *folio) if (folio_test_clear_dirty(folio)) { long nr = folio_nr_pages(folio); lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, -nr); + if (folio_test_dropbehind(folio)) + wb_stat_mod(wb, WB_DONTCACHE_DIRTY, -nr); zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr); wb_stat_mod(wb, WB_RECLAIMABLE, -nr); ret = true; From e1bf79628453e6afac81ffa57f4f40f28e5512ff Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 11 May 2026 07:58:29 -0400 Subject: [PATCH 6/6] mm: kick writeback flusher for IOCB_DONTCACHE with targeted dirty tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The IOCB_DONTCACHE writeback path in generic_write_sync() calls filemap_flush_range() on every write, submitting writeback inline in the writer's context. Perf lock contention profiling shows the performance problem is not lock contention but the writeback submission work itself — walking the page tree and submitting I/O blocks the writer for milliseconds, inflating p99.9 latency from 23ms (buffered) to 93ms (dontcache). Replace the inline filemap_flush_range() call with a flusher kick that drains dirty pages in the background. This moves writeback submission completely off the writer's hot path. To avoid flushing unrelated buffered dirty data, add a dedicated WB_start_dontcache bit and wb_check_start_dontcache() handler that uses the per-wb WB_DONTCACHE_DIRTY counter to determine how many pages to write back. The flusher writes back that many pages from the oldest dirty inodes (not restricted to dontcache-specific inodes). This helps preserve I/O batching while limiting the scope of expedited writeback. Like WB_start_all, the WB_start_dontcache bit coalesces multiple DONTCACHE writes into a single flusher wakeup without per-write allocations. Use test_and_clear_bit to atomically consume the kick request before reading the dirty counter and starting writeback, so that concurrent DONTCACHE writes during writeback can re-set the bit and schedule a follow-up flusher run. Read the dirty counter with wb_stat_sum() (aggregating per-CPU batches) rather than wb_stat() (which reads only the global counter) to ensure small writes below the percpu batch threshold are visible to the flusher. In filemap_dontcache_kick_writeback(), set the WB_start_dontcache bit inside the unlocked_inode_to_wb_begin/end section for correct cgroup writeback domain targeting, but defer the wb_wakeup() call until after the section ends, since wb_wakeup() uses spin_unlock_irq() which would unconditionally re-enable interrupts while the i_pages xa_lock may still be held under irqsave during a cgroup writeback switch. Pin the wb with wb_get() inside the RCU critical section before calling wb_wakeup() outside it, since cgroup bdi_writeback structures are RCU-freed and the wb pointer could become invalid after unlocked_inode_to_wb_end() drops the RCU read lock. Also add WB_REASON_DONTCACHE as a new writeback reason for tracing visibility. dontcache-bench results (same host, T6F_SKL_1920GBF, 251 GiB RAM, xfs on NVMe, fio io_uring): Buffered and direct I/O paths are unaffected by this patchset. All improvements are confined to the dontcache path: Single-stream throughput (MB/s): Before After Change seq-write/dontcache 298 897 +201% rand-write/dontcache 131 236 +80% Tail latency improvements (seq-write/dontcache): p99: 135,266 us -> 23,986 us (-82%) p99.9: 8,925,479 us -> 28,443 us (-99.7%) Multi-writer (4 jobs, sequential write): Before After Change dontcache aggregate (MB/s) 2,529 4,532 +79% dontcache p99 (us) 8,553 1,002 -88% dontcache p99.9 (us) 109,314 1,057 -99% Dontcache multi-writer throughput now matches buffered (4,532 vs 4,616 MB/s). 32-file write (Axboe test): Before After Change dontcache aggregate (MB/s) 1,548 3,499 +126% dontcache p99 (us) 10,170 602 -94% Peak dirty pages (MB) 1,837 213 -88% Dontcache now reaches 81% of buffered throughput (was 35%). Competing writers (dontcache vs buffered, separate files): Before After buffered writer 868 433 MB/s dontcache writer 415 433 MB/s Aggregate 1,284 866 MB/s Previously the buffered writer starved the dontcache writer 2:1. With per-bdi_writeback tracking, both writers now receive equal bandwidth. The aggregate matches the buffered-vs-buffered baseline (863 MB/s), indicating fair sharing regardless of I/O mode. The dontcache writer's p99.9 latency collapsed from 119 ms to 33 ms (-73%), eliminating the severe periodic stalls seen in the baseline. Both writers now share identical latency profiles, matching the buffered-vs-buffered pattern. The per-bdi_writeback dirty tracking dramatically reduces peak dirty pages in dontcache workloads, with the 32-file test dropping from 1.8 GB to 213 MB. Dontcache sequential write throughput triples and multi-writer throughput reaches parity with buffered I/O, with tail latencies collapsing by 1-2 orders of magnitude. Assisted-by: Claude:claude-opus-4-6 Signed-off-by: Jeff Layton Link: https://patch.msgid.link/20260511-dontcache-v7-3-2848ddce8090@kernel.org Reviewed-by: Jan Kara Reviewed-by: Ritesh Harjani (IBM) Signed-off-by: Christian Brauner (Amutable) --- fs/fs-writeback.c | 63 ++++++++++++++++++++++++++++++++ include/linux/backing-dev-defs.h | 2 + include/linux/fs.h | 6 +-- include/trace/events/writeback.h | 3 +- 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index b303516f2753..fdb8766d275a 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -2396,6 +2396,27 @@ static long wb_check_start_all(struct bdi_writeback *wb) return nr_pages; } +static long wb_check_start_dontcache(struct bdi_writeback *wb) +{ + long nr_pages; + + if (!test_and_clear_bit(WB_start_dontcache, &wb->state)) + return 0; + + nr_pages = wb_stat_sum(wb, WB_DONTCACHE_DIRTY); + if (nr_pages) { + struct wb_writeback_work work = { + .nr_pages = nr_pages, + .sync_mode = WB_SYNC_NONE, + .range_cyclic = 1, + .reason = WB_REASON_DONTCACHE, + }; + + nr_pages = wb_writeback(wb, &work); + } + + return nr_pages; +} /* * Retrieve work items and do the writeback they describe @@ -2417,6 +2438,11 @@ static long wb_do_writeback(struct bdi_writeback *wb) */ wrote += wb_check_start_all(wb); + /* + * Check for dontcache writeback request + */ + wrote += wb_check_start_dontcache(wb); + /* * Check for periodic writeback, kupdated() style */ @@ -2491,6 +2517,43 @@ void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi, rcu_read_unlock(); } +/** + * filemap_dontcache_kick_writeback - kick flusher for IOCB_DONTCACHE writes + * @mapping: address_space that was just written to + * + * Kick the writeback flusher thread to expedite writeback of dontcache dirty + * pages. Queue writeback for the inode's wb for as many pages as there are + * dontcache pages, but don't restrict writeback to dontcache pages only. + * + * This significantly improves performance over either writing all wb's pages + * or writing only dontcache pages. Although it doesn't guarantee quick + * writeback and reclaim of dontcache pages, it keeps the amount of dirty pages + * in check. Over longer term dontcache pages get written and reclaimed by + * background writeback even with this rough heuristic. + */ +void filemap_dontcache_kick_writeback(struct address_space *mapping) +{ + struct inode *inode = mapping->host; + struct bdi_writeback *wb; + struct wb_lock_cookie cookie = {}; + bool need_wakeup = false; + + wb = unlocked_inode_to_wb_begin(inode, &cookie); + if (wb_has_dirty_io(wb) && + !test_bit(WB_start_dontcache, &wb->state) && + !test_and_set_bit(WB_start_dontcache, &wb->state)) { + wb_get(wb); + need_wakeup = true; + } + unlocked_inode_to_wb_end(inode, &cookie); + + if (need_wakeup) { + wb_wakeup(wb); + wb_put(wb); + } +} +EXPORT_SYMBOL_GPL(filemap_dontcache_kick_writeback); + /* * Wakeup the flusher threads to start writeback of all currently dirty pages */ diff --git a/include/linux/backing-dev-defs.h b/include/linux/backing-dev-defs.h index cb660dd37286..4f1084937315 100644 --- a/include/linux/backing-dev-defs.h +++ b/include/linux/backing-dev-defs.h @@ -26,6 +26,7 @@ enum wb_state { WB_writeback_running, /* Writeback is in progress */ WB_has_dirty_io, /* Dirty inodes on ->b_{dirty|io|more_io} */ WB_start_all, /* nr_pages == 0 (all) work pending */ + WB_start_dontcache, /* dontcache writeback pending */ }; enum wb_stat_item { @@ -56,6 +57,7 @@ enum wb_reason { */ WB_REASON_FORKER_THREAD, WB_REASON_FOREIGN_FLUSH, + WB_REASON_DONTCACHE, WB_REASON_MAX, }; diff --git a/include/linux/fs.h b/include/linux/fs.h index 11559c513dfb..df72b42a9e9b 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2624,6 +2624,7 @@ extern int __must_check file_write_and_wait_range(struct file *file, loff_t start, loff_t end); int filemap_flush_range(struct address_space *mapping, loff_t start, loff_t end); +void filemap_dontcache_kick_writeback(struct address_space *mapping); static inline int file_write_and_wait(struct file *file) { @@ -2657,10 +2658,7 @@ static inline ssize_t generic_write_sync(struct kiocb *iocb, ssize_t count) if (ret) return ret; } else if (iocb->ki_flags & IOCB_DONTCACHE) { - struct address_space *mapping = iocb->ki_filp->f_mapping; - - filemap_flush_range(mapping, iocb->ki_pos - count, - iocb->ki_pos - 1); + filemap_dontcache_kick_writeback(iocb->ki_filp->f_mapping); } return count; diff --git a/include/trace/events/writeback.h b/include/trace/events/writeback.h index bdac0d685a98..13ee076ccd16 100644 --- a/include/trace/events/writeback.h +++ b/include/trace/events/writeback.h @@ -44,7 +44,8 @@ EM( WB_REASON_PERIODIC, "periodic") \ EM( WB_REASON_FS_FREE_SPACE, "fs_free_space") \ EM( WB_REASON_FORKER_THREAD, "forker_thread") \ - EMe(WB_REASON_FOREIGN_FLUSH, "foreign_flush") + EM( WB_REASON_FOREIGN_FLUSH, "foreign_flush") \ + EMe(WB_REASON_DONTCACHE, "dontcache") WB_WORK_REASON