mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 03:35:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

fs/pipe: unify the page pools into a single per-pipe pool

Browse Source 
Pipes keep two separate page caches:
 a) The per-pipe, lock-protected tmp_page[2]
 b) An on-stack anon_pipe_prealloc burst pool of up to eight pages
    filled before the lock

Converge them into a single per-pipe pool (struct anon_pipe_prealloc
embedded in pipe_inode_info) with the same budget as before: up to
PIPE_PREALLOC_MAX (8) pages, trimmed back to PIPE_PREALLOC_KEEP (2)
after each operation. tmp_page[2] is removed.

Pages are still allocated and freed outside pipe->mutex; only the
assignment into the pool is done under it. The pool count is also read
locklessly in the prefill path, so it is annotated __data_racy.

anon_pipe_prefill_and_lock() tops the pool up to the write's page count
-- and returns with pipe->mutex held, so a write acquires the lock only
once.

anon_pipe_trim_and_unlock() trims the pool under that same lock before
dropping it, then frees the excess.

Signed-off-by: Breno Leitao <leitao@debian.org>
Link: https://patch.msgid.link/20260720-b4-pipe-unification-v5-1-9002a3fe5e6d@debian.org
Reviewed-by: Mateusz Guzik <mjguzik@gmail.com>
Reviewed-by: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Christian Brauner (Amutable) <brauner@kernel.org>
This commit is contained in:
Breno Leitao
2026-07-20 03:39:33 -07:00
committed by Christian Brauner
parent c610d2d078
commit f7f4665dc5
2 changed files with 103 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
fs/pipe.c
View File

@@ -111,47 +111,6 @@ void pipe_double_lock(struct pipe_inode_info *pipe1,
pipe_lock(pipe2);
}
#define PIPE_PREALLOC_MAX 8
struct anon_pipe_prealloc {
struct page *pages[PIPE_PREALLOC_MAX];
unsigned int count;
};
/*
* Pre-allocate pages outside pipe->mutex for multi-page writes.
* alloc_page() with GFP_HIGHUSER can sleep in reclaim and runs memcg
* charging; doing it under the mutex stalls a concurrent reader.
*
* Loop alloc_page() instead of alloc_pages_bulk_*(): the bulk path refuses
* __GFP_ACCOUNT under memcg (see commit 8dcb3060d81d "memcg: page_alloc:
* skip bulk allocator for __GFP_ACCOUNT") and silently degrades to a single
* page. A per-page loop keeps memcg accounting and the task NUMA mempolicy
* honoured for every page; the per-call overhead is small compared to the
* pipe->mutex hold-time being shrunk. Any shortfall is covered by the
* in-lock alloc_page() fallback in anon_pipe_get_page().
*/
static void anon_pipe_get_page_prealloc(struct anon_pipe_prealloc *prealloc,
size_t total_len)
{
unsigned int want, i;
struct page *page;
prealloc->count = 0;
if (total_len <= PAGE_SIZE)
return;
want = min_t(unsigned int, DIV_ROUND_UP(total_len, PAGE_SIZE),
PIPE_PREALLOC_MAX);
for (i = 0; i < want; i++) {
page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
if (!page)
break;
prealloc->pages[prealloc->count++] = page;
}
}
static struct page *anon_pipe_prealloc_pop(struct anon_pipe_prealloc *prealloc)
{
if (!prealloc->count)
@@ -162,24 +121,85 @@ static struct page *anon_pipe_prealloc_pop(struct anon_pipe_prealloc *prealloc)
return prealloc->pages[prealloc->count];
}
static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe,
struct anon_pipe_prealloc *prealloc)
/* Push a page to the prealloc pool. Returns true if added, false if full. */
static bool anon_pipe_prealloc_push(struct anon_pipe_prealloc *prealloc,
struct page *page)
{
if (prealloc->count >= PIPE_PREALLOC_MAX)
return false;
prealloc->pages[prealloc->count++] = page;
return true;
}
/*
* Top up the pipe's own pool, then take pipe->mutex and return with it held.
* The shortfall is allocated outside the lock; the push and the caller's write
* then run under a single lock acquisition, avoiding a separate prefill
* lock/unlock cycle. anon_pipe_get_page() drains the pool instead of allocating
* under the lock.
*/
static void anon_pipe_prefill_and_lock(struct pipe_inode_info *pipe, size_t total_len)
{
struct page *pages[PIPE_PREALLOC_MAX];
unsigned int want, have, need, n = 0;
want = min_t(unsigned int, DIV_ROUND_UP(total_len, PAGE_SIZE),
PIPE_PREALLOC_MAX);
/* Unlocked read; the pool is refilled under the lock below. */
have = min_t(unsigned int, READ_ONCE(pipe->prealloc.count), want);
need = want - have;
if (!need) {
mutex_lock(&pipe->mutex);
return;
}
while (n < need) {
struct page *page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
if (!page)
break;
pages[n++] = page;
}
mutex_lock(&pipe->mutex);
while (n && anon_pipe_prealloc_push(&pipe->prealloc, pages[n - 1]))
n--;
/*
* Just flush any extra page that got affected by the TOCTOU
* effect
*/
while (n)
put_page(pages[--n]);
}
/*
* Called with pipe->mutex held. Trim the pool down to PIPE_PREALLOC_KEEP under
* the lock, drop it, then free the excess outside the critical section.
*/
static void anon_pipe_trim_and_unlock(struct pipe_inode_info *pipe)
{
struct page *excess[PIPE_PREALLOC_MAX];
unsigned int nexcess = 0;
while (pipe->prealloc.count > PIPE_PREALLOC_KEEP)
excess[nexcess++] = anon_pipe_prealloc_pop(&pipe->prealloc);
mutex_unlock(&pipe->mutex);
while (nexcess)
put_page(excess[--nexcess]);
}
static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe)
{
struct page *page;
/* Drain prealloc first to keep tmp_page[] hot for later small writes. */
page = anon_pipe_prealloc_pop(prealloc);
/* Drain the prealloc pool before allocating. Called with mutex held. */
page = anon_pipe_prealloc_pop(&pipe->prealloc);
if (page)
return page;
for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
if (pipe->tmp_page[i]) {
page = pipe->tmp_page[i];
pipe->tmp_page[i] = NULL;
return page;
}
}
/* FWIW: This is called with pipe->mutex held */
return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
}
@@ -187,48 +207,11 @@ static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe,
static void anon_pipe_put_page(struct pipe_inode_info *pipe,
struct page *page)
{
if (page_count(page) == 1) {
for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
if (!pipe->tmp_page[i]) {
pipe->tmp_page[i] = page;
return;
}
}
}
put_page(page);
}
/*
* Stash leftover prealloc pages in tmp_page[] so the next write to this
* pipe gets a hot page without entering the allocator.
*/
static void anon_pipe_refill_tmp_pages(struct pipe_inode_info *pipe,
struct anon_pipe_prealloc *prealloc)
{
int i, idx;
if (!prealloc->count)
if (page_count(page) == 1 &&
anon_pipe_prealloc_push(&pipe->prealloc, page))
return;
for (i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
if (pipe->tmp_page[i])
continue;
if (!prealloc->count)
return;
idx = --prealloc->count;
pipe->tmp_page[i] = prealloc->pages[idx];
prealloc->pages[idx] = NULL;
}
}
/* Runs after mutex_unlock() to keep put_page() out of the critical section. */
static void anon_pipe_free_pages(struct anon_pipe_prealloc *prealloc)
{
while (prealloc->count) {
prealloc->count--;
put_page(prealloc->pages[prealloc->count]);
}
put_page(page);
}
static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
@@ -485,7 +468,8 @@ anon_pipe_read(struct kiocb *iocb, struct iov_iter *to)
}
if (pipe_is_empty(pipe))
wake_next_reader = false;
mutex_unlock(&pipe->mutex);
/* Consumed buffers may have refilled the pool; trim it and unlock. */
anon_pipe_trim_and_unlock(pipe);
if (wake_writer)
wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
@@ -524,7 +508,6 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
{
struct file *filp = iocb->ki_filp;
struct pipe_inode_info *pipe = filp->private_data;
struct anon_pipe_prealloc prealloc;
unsigned int head;
ssize_t ret = 0;
size_t total_len = iov_iter_count(from);
@@ -548,9 +531,7 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
if (unlikely(total_len == 0))
return 0;
anon_pipe_get_page_prealloc(&prealloc, total_len);
mutex_lock(&pipe->mutex);
anon_pipe_prefill_and_lock(pipe, total_len);
if (!pipe->readers) {
if ((iocb->ki_flags & IOCB_NOSIGNAL) == 0)
@@ -607,7 +588,7 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
struct page *page;
int copied;
page = anon_pipe_get_page(pipe, &prealloc);
page = anon_pipe_get_page(pipe);
if (unlikely(!page)) {
if (!ret)
ret = -ENOMEM;
@@ -671,11 +652,9 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
wake_next_writer = true;
}
out:
anon_pipe_refill_tmp_pages(pipe, &prealloc);
if (pipe_is_full(pipe))
wake_next_writer = false;
mutex_unlock(&pipe->mutex);
anon_pipe_free_pages(&prealloc);
anon_pipe_trim_and_unlock(pipe);
/*
* If we do do a wakeup event, we do a 'sync' wakeup, because we
@@ -956,10 +935,8 @@ void free_pipe_info(struct pipe_inode_info *pipe)
if (pipe->watch_queue)
put_watch_queue(pipe->watch_queue);
#endif
for (i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
if (pipe->tmp_page[i])
__free_page(pipe->tmp_page[i]);
}
for (i = 0; i < pipe->prealloc.count; i++)
__free_page(pipe->prealloc.pages[i]);
kfree(pipe->bufs);
kfree(pipe);
}

22
include/linux/pipe_fs_i.h
View File

@@ -14,6 +14,9 @@
#define PIPE_BUF_FLAG_LOSS 0x40 /* Message loss happened after this buffer */
#endif
#define PIPE_PREALLOC_MAX 8 /* max pages in prealloc pool */
#define PIPE_PREALLOC_KEEP 2 /* keep at least this many after trim */
/**
* struct pipe_buffer - a linux kernel pipe buffer
* @page: the page containing the data for the pipe buffer
@@ -58,6 +61,21 @@ union pipe_index {
};
};
/**
* struct anon_pipe_prealloc - per-pipe page preallocation pool
* @pages: array of cached pages (pool)
* @count: number of pages currently in the pool
*
* Each pipe keeps a small bounded pool of preallocated pages to reduce
* allocation overhead during writes. The pool is bounded at PIPE_PREALLOC_MAX
* and trimmed down to PIPE_PREALLOC_KEEP after a write completes.
*/
struct anon_pipe_prealloc {
struct page *pages[PIPE_PREALLOC_MAX];
unsigned int __data_racy count;
};
/**
* struct pipe_inode_info - a linux kernel pipe
* @mutex: mutex protecting the whole thing
@@ -68,7 +86,7 @@ union pipe_index {
* @max_usage: The maximum number of slots that may be used in the ring
* @ring_size: total number of buffers (should be a power of 2)
* @nr_accounted: The amount this pipe accounts for in user->pipe_bufs
* @tmp_page: cached released page
* @prealloc: per-pipe page preallocation pool
* @readers: number of current readers of this pipe
* @writers: number of current writers of this pipe
* @files: number of struct file referring this pipe (protected by ->i_lock)
@@ -99,7 +117,7 @@ struct pipe_inode_info {
#ifdef CONFIG_WATCH_QUEUE
bool note_loss;
#endif
struct page *tmp_page[2];
struct anon_pipe_prealloc prealloc;
struct fasync_struct *fasync_readers;
struct fasync_struct *fasync_writers;
struct pipe_buffer *bufs;