mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-09-12 02:35:41 -04:00
fed0b33e6c
There are some synchronisation issues that derive from the app thread
adding more folios to the rolling buffer whilst the collector thread is
looking at them or trying to clear them, such as determining the setting of
front_folio_order when the next folio hasn't been added yet,
The reason for the rolling buffer approach is that loading the buffer
upfront and then dropping all the refs just acquired is quite a slow
operation, and loading progressively allows some of the cost to be deferred
until after at least some of the I/O is started.
Instead, a better way is to load all the folios into the rolling buffer
upfront - and then drop the refs later, once the I/O is in progress. (Even
better would be for the refs not to be there at all.)
Fix this by changing the rolling buffer loader to load all the folios
selected by the VM for readahead upfront into the folio queue. The folio
queue is allocated a batch worth at a time as we don't know how many folios
are involved (the readahead_control struct, alas, has a page count, not a
folio count).
The folio refs acquired from readahead are then dropped in bulk once the
first subrequest is dispatched as it's quite a slow operation. The
collector waits for NETFS_RREQ_NEED_PUT_RA_REFS to be cleared so that it
doesn't unlock folios before the xarray has been scanned for them.
This simplifies the buffer handling later and isn't noticeably slower as
the xarray doesn't need to be modified and the folios are all already
pre-locked.
Fixes: ee4cdf7ba8 ("netfs: Speed up buffered reading")
Link: https://sashiko.dev/#/patchset/20260824120224.504575-1-dhowells%40redhat.com
Signed-off-by: David Howells <dhowells@redhat.com>
Link: https://patch.msgid.link/20260827134304.2075713-8-dhowells@redhat.com
Acked-by: Paulo Alcantara <pc@manguebit.org>
cc: Paulo Alcantara (Red Hat) <pc@manguebit.org>
cc: Matthew Wilcox <willy@infradead.org>
cc: netfs@lists.linux.dev
cc: linux-mm@kvack.org
cc: linux-fsdevel@vger.kernel.org
Signed-off-by: Christian Brauner (Amutable) <brauner@kernel.org>
62 lines
2.2 KiB
C
62 lines
2.2 KiB
C
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
|
/* Rolling buffer of folios
|
|
*
|
|
* Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
|
|
* Written by David Howells (dhowells@redhat.com)
|
|
*/
|
|
|
|
#ifndef _ROLLING_BUFFER_H
|
|
#define _ROLLING_BUFFER_H
|
|
|
|
#include <linux/folio_queue.h>
|
|
#include <linux/uio.h>
|
|
|
|
/*
|
|
* Rolling buffer. Whilst the buffer is live and in use, folios and folio
|
|
* queue segments can be added to one end by one thread and removed from the
|
|
* other end by another thread. The buffer isn't allowed to be empty; it must
|
|
* always have at least one folio_queue in it so that neither side has to
|
|
* modify both queue pointers.
|
|
*
|
|
* The iterator in the buffer is extended as buffers are inserted. It can be
|
|
* snapshotted to use a segment of the buffer.
|
|
*/
|
|
struct rolling_buffer {
|
|
struct folio_queue *head; /* Producer's insertion point */
|
|
struct folio_queue *tail; /* Consumer's removal point */
|
|
struct iov_iter iter; /* Iterator tracking what's left in the buffer */
|
|
u8 next_head_slot; /* Next slot in ->head */
|
|
u8 first_tail_slot; /* First slot in ->tail */
|
|
};
|
|
|
|
/*
|
|
* Snapshot of a rolling buffer.
|
|
*/
|
|
struct rolling_buffer_snapshot {
|
|
struct folio_queue *curr_folioq; /* Queue segment in which current folio resides */
|
|
unsigned char curr_slot; /* Folio currently being read */
|
|
unsigned char curr_order; /* Order of folio */
|
|
};
|
|
|
|
/* Marks to store per-folio in the internal folio_queue structs. */
|
|
#define ROLLBUF_MARK_1 BIT(0)
|
|
#define ROLLBUF_MARK_2 BIT(1)
|
|
|
|
int rolling_buffer_init(struct rolling_buffer *roll, unsigned int rreq_id,
|
|
unsigned int direction, gfp_t gfp);
|
|
int rolling_buffer_make_space(struct rolling_buffer *roll, gfp_t gfp);
|
|
ssize_t rolling_buffer_bulk_load_from_ra(struct rolling_buffer *roll,
|
|
struct readahead_control *ractl,
|
|
unsigned int rreq_id, gfp_t gfp);
|
|
ssize_t rolling_buffer_append(struct rolling_buffer *roll, struct folio *folio,
|
|
unsigned int flags, gfp_t gfp);
|
|
struct folio_queue *rolling_buffer_delete_spent(struct rolling_buffer *roll);
|
|
void rolling_buffer_clear(struct rolling_buffer *roll);
|
|
|
|
static inline void rolling_buffer_advance(struct rolling_buffer *roll, size_t amount)
|
|
{
|
|
iov_iter_advance(&roll->iter, amount);
|
|
}
|
|
|
|
#endif /* _ROLLING_BUFFER_H */
|